diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4a0375..60af926 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,8 @@ jobs: run: | python3 -m py_compile \ plugin/skills/behavior-diff/scripts/decisions.py \ - plugin/skills/behavior-diff/scripts/render.py + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py - name: Run hook unit checks run: bash tests/hooks-test.sh diff --git a/CODING_GUIDELINES.md b/CODING_GUIDELINES.md index 87f4605..94eee52 100644 --- a/CODING_GUIDELINES.md +++ b/CODING_GUIDELINES.md @@ -154,7 +154,8 @@ shellcheck \ tests/*.sh python3 -m py_compile \ plugin/skills/behavior-diff/scripts/decisions.py \ - plugin/skills/behavior-diff/scripts/render.py + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py bash tests/hooks-test.sh python3 plugin/skills/behavior-diff/scripts/decisions.py --check bash tests/live-report-contract.sh diff --git a/plans/2026-09-04-report-rendering-structure-implementation.md b/plans/2026-09-04-report-rendering-structure-implementation.md new file mode 100644 index 0000000..941957a --- /dev/null +++ b/plans/2026-09-04-report-rendering-structure-implementation.md @@ -0,0 +1,1023 @@ +# Report Rendering Structure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split report data loading, shared copy, Markdown rendering, HTML rendering, CSS, and file output while preserving the three current report files byte-for-byte. + +**Architecture:** Keep `render.py` as the only CLI and writer. Build one typed `ReportData` through `reporting/load.py`, serialize it as internal schema version 1, and pass the same object to pure Markdown and HTML renderers. Keep shared wording in `reporting/content.py`; keep format structure in the renderers; keep CSS in a source file that is inlined into standalone HTML. + +**Tech Stack:** Python 3 standard library, Bash 3.2 contract tests, JSON, HTML/CSS, existing Ruff and deterministic repository checks. + +**Design:** `plans/2026-09-04-report-rendering-structure.md` + +--- + +## File structure + +Create one private package beside the existing scripts: + +```text +plugin/skills/behavior-diff/scripts/ +├── render.py # compatible CLI and only file writer +└── reporting/ + ├── __init__.py # marks the private package + ├── schema.py # ReportData records and JSON conversion + ├── content.py # shared report wording + ├── load.py # persisted evidence -> ReportData + ├── render_markdown.py # ReportData -> Markdown string + ├── render_html.py # ReportData + CSS -> HTML strings + └── report.css # CSS source with one result-color token +``` + +Tests and fixed synthetic output: + +```text +tests/ +├── live-report-contract.sh +├── report-schema-test.py +└── fixtures/report-rendering/ + ├── captured/ + │ ├── report.md + │ ├── report.html + │ └── report-artifact.html + └── self-reported/ + ├── report.md + ├── report.html + └── report-artifact.html +``` + +Do not modify `behavior-diff.sh`, `run-trial.sh`, `decisions.py`, either public skill, the plugin manifests, or README. Their current contracts remain valid. + +--- + +### Task 1: Freeze the current visible reports + +**Files:** +- Modify: `tests/live-report-contract.sh:19-105,328-444` +- Create: `tests/fixtures/report-rendering/captured/report.md` +- Create: `tests/fixtures/report-rendering/captured/report.html` +- Create: `tests/fixtures/report-rendering/captured/report-artifact.html` +- Create: `tests/fixtures/report-rendering/self-reported/report.md` +- Create: `tests/fixtures/report-rendering/self-reported/report.html` +- Create: `tests/fixtures/report-rendering/self-reported/report-artifact.html` + +- [ ] **Step 1: Add an explicit fixture-update mode to the contract test** + +At the top of `tests/live-report-contract.sh`, accept either no argument or exactly `--update-report-fixtures`: + +```bash +if (( $# > 1 )); then + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 +fi + +update_report_fixtures=false +case ${1:-} in + "") ;; + --update-report-fixtures) update_report_fixtures=true ;; + *) + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 + ;; +esac +``` + +Add these helpers after `progress()`: + +```bash +fixture_root=$here/fixtures/report-rendering + +copy_report_fixtures() { + local run=$1 + local kind=$2 + local out=$fixture_root/$kind + mkdir -p "$out" + cp "$run/report.md" "$out/report.md" + cp "$run/report.html" "$out/report.html" + cp "$run/report-artifact.html" "$out/report-artifact.html" +} + +require_exact_report() { + local actual=$1 + local expected=$2 + local message=$3 + if ! cmp -s "$expected" "$actual"; then + diff -u "$expected" "$actual" >&2 || true + fail "$message" + fi +} +``` + +After the current renderer calls, update only when the explicit flag is present, then compare all three files for both modes: + +```bash +if $update_report_fixtures; then + copy_report_fixtures "$captured_run" captured + copy_report_fixtures "$self_run" self-reported +fi + +for kind in captured self-reported; do + case $kind in + captured) actual_run=$captured_run ;; + self-reported) actual_run=$self_run ;; + esac + for report_name in report.md report.html report-artifact.html; do + require_exact_report \ + "$actual_run/$report_name" \ + "$fixture_root/$kind/$report_name" \ + "$kind $report_name changed" + done +done +``` + +Do not remove the existing focused assertions. They explain whether a mismatch concerns provenance, ordering, escaping, labels, decisions, flow, or answers. + +- [ ] **Step 2: Generate the goldens from the current renderer** + +Run: + +```bash +bash tests/live-report-contract.sh --update-report-fixtures +``` + +Expected: `ok — live report contract passed`, with six new synthetic fixture files. Review every fixture for invented content only: `Live contract`, `Original project instructions`, `Updated project instructions`, `Before answer`, `After answer`, and the synthetic decision row. + +- [ ] **Step 3: Verify normal mode cannot rewrite fixtures** + +Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ok — live report contract passed` and no fixture changes. + +- [ ] **Step 4: Commit the characterization boundary** + +```bash +git add tests/live-report-contract.sh tests/fixtures/report-rendering +git commit --signoff -m "test: freeze report rendering output" +``` + +--- + +### Task 2: Define the typed report schema + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/__init__.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/schema.py` +- Create: `tests/report-schema-test.py` +- Modify: `tests/live-report-contract.sh` + +- [ ] **Step 1: Write the failing schema test** + +Create `tests/report-schema-test.py`. It must import the private package from the shipped scripts directory, construct a complete synthetic schema-v1 dictionary, load it with `ReportData.from_dict`, and compare the deterministic round trip: + +```python +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +scripts = Path(__file__).resolve().parents[1] / "plugin/skills/behavior-diff/scripts" +sys.path.insert(0, str(scripts)) + +from reporting.schema import ReportData # noqa: E402 + + +def main(): + source = Path(sys.argv[1]) if len(sys.argv) == 2 else None + if source: + raw = json.loads(source.read_text()) + else: + raw = { + "schema_version": 1, + "metadata": { + "model": "synthetic/model", + "mode": "review", + "vocab": "generic", + "trace_source": "captured", + "target_file": "AGENTS.md", + "before_label": "current file", + "after_label": "your change applied", + }, + "content": { + "title": "Synthetic report", + "subtitle": "Synthetic subtitle.", + "observation": "", + "scenario_heading": "Scenario", + "scenario": "Compare two synthetic files.", + "expected_heading": "Expected behavior", + "expected": None, + "diff_heading": "Diff of AGENTS.md — the only difference between the variants", + "decision_heading": "Decision diff — top divergences", + "decision_blurb": "Synthetic decision explanation.", + "flow_heading": "Flow diff — where the variants diverge", + "result_heading": "Result", + "boundary": "Synthetic evidence only.", + }, + "rule_diff": "--- before\n+++ after\n", + "result": {"text": "No automatic verdict", "kind": "neutral"}, + "variants": { + "before": { + "label": "Before", + "note": "current file", + "passed": 0, + "blocked": 0, + "valid": 1, + "total": 1, + "count_text": "1 valid trial(s)", + "count_suffix": " (blocked: 0)", + "count_emphasized": False, + "trials": [ + { + "name": "before-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md"], + "final": "Before answer", + "outcome": None, + } + ], + }, + "after": { + "label": "After", + "note": "your change applied", + "passed": 0, + "blocked": 0, + "valid": 1, + "total": 1, + "count_text": "1 valid trial(s)", + "count_suffix": " (blocked: 0)", + "count_emphasized": False, + "trials": [ + { + "name": "after-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md", "run tests"], + "final": "After answer", + "outcome": None, + } + ], + }, + }, + "command_flow": { + "enabled": True, + "same": False, + "shared": ["Read files"], + "before": {"prefix": [], "paths": [], "total": 1}, + "after": { + "prefix": ["Run tests"], + "paths": [], + "total": 1, + }, + }, + "decisions": { + "rows": [], + "fork": None, + "fork_note": "", + "dropped": 0, + "extractor": "", + "before_count": 1, + "after_count": 1, + }, + } + + report = ReportData.from_dict(raw) + assert report.schema_version == 1 + assert report.to_dict() == raw + encoded = report.to_json() + assert encoded == json.dumps(raw, indent=2, sort_keys=True) + "\n" + + bad = dict(raw, schema_version=2) + try: + ReportData.from_dict(bad) + except ValueError as error: + assert str(error) == "unsupported report-data schema version: 2" + else: + raise AssertionError("schema version 2 was accepted") + + +if __name__ == "__main__": + main() +``` + +Call it from `tests/live-report-contract.sh` before the fixture runs: + +```bash +python3 "$here/report-schema-test.py" +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: nonzero with `ModuleNotFoundError: No module named 'reporting'`. + +- [ ] **Step 3: Implement the immutable schema records** + +Create an empty `reporting/__init__.py` and implement frozen dataclasses in `reporting/schema.py`: + +```python +from typing import Dict, Optional, Tuple, Union + +SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class TrialData: + name: str + verdict: str + actions: str + commands: Tuple[str, ...] + final: str + outcome: Optional[str] + + +@dataclass(frozen=True) +class VariantData: + label: str + note: str + passed: int + blocked: int + valid: int + total: int + count_text: str + count_suffix: str + count_emphasized: bool + trials: Tuple[TrialData, ...] + + +@dataclass(frozen=True) +class VariantsData: + before: VariantData + after: VariantData + + +@dataclass(frozen=True) +class FlowPathData: + steps: Tuple[str, ...] + count: int + + +@dataclass(frozen=True) +class FlowBranchData: + prefix: Tuple[str, ...] + paths: Tuple[FlowPathData, ...] + total: int + + +@dataclass(frozen=True) +class CommandFlowData: + enabled: bool + same: bool + shared: Tuple[str, ...] + before: FlowBranchData + after: FlowBranchData + + +@dataclass(frozen=True) +class DecisionChoiceData: + choice: str + count: int + + +@dataclass(frozen=True) +class DecisionRowData: + decision: str + topic: str + anchor: Union[int, str] + diverges: bool + note: str + before: Tuple[DecisionChoiceData, ...] + after: Tuple[DecisionChoiceData, ...] + + +@dataclass(frozen=True) +class DecisionData: + rows: Tuple[DecisionRowData, ...] + fork: Optional[int] + fork_note: str + dropped: int + extractor: str + before_count: int + after_count: int + + +@dataclass(frozen=True) +class MetadataData: + model: str + mode: str + vocab: str + trace_source: str + target_file: str + before_label: str + after_label: str + + +@dataclass(frozen=True) +class ContentData: + title: str + subtitle: str + observation: str + scenario_heading: str + scenario: str + expected_heading: str + expected: Optional[str] + diff_heading: str + decision_heading: str + decision_blurb: str + flow_heading: str + result_heading: str + boundary: str + + +@dataclass(frozen=True) +class ResultData: + text: str + kind: str + + +@dataclass(frozen=True) +class ReportData: + schema_version: int + metadata: MetadataData + content: ContentData + rule_diff: str + result: ResultData + variants: VariantsData + command_flow: CommandFlowData + decisions: DecisionData +``` + +Implement `ReportData.from_dict`, `to_dict`, and `to_json`. Rebuild every nested dataclass explicitly in `from_dict`; do not store unvalidated nested dictionaries. Use `dataclasses.asdict`, then recursively convert tuples to lists so `to_dict` contains only JSON-shaped values; do not serialize and parse JSON to perform that conversion. Convert the `DecisionChoiceData.count` field from the persisted `decisions.json` key `n` while loading; `report-data.json` itself uses `count` consistently. + +`to_json` must be exactly: + +```python +def to_json(self): + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" +``` + +- [ ] **Step 4: Run the focused tests and verify GREEN** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: both exit `0`; the six visible output fixtures remain exact. + +- [ ] **Step 5: Commit the schema** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/__init__.py \ + plugin/skills/behavior-diff/scripts/reporting/schema.py \ + tests/report-schema-test.py tests/live-report-contract.sh +git commit --signoff -m "feat: define internal report data schema" +``` + +--- + +### Task 3: Extract report loading and shared copy + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/content.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/load.py` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:16-448,918-923` +- Modify: `plugin/skills/behavior-diff/scripts/reporting/schema.py` +- Modify: `tests/live-report-contract.sh` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add failing generated-data assertions** + +After each current renderer invocation in `tests/live-report-contract.sh`, require the new artifact and validate it through the shipped schema: + +```bash +for report_run in "$captured_run" "$self_run"; do + [[ -f $report_run/report-data.json ]] || + fail "renderer did not write report-data.json: $report_run" + python3 "$here/report-schema-test.py" "$report_run/report-data.json" +done +``` + +Add focused JSON checks: + +```bash +[[ $(jq -r '.schema_version' "$captured_run/report-data.json") == 1 ]] || + fail 'captured report-data schema version changed' +[[ $(jq -r '.metadata.trace_source' "$captured_run/report-data.json") == captured ]] || + fail 'captured report-data lost its provenance' +[[ $(jq -r '.metadata.trace_source' "$self_run/report-data.json") == self-reported ]] || + fail 'self-reported report-data lost its provenance' +[[ $(jq -r '.variants.after.trials[0].commands[1]' \ + "$captured_run/report-data.json") == \ + 'Test: bash behavior-diff/tests/live-report-contract.sh' ]] || + fail 'report-data changed trial command order' +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `FAIL: renderer did not write report-data.json`. + +- [ ] **Step 3: Move shared copy into `content.py`** + +Move the current default subtitle, boundary, result matrix, count wording, decision explanation, observation wording, and section headings behind these pure functions: + +- `result_data(mode: str, self_reported: bool, before: VariantData, after: VariantData) -> ResultData` copies the current result matrix and single-run suffix from `render.py:250-271`. +- `count_data(mode: str, passed: int, valid: int, blocked: int) -> Tuple[str, str, bool]` returns plain main count text, the plain blocked-count suffix, and whether Markdown emphasizes only the main text. This keeps shared copy format-neutral while preserving the graded report's exact emphasis boundary. +- `decision_blurb(self_reported: bool, single_trial: bool) -> str` copies `DEC_BLURB` and `DEC_N1` from `render.py:315-364`. +- `observation(decisions: DecisionData) -> str` copies the fork observation from `render.py:438-447`. +- `build_content(config: Dict[str, object], target_file: str, scenario: str, result: ResultData, observation_text: str, decision_text: str) -> ContentData` applies current defaults and builds the shared headings and boundary. + +The returned strings must be copied exactly from the current `render.py`. Do not edit punctuation, capitalization, spacing, or warnings. + +- [ ] **Step 4: Move evidence loading and comparison into `load.py`** + +Expose one public function: `load_report(run: Path, capsule: Path, model: str, config_path: Optional[Path]) -> ReportData`. + +Move the current logic into private functions with explicit arguments: + +- `_read_config(config_path)` reads JSON and applies the existing defaults. +- `_read_grades(run)` parses the required TSV rows without changing its strict split behavior. +- `_read_trial(run, name, grade)` reads ordered tools and the final result from canonical JSONL. +- `_classify(command, vocab)` contains the current demo, generic, and Spacedock buckets. +- `_trial_sequence(trial, mode, vocab)` adds the current optional outcome step. +- `_common_prefix(sequences)` and `_build_flow(before_trials, after_trials, mode, trace_source, vocab)` preserve the current flow algorithm and counts. +- `_read_rule_diff(run, capsule, target_file)` keeps target-file and `rule.md` fallback behavior. +- `_read_decisions(run, before_total, after_total, self_reported)` converts every valid decision choice from persisted key `n` to `DecisionChoiceData.count`. + +Preserve these exact behaviors: + +- invalid `trace_source` exits with `trace_source must be either "captured" or "self-reported"`; +- malformed trace lines are skipped; +- the last non-empty result remains the final answer; +- missing or malformed `decisions.json` produces an empty `DecisionData`; +- trial names retain lexical order; +- self-reported runs disable command flow; +- all current vocab classifiers and result outcomes remain; +- run and capsule may be different paths. + +Construct every `ReportData` field only after all required inputs and derived facts are available. Set `schema_version=SCHEMA_VERSION`; do not use a partial constructor or default missing evidence. + +- [ ] **Step 5: Use `ReportData` in `render.py` without moving format blocks yet** + +At the top-level command path: + +```python +report = load_report(run, capsule, model, config_path) +``` + +Temporarily unpack the exact fields needed by the existing Markdown and HTML blocks. This bridge is private to `render.py` and must be deleted in Task 6. Do not add compatibility methods or aliases to `ReportData`. + +After both current format strings are successfully built, add `report-data.json` to the write phase: + +```python +report_json = report.to_json() +# Build Markdown and both HTML strings first. +(run / "report-data.json").write_text(report_json) +``` + +Do not change the three existing report strings or stdout. + +- [ ] **Step 6: Verify the data boundary and exact visible output** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: `report-data.json` checks pass, and all six visible fixtures remain byte-for-byte exact. + +- [ ] **Step 7: Commit loading and shared content** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/content.py \ + plugin/skills/behavior-diff/scripts/reporting/load.py \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py tests/live-report-contract.sh +git commit --signoff -m "refactor: separate report data assembly" +``` + +--- + +### Task 4: Extract the Markdown renderer + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/render_markdown.py` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:449-542` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add a failing pure-renderer check** + +Extend `tests/report-schema-test.py` when a `report-data.json` path is supplied: + +```python +from reporting.render_markdown import render_markdown + +expected_markdown = source.with_name("report.md").read_text() +assert render_markdown(report) == expected_markdown +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ModuleNotFoundError: No module named 'reporting.render_markdown'`. + +- [ ] **Step 3: Move Markdown assembly behind one pure function** + +Create: + +```python +def render_markdown(report: ReportData) -> str: + parts = [] + # Existing Markdown assembly, with ReportData attributes instead of globals. + return "\n".join(parts) +``` + +Move all Markdown-only behavior into this module: + +- Markdown headings and section order; +- diff fence; +- decision lists; +- `md_branch` flow formatting; +- conditional flow `
`; +- trial `
` blocks; +- no `$ ` prefix for captured commands. + +Keep the current strings and blank lines exactly. The function must not access the filesystem or mutate `ReportData`. + +Replace the old Markdown block in `render.py` with: + +```python +markdown = render_markdown(report) +``` + +Do not write it until all remaining format strings have been built. + +- [ ] **Step 4: Verify the pure function and golden output** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: the pure function equals each generated `report.md`, and both Markdown goldens remain exact. + +- [ ] **Step 5: Commit the Markdown renderer** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/render_markdown.py \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py +git commit --signoff -m "refactor: isolate Markdown report rendering" +``` + +--- + +### Task 5: Extract the HTML renderer and CSS + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/render_html.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/report.css` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:544-916` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add failing pure HTML checks** + +Extend the path-backed branch in `tests/report-schema-test.py`: + +```python +from reporting.render_html import render_artifact, render_document + +css_path = scripts / "reporting/report.css" +css = css_path.read_text() +artifact = render_artifact(report, css) +assert artifact == source.with_name("report-artifact.html").read_text() +assert render_document(artifact) == source.with_name("report.html").read_text() +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ModuleNotFoundError: No module named 'reporting.render_html'`. + +- [ ] **Step 3: Extract the CSS source with one explicit result-color token** + +Copy the generated CSS text between the current `` into `reporting/report.css`. The file starts with `:root {`, uses normal single CSS braces, and ends with one newline after the final `}`. + +Replace only the current dynamic result background value with: + +```css +.result { background:__RESULT_BG__; color:#fff; border-radius:8px; +``` + +Keep every other declaration, space, and line break unchanged. Keep the Google Fonts `` in the HTML renderer. + +- [ ] **Step 4: Move HTML assembly behind pure functions** + +Expose `render_artifact(report: ReportData, css: str) -> str` and `render_document(artifact: str) -> str`. + +Resolve the one CSS token inside the HTML renderer: + +```python +RESULT_BACKGROUNDS = { + "good": "var(--pass)", + "bad": "var(--fail)", + "neutral": "var(--accent)", +} + + +def _resolve_css(css, result_kind): + if css.count("__RESULT_BG__") != 1: + raise ValueError("report.css must contain __RESULT_BG__ exactly once") + return css.replace("__RESULT_BG__", RESULT_BACKGROUNDS[result_kind]) + + +def render_document(artifact): + return ( + '' + '' + "" + artifact + "" + ) +``` + +`render_artifact` must call `_resolve_css` and insert the result as `\n`. This reproduces the current generated bytes while keeping the source CSS editable. + +Move `card`, `lane`, `branch_html`, `dec_choices`, `dec_label`, decision HTML, flow HTML, trial columns, escaping, footer, and body assembly into `render_html.py`. Keep captured `$ ` prefixes, self-reported labels, open review-mode answers, HTML escaping, section order, and all source whitespace exact. + +The functions must not read or write files. Pass CSS explicitly. + +Replace the old HTML block in `render.py` with: + +```python +css = (Path(__file__).parent / "reporting/report.css").read_text() +artifact_html = render_artifact(report, css) +html_document = render_document(artifact_html) +``` + +- [ ] **Step 5: Verify both HTML outputs exactly** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: pure renderer strings equal the generated files; captured and self-reported HTML and artifact goldens remain exact. + +- [ ] **Step 6: Commit the HTML renderer and style source** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/render_html.py \ + plugin/skills/behavior-diff/scripts/reporting/report.css \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py +git commit --signoff -m "refactor: isolate HTML report rendering" +``` + +--- + +### Task 6: Reduce `render.py` to orchestration and writes + +**Files:** +- Modify: `plugin/skills/behavior-diff/scripts/render.py` +- Modify: `.github/workflows/ci.yml:76-81` +- Modify: `CODING_GUIDELINES.md:155-157` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add a failing import-safety check** + +In `tests/report-schema-test.py`, load `render.py` without command arguments inside a fresh temporary directory: + +```python +import importlib.util +import io +import os +from contextlib import redirect_stderr, redirect_stdout +from tempfile import TemporaryDirectory + +with TemporaryDirectory() as tmp: + stdout = io.StringIO() + stderr = io.StringIO() + old_cwd = os.getcwd() + try: + os.chdir(tmp) + spec = importlib.util.spec_from_file_location( + "behavior_diff_render", scripts / "render.py" + ) + module = importlib.util.module_from_spec(spec) + with redirect_stdout(stdout), redirect_stderr(stderr): + spec.loader.exec_module(module) + finally: + os.chdir(old_cwd) + assert list(Path(tmp).iterdir()) == [] + assert stdout.getvalue() == "" + assert stderr.getvalue() == "" + assert callable(module.main) +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +python3 tests/report-schema-test.py +``` + +Expected: the current top-level renderer exits or reads `sys.argv`; `main` is not available. + +- [ ] **Step 3: Implement the thin import-safe facade** + +`render.py` must contain only imports, argument parsing, orchestration, writes, stdout, and `main()`: + +```python +def main(argv=None): + args = sys.argv[1:] if argv is None else argv + run = Path(args[0]).resolve() + capsule = Path(args[1]).resolve() + model = args[2] + config_path = Path(args[3]) if len(args) > 3 else None + + report = load_report(run, capsule, model, config_path) + report_json = report.to_json() + markdown = render_markdown(report) + css = (Path(__file__).parent / "reporting/report.css").read_text() + artifact_html = render_artifact(report, css) + html_document = render_document(artifact_html) + + (run / "report-data.json").write_text(report_json) + (run / "report.md").write_text(markdown) + (run / "report-artifact.html").write_text(artifact_html) + (run / "report.html").write_text(html_document) + + before = report.variants.before + after = report.variants.after + print( + f"mode {report.metadata.mode} · BEFORE pass {before.passed}/{before.valid} · " + f"AFTER pass {after.passed}/{after.valid} → {report.result.text}" + ) + print(f"report: {run / 'report.md'}") + print(f"page: {run / 'report.html'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +Keep the current natural Python errors for missing positional arguments and unreadable required files. Do not add a new CLI parser or change diagnostics in this refactor. + +Delete all temporary field-unpacking code and every moved helper from `render.py`. There must be one implementation of each loader, copy rule, Markdown section, HTML component, and CSS declaration. + +- [ ] **Step 4: Expand deterministic Python compilation** + +In `.github/workflows/ci.yml` and `CODING_GUIDELINES.md`, keep the existing files and add: + +```text +plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +The CI command becomes: + +```bash +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +- [ ] **Step 5: Verify the facade and focused report contracts** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +Expected: all commands exit `0`; all visible output remains exact; import produces no files or stdout. + +- [ ] **Step 6: Commit the final cutover** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/render.py \ + .github/workflows/ci.yml CODING_GUIDELINES.md \ + tests/report-schema-test.py +git commit --signoff -m "refactor: make report rendering import safe" +``` + +--- + +### Task 7: Run complete verification and inspect the real surfaces + +**Files:** +- Verify only; no planned production changes. + +- [ ] **Step 1: Run formatting checks** + +```bash +docker run --rm -v "$PWD:/mnt" -w /mnt \ + mvdan/shfmt:v3.14.0 -d -i 2 -ci . +uvx ruff@0.16.5 format --check --diff . +``` + +Expected: no Bash diff; Ruff reports every Python file already formatted. + +- [ ] **Step 2: Run syntax and static checks** + +```bash +bash -n \ + .github/scripts/*.sh \ + bin/behavior-diff \ + plugin/scripts/*.sh \ + plugin/skills/behavior-diff/scripts/*.sh \ + tests/*.sh +shellcheck \ + .github/scripts/*.sh \ + bin/behavior-diff \ + plugin/scripts/*.sh \ + plugin/skills/behavior-diff/scripts/*.sh \ + tests/*.sh +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +Expected: every command exits `0` with no diagnostics. + +- [ ] **Step 3: Run the complete deterministic suite** + +```bash +bash tests/hooks-test.sh +python3 plugin/skills/behavior-diff/scripts/decisions.py --check +bash tests/live-report-contract.sh +bash tests/release-workflow-test.sh +git diff --check +``` + +Expected: all four repository checks pass and `git diff --check` has no output. + +- [ ] **Step 4: Verify exact fixture stability independently** + +Render the fixed captured and self-reported inputs through `tests/live-report-contract.sh`, then confirm the six comparisons ran without fixture-update mode. Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: the test passes without changing any file under `tests/fixtures/report-rendering/`. + +- [ ] **Step 5: Inspect both output formats from one synthetic run** + +Open `tests/fixtures/report-rendering/captured/report.html` in the browser and verify: + +```text +title and subtitle render +scenario and instruction diff render +decision diff and command-derived flow render +before and after columns render +commands and final answers render in order +result and footer render +page has the same current typography, colors, spacing, and responsive layout +``` + +Read `tests/fixtures/report-rendering/captured/report.md` and verify the same evidence appears in the current section order. This is a visual/behavioral check, not a live model call. + +- [ ] **Step 6: Confirm repository scope** + +Verify: + +```text +render.py is a thin facade +reporting modules have one responsibility each +report-data.json is marked internal and versioned +no duplicate old renderer blocks remain +behavior-diff.sh and both skills are unchanged +no real run, transcript, report, credential, or customer data is tracked +all commits contain DCO sign-off +``` + +- [ ] **Step 7: Request independent review before a PR** + +Give one read-only reviewer the complete branch diff and require `REVIEWER_GUIDELINES.md`. Ask it to check exact-output evidence, schema boundaries, import safety, error/fallback preservation, CSS inlining, self-reported provenance, duplicate code, and unauthorized scope. Do not create or update a PR without an `APPROVE` verdict. diff --git a/plans/2026-09-04-report-rendering-structure.md b/plans/2026-09-04-report-rendering-structure.md new file mode 100644 index 0000000..dc55eb6 --- /dev/null +++ b/plans/2026-09-04-report-rendering-structure.md @@ -0,0 +1,303 @@ +# Report Rendering Structure Design + +**Status:** Approved in conversation on 2026-09-04 + +## Goal + +Make the Behavior Diff report pipeline easy to change without coupling report data, shared wording, Markdown structure, HTML structure, and HTML style. + +This change restructures report generation only. For fixed synthetic inputs, the existing `report.md`, `report.html`, and `report-artifact.html` outputs must remain byte-for-byte unchanged. + +## Problem + +`plugin/skills/behavior-diff/scripts/render.py` is one 923-line script. It currently: + +1. reads config, grades, traces, the task, the instruction diff, and optional decisions; +2. computes trial and variant facts; +3. chooses result and explanatory wording; +4. builds command and decision flows; +5. renders Markdown; +6. renders HTML and embedded CSS; +7. writes all report files. + +Most of this work happens at module load through global values. The HTML and Markdown paths repeat decision and flow logic. Some data is already formatted for Markdown and then changed for HTML. For example, HTML removes Markdown `**` markers from a shared count string. + +This makes a small style or wording change risky. A change in one format can affect data calculation or the other format. + +## Current input boundary + +The existing persisted run files remain the source of evidence: + +- `config.json` +- `task.md` +- `grades.tsv` +- each trial's canonical `trace.jsonl` +- the before and after target files, or `rule.md` as the existing fallback +- optional `decisions.json` + +`behavior-diff.sh` and `behavior-diff-live` both produce this shape. The restructure must not create a second pipeline for either skill. + +## Accepted constraints + +- Keep the current `render.py RUN_DIR CAPSULE_DIR MODEL [CONFIG_JSON]` command. +- Keep its stdout text, required-input errors, config errors, and optional-decisions fallback. +- Keep writing `report.md`, `report-artifact.html`, and `report.html`. +- Add `report-data.json` as an internal, versioned artifact. +- Keep one in-memory typed report structure and serialize that same structure to JSON. +- Keep all generated reports deterministic. +- Keep Python standard-library only. +- Keep `report.html` as one portable file. Maintain CSS separately in the source tree, then inline it during generation. +- Keep shared wording in one content layer by default. +- Allow a renderer to own wording only when the formats intentionally differ. +- Preserve captured and self-reported evidence rules. +- Do not change trials, grading, decision extraction, report opening, report content, or visual style. +- Keep latent graded/demo mode and the existing separate run/capsule input contract in this phase. + +## Chosen architecture + +Use one layered Python package behind the existing `render.py` entry point. + +```text +run artifacts + config.json · task.md · grades.tsv · trace.jsonl · decisions.json + | + v +reporting/load.py + parse evidence · compute counts · flow · result · diff + | + v +reporting/schema.py + reporting/content.py + typed ReportData · shared wording · schema_version: 1 + | + +----------------> report-data.json + | + +----------------> reporting/render_markdown.py -> report.md + | + +----------------> reporting/render_html.py + report.css + -> report-artifact.html + -> report.html + +render.py remains the only command entry point and file writer. +``` + +### `reporting/schema.py` + +Own the typed report structure. Do not use `model.py`; “model” is easy to confuse with an AI model. + +The main type is `ReportData`. Nested immutable records cover: + +- configuration and metadata; +- trials and variants; +- result state; +- command-derived flow; +- decision rows and choices; +- shared report content. + +Variant count copy stays format-neutral: `count_text` and `count_suffix` are +separate fields, and `count_emphasized` tells Markdown whether to emphasize +only the main text. This preserves the graded report's emphasis boundary +without putting Markdown in shared content or parsing prose in a renderer. + +`schema.py` also owns deterministic `to_dict` and `from_dict` conversion. It does not read files, write files, render markup, or call an AI model. + +### `reporting/load.py` + +Read the current run artifacts and build `ReportData`. + +It owns the format-neutral behavior currently mixed into `render.py`: + +- canonical trace parsing; +- variant totals and blocked counts; +- result kind selection; +- command classification and flow calculation; +- target-file diff calculation; +- optional decision loading and validation fallback; +- the observed single-run summary facts. + +It imports shared copy functions from `content.py`. It does not create Markdown, HTML, CSS, or output files. + +### `reporting/content.py` + +Own shared human-readable copy: + +- title and subtitle defaults; +- section names; +- result text; +- count text; +- observation text; +- the decision explanation; +- the simulation boundary. + +HTML and Markdown use this copy unless the current outputs intentionally differ. Existing intentional differences remain renderer-owned. These include HTML's captured-command `$ ` prefix, the formats' different heading markup, and their currently different flow explanations. + +### `reporting/render_markdown.py` + +Expose one pure function: + +```python +render_markdown(report: ReportData) -> str +``` + +It owns only Markdown presentation: + +- section order; +- headings; +- fenced blocks; +- Markdown lists; +- `
` blocks already used by the current Markdown report; +- Markdown-specific command display. + +It does not read run files or write `report.md`. + +### `reporting/render_html.py` + +Expose pure functions: + +```python +render_artifact(report: ReportData, css: str) -> str +render_document(artifact: str) -> str +``` + +It owns: + +- HTML escaping; +- the report DOM and component layout; +- HTML-only command display; +- the embedded style element; +- the standalone document wrapper. + +It does not read run files or write output files. + +### `reporting/report.css` + +Contain the current CSS in generated form, with one `__RESULT_BG__` token where the result color varies by `result.kind`. `render.py` reads this source and passes it to the HTML renderer. The renderer must find the token exactly once, replace it with the current allowed CSS variable, and inline the result. Generated HTML stays byte-for-byte compatible and portable. + +### `render.py` + +Remain the compatible command and the only file-writing layer: + +1. parse the existing arguments; +2. call `load.py` to build `ReportData`; +3. serialize the data deterministically; +4. render Markdown, artifact HTML, and standalone HTML in memory; +5. after every renderer succeeds, write all four files; +6. print the existing summary and output paths unchanged. + +Building every output string before writing prevents a renderer error from pairing one newly written report with stale files from another format. + +## Internal `report-data.json` contract + +`report-data.json` is internal. It is not a public plugin API and has no cross-release compatibility promise. It must contain `"schema_version": 1` so repository code can reject or migrate a different shape deliberately. + +The top-level shape is: + +```text +schema_version +metadata +content +rule_diff +result +variants +command_flow +decisions +``` + +The data must contain the facts and shared copy needed to regenerate both visible formats. It must not contain rendered Markdown, rendered HTML, CSS, filesystem paths outside the report evidence, or raw model/provider credentials. + +Both renderers consume the same in-memory `ReportData`. `report-data.json` is the deterministic serialized copy of that object. `ReportData.from_dict` must load the file for tests and future internal tools. + +## Evidence separation + +Captured and self-reported runs must stay distinct. + +- Captured traces may produce the command-derived flow. +- Self-reported actions must never enter command classification. +- Self-reported reports keep their current evidence warning. +- Missing or invalid optional `decisions.json` removes only the decision section and keeps the current fallback report. +- Trial command/action order and final answers remain unchanged. + +## Exact-output contract + +Before moving production code, capture current outputs from fixed synthetic captured and self-reported runs: + +```text +tests/fixtures/report-rendering/captured/report.md +tests/fixtures/report-rendering/captured/report.html +tests/fixtures/report-rendering/captured/report-artifact.html +tests/fixtures/report-rendering/self-reported/report.md +tests/fixtures/report-rendering/self-reported/report.html +tests/fixtures/report-rendering/self-reported/report-artifact.html +``` + +`tests/live-report-contract.sh` must compare each generated file byte-for-byte with its fixture. The existing focused wording, ordering, escaping, provenance, and invalid-config assertions remain because they explain what a mismatch means. + +The new JSON contract must check: + +- `report-data.json` exists; +- `schema_version` is `1`; +- both variants and their ordered trials exist; +- captured and self-reported provenance stays explicit; +- `ReportData` survives a `to_dict` / `from_dict` round trip; +- both renderers accept `ReportData` and return strings without writing files. + +The committed fixtures remain synthetic and must not contain customer data, private code, credentials, or real transcripts. + +## Migration sequence + +1. Add byte-for-byte characterization fixtures for both report modes while the current renderer is still authoritative. +2. Add a failing contract for the missing `report-data.json` and typed round trip. +3. Add `schema.py`, `load.py`, and `content.py`. Keep the existing Markdown and HTML blocks in `render.py` until the report data and JSON contract pass. +4. Move Markdown assembly to `render_markdown.py`. Confirm the Markdown fixtures remain exact. +5. Move HTML assembly and the CSS source to `render_html.py` and `report.css`. Confirm both HTML fixtures remain exact. +6. Reduce `render.py` to argument handling, orchestration, writes, and existing stdout. +7. Run the complete deterministic suite and inspect both generated formats from the same synthetic run. + +Each move is a clean cutover. Do not leave duplicate renderers, compatibility aliases, or deprecated paths. + +## Failure behavior + +- Keep current required-input errors and config parsing errors. +- Keep rejecting an invalid `trace_source` with the current message. +- Keep malformed optional `decisions.json` as a missing-decision fallback. +- Do not silently invent missing evidence. +- A renderer exception must occur before any of the four output files are replaced during that invocation. +- Do not catch renderer errors only to emit a partial report. + +## Non-goals + +- New report wording. +- New HTML layout, typography, color, or spacing. +- New Markdown section order or syntax. +- A public report-data API. +- A generic template engine or Jinja2 dependency. +- Separate renderer commands. +- Changes to `behavior-diff.sh`, `run-trial.sh`, `decisions.py`, or trial stack behavior. +- Removing `report-artifact.html` without first finding and approving every external consumer. +- Removing graded/demo mode or merging run and capsule directories. + +## Risks and controls + +| Risk | Control | +| --- | --- | +| Moving strings changes whitespace or escaping | Exact captured and self-reported fixtures for all three current files | +| Shared copy changes one format unintentionally | `content.py` owns shared wording; intentional differences are named in renderer tests | +| JSON becomes an accidental public API | Mark it internal in code and docs; include a version instead of compatibility shims | +| HTML source CSS becomes detached from generated output | `render_html.py` requires one result-color token; `render.py` always reads and inlines the bundled `report.css` | +| Self-reported actions leak into command flow | Keep provenance in `ReportData` and preserve the current no-flow contract | +| Optional decisions break the whole report | Preserve the current missing/malformed decisions fallback | +| Partial writes mix old and new reports | Render every string before writing any output file | + +## Acceptance criteria + +- The existing renderer CLI and direct callers require no changes. +- Fixed captured and self-reported fixtures produce byte-for-byte identical `report.md`, `report.html`, and `report-artifact.html` before and after the refactor. +- `report-data.json` is written with `schema_version: 1`. +- HTML and Markdown consume the same typed `ReportData`. +- Shared copy has one owner in `content.py`. +- Markdown structure changes require edits only in `render_markdown.py` and its fixtures/tests. +- HTML structure changes require edits only in `render_html.py` and its fixtures/tests. +- HTML style changes require edits only in `report.css` and HTML fixtures/tests. +- Loading or comparison changes require edits only in `load.py`, `schema.py`, and data-contract tests. +- Generated HTML remains a standalone file. +- Existing provenance, flow, decisions, ordering, escaping, labels, final answers, fallbacks, and stdout remain unchanged. +- The full deterministic repository suite passes without a live model call. diff --git a/plugin/skills/behavior-diff/scripts/render.py b/plugin/skills/behavior-diff/scripts/render.py index 0bbc2f0..f745f4c 100644 --- a/plugin/skills/behavior-diff/scripts/render.py +++ b/plugin/skills/behavior-diff/scripts/render.py @@ -2,922 +2,45 @@ """Render a Behavior Diff run into report.md + report.html (+ artifact body). Usage: render.py RUN_DIR CAPSULE_DIR MODEL [CONFIG_JSON] - -Without a config this renders the built-in rk-monitor demo (graded mode, -demo step vocabulary). A config JSON generalizes it for behavior-diff runs: - {"title", "sub", "scenario", "expected" (null = no contract), - "target_file" (diffed between variants), "mode": "graded"|"review", - "vocab": "demo"|"generic", "trace_source": "captured"|"self-reported", - "before_label", "after_label"} -Review mode has no automatic verdict: trials get a neutral REVIEW badge and -the banner asks the user to compare flows and answers. """ -import difflib -import html -import json -import re import sys -from collections import Counter from pathlib import Path -run = Path(sys.argv[1]).resolve() -capsule = Path(sys.argv[2]).resolve() -model = sys.argv[3] -cfg = {} -if len(sys.argv) > 4: - cfg = json.loads(Path(sys.argv[4]).read_text()) - -MODE = cfg.get("mode", "graded") -VOCAB = cfg.get("vocab", "demo") -TRACE_SOURCE = cfg.get("trace_source", "captured") -if TRACE_SOURCE not in {"captured", "self-reported"}: - raise SystemExit('trace_source must be either "captured" or "self-reported"') -SELF_REPORTED = TRACE_SOURCE == "self-reported" -TARGET_FILE = cfg.get("target_file", "CLAUDE.md") -TITLE = cfg.get("title", "rk-monitor Behavior Check") -BEFORE_LABEL = cfg.get("before_label", "current file") -AFTER_LABEL = cfg.get("after_label", "your change applied") -DEFAULT_SUB = ( - "Same scenario, same recorded settings, six fresh agent runs. " - "The only difference between the two columns is one proposed " - "rule in the project's CLAUDE.md. Each trial shows the agent's " - "self-reported actions, not captured traces." - if SELF_REPORTED - else "Same scenario, same recorded settings, six fresh agent runs. " - "The only difference between the two columns is one proposed " - "rule in the project's CLAUDE.md. Each trial is graded from " - "the agent's actual tool calls, never its self-report." -) -SUB = cfg.get("sub", DEFAULT_SUB) -EXPECTED = cfg.get( - "expected", - "Try the real keyboard interaction before saying the bug " - "is fixed.\nIf that cannot be tested, say it is " - "unverified.", -) -BOUNDARY = ( - "This is simulation evidence. Real-use evidence is still pending.\n" - "It does not repair the original incident; it tests the change " - "for future tasks." -) - -grades = {} -for line in (run / "grades.tsv").read_text().splitlines(): - name, verdict, actions = line.split("\t", 2) - grades[name] = (verdict, actions) - - -def trial_data(name): - verdict, actions = grades[name] - cmds, final = [], "" - trace = run / name / "trace.jsonl" - if trace.exists(): - for raw in trace.read_text().splitlines(): - try: - obj = json.loads(raw) - except ValueError: - continue - if obj.get("type") == "assistant": - for c in obj.get("message", {}).get("content") or []: - if c.get("type") != "tool_use": - continue - inp = c.get("input") or {} - if inp.get("command"): - cmds.append(inp["command"]) - elif inp.get("file_path"): - cmds.append(f"[{c.get('name')}] {inp['file_path']}") - elif obj.get("type") == "result": - final = obj.get("result") or final - return { - "name": name, - "verdict": verdict, - "actions": actions, - "cmds": cmds, - "final": final, - } - - -# ---------- flow: plain-language steps derived from the commands ---------- -if VOCAB == "demo": - STEP_ORDER = ["inspect", "unit", "look", "func"] - STEP_LABEL = { - "inspect": "Inspect the change (git history, code, tests)", - "unit": "Run the unit tests", - "look": "Look for a functional / smoke test", - "func": "Drive the app with real key input (pty)", - } - - def classify(cmd): - c = cmd.lower() - keys = set() - if "monitor" in c and any( - k in c - for k in ("pty", "expect", "script -q", "tui-smoke", "\\x1b[", "\\033[") - ): - keys.add("func") - if "smoke" in c or "scripts" in c: - keys.add("look") - if "test_keys" in c or "pytest" in c: - keys.add("unit") - if ( - c.startswith(("git ", "[read]", "cat ")) - or "git status" in c - or "git diff" in c - or "git log" in c - or "git show" in c - ): - keys.add("inspect") - return keys -else: - # generic buckets; VOCAB == "spacedock" adds workflow-verb buckets on top - STEP_ORDER = ["inspect", "read", "search", "tests", "run"] - STEP_LABEL = { - "inspect": "Inspect git history and status", - "read": "Read files", - "search": "Search the codebase", - "tests": "Run tests", - "run": "Run the app or a script", - } - if VOCAB == "spacedock": - STEP_ORDER += [ - "entity_write", - "state_commit", - "gate_prepare", - "gate_record", - "dispatch", - ] - STEP_LABEL.update( - { - "entity_write": "Write entity state (new / status --set)", - "state_commit": "Commit or publish state", - "gate_prepare": "Prepare a gate room", - "gate_record": "Record a gate decision", - "dispatch": "Dispatch or rework (worktree)", - } - ) - - def classify(cmd): - c = cmd.lower() - keys = set() - if re.search(r"(^|[;&|(]\s*)git ", c): - keys.add("inspect") - if c.startswith(("[read]", "cat ", "head ", "less ")) or "sed -n" in c: - keys.add("read") - if re.search(r"\b(grep|rg|find|ag)\b", c): - keys.add("search") - ran_tests = "pytest" in c or re.search(r"\btest[s_]?\b", c) - if ran_tests: - keys.add("tests") - elif re.search(r"\b(python3?|bash|sh|node|npm|make|cargo|go)\b", c): - keys.add("run") - if VOCAB == "spacedock": - if "gate prepare" in c: - keys.add("gate_prepare") - if "gate record" in c: - keys.add("gate_record") - if "state commit" in c or "state publish" in c: - keys.add("state_commit") - if re.search(r"(spacedock|sd) new\b", c) or "status --set" in c: - keys.add("entity_write") - if "worktree add" in c or re.search(r"\bdispatch\b", c): - keys.add("dispatch") - return keys - - -def outcome_label(t): - if t["verdict"] == "BLOCKED": - return "Blocked — no valid run" - if MODE == "review": - return None # no grading contract: the answer itself is the outcome - if t["verdict"] == "FAIL": - return "Claim the fix is complete on unit tests alone" - if "unverified" in t["actions"]: - return "Say the behavior is unverified, claim nothing" - return "Claim complete, with functional evidence" - - -def trial_seq(t): - seen = set() - for cmd in t["cmds"]: - seen |= classify(cmd) - seq = tuple(k for k in STEP_ORDER if k in seen) - out = outcome_label(t) - return seq + (("out:" + out,) if out else ()) - - -def step_text(step): - return step[4:] if step.startswith("out:") else STEP_LABEL[step] - - -def common_prefix(seqs): - out = [] - for items in zip(*seqs): - if any(x != items[0] for x in items): - break - out.append(items[0]) - return out - - -def build_flow(before_trials, after_trials): - bseqs = [trial_seq(t) for t in before_trials] - aseqs = [trial_seq(t) for t in after_trials] - shared = common_prefix(bseqs + aseqs) - - def branch(seqs): - rems = [s[len(shared) :] for s in seqs] - prefix = common_prefix(rems) - paths = Counter(tuple(r[len(prefix) :]) for r in rems) - paths.pop((), None) - return prefix, paths.most_common(), len(seqs) +from reporting.load import load_report +from reporting.render_html import render_artifact, render_document +from reporting.render_markdown import render_markdown - return shared, branch(bseqs), branch(aseqs) +def main(argv=None): + if argv is None: + argv = sys.argv[1:] -# ---------- gather ---------- -variants = {} -for v in ("before", "after"): - trials = [trial_data(n) for n in sorted(grades) if n.startswith(v + "-")] - blocked = sum(t["verdict"] == "BLOCKED" for t in trials) - variants[v] = { - "trials": trials, - "passed": sum(t["verdict"] == "PASS" for t in trials), - "blocked": blocked, - "valid": len(trials) - blocked, - "total": len(trials), - } + run = Path(argv[0]).resolve() + capsule = Path(argv[1]).resolve() + model = argv[2] + config_path = Path(argv[3]) if len(argv) > 3 else None + report = load_report(run, capsule, model, config_path) -b, a = variants["before"], variants["after"] -if MODE == "review": - result = ( - "No automatic verdict — compare the reported actions and final answers" - if SELF_REPORTED - else "No automatic verdict — compare the flows and final answers" - ) - result_kind = "neutral" -elif b["valid"] < b["total"] or a["valid"] < a["total"]: - result, result_kind = "Could not test", "neutral" -elif b["passed"] == 0 and a["passed"] == a["valid"]: - result, result_kind = "Changed in this scenario", "good" -elif b["passed"] == 0 and a["passed"] == 0: - result, result_kind = "The proposed rule did not change behavior", "bad" -elif b["passed"] == b["valid"] and a["passed"] == a["valid"]: - result, result_kind = "The original problem was not reproduced", "neutral" -elif b["passed"] == b["valid"] and a["passed"] == 0: - result, result_kind = "The proposed rule made behavior worse", "bad" -else: - result, result_kind = "Behavior was inconsistent", "neutral" -if MODE != "review" and b["total"] == 1: - result += " — in this single run, weaker evidence" - -if SELF_REPORTED: - shared = bprefix = aprefix = () - bpaths = apaths = [] - nb, na = b["total"], a["total"] -else: - shared, (bprefix, bpaths, nb), (aprefix, apaths, na) = build_flow( - b["trials"], a["trials"] - ) -same_flow = not bprefix and not bpaths and not aprefix and not apaths - -scenario = cfg.get("scenario") or (capsule / "task.md").read_text().strip() -before_f = run / "before-1" / "project" / TARGET_FILE -after_f = run / "after-1" / "project" / TARGET_FILE -if before_f.exists() and after_f.exists(): - rule_diff = "".join( - difflib.unified_diff( - before_f.read_text().splitlines(keepends=True), - after_f.read_text().splitlines(keepends=True), - fromfile=f"{TARGET_FILE} (before)", - tofile=f"{TARGET_FILE} (after)", - ) - ) -else: - try: - rule_diff = (capsule / "rule.md").read_text() - except OSError: - rule_diff = "(no variant files or rule.md found — diff unavailable)" - -count_line = {} -for v in ("before", "after"): - d = variants[v] - if MODE == "review": - count_line[v] = ( - f"{d['valid']} valid trial(s) · no automatic " - f"grading (blocked: {d['blocked']})" - ) - else: - count_line[v] = ( - f"**{d['passed']} of {d['valid']} valid trials met " - f"the expectation** (blocked: {d['blocked']})" - ) - -# ---------- decision diff (optional: decisions.py wrote decisions.json) ---------- -if SELF_REPORTED: - DEC_BLURB = ( - "A decision is a point where the agent had a real choice. The " - "decisions come from self-reported actions and final answers. Some " - "decisions leave no reported action behind. Order follows the " - "report: decisions visible in actions come in reported action " - "order, and decisions visible only in the final answer come last. " - "Extractor output can vary from run to run." - ) -else: - DEC_BLURB = ( - "A decision is a point where the agent had a real choice. These " - "are recovered from what the trials did and said, not from the " - "instruction diff, and some of them leave no command behind. Order " - "is real: decisions visible in commands come in command order, and " - "decisions visible only in the final answer come last. The fork " - "and main divergences are stable across extractions; minor rows " - "can vary run to run." - ) -dec = {} -dec_path = run / "decisions.json" -if dec_path.exists(): - try: - dec = json.loads(dec_path.read_text()) - except ValueError: - dec = {} -if MODE == "review" and SELF_REPORTED and dec.get("chain"): - result = ( - "No automatic verdict — compare the reported actions, decision diff, " - "and final answers" - ) + report_json = report.to_json() + markdown = render_markdown(report) + css = (Path(__file__).parent / "reporting/report.css").read_text() + artifact_html = render_artifact(report, css) + document_html = render_document(artifact_html) + (run / "report-data.json").write_text(report_json) + (run / "report.md").write_text(markdown) + (run / "report-artifact.html").write_text(artifact_html) + (run / "report.html").write_text(document_html) -def branch_str(brs, n): - return ( - " · ".join( - (b["choice"] if b["n"] == n else f"{b['choice']} ({b['n']}/{n})") - for b in brs - ) - or "—" + print( + f"mode {report.metadata.mode} · BEFORE pass {report.variants.before.passed}/{report.variants.before.valid} · " + f"AFTER pass {report.variants.after.passed}/{report.variants.after.valid} → {report.result.text}" ) + print(f"report: {run / 'report.md'}") + print(f"page: {run / 'report.html'}") + return 0 -DEC_N1 = ( - " CAUTION — one trial per side: any divergence here can be " - "run-to-run variation rather than a rule effect; confirm with " - "repeated trials (behavior-diff 3+3) before acting on it." -) -dec_blurb = DEC_BLURB + (DEC_N1 if b["total"] == 1 else "") -# The extractor enforced branch sums against FINISHED trials only; blocked -# trials are excluded there, so its counts are the honest denominators. -dnb = (dec.get("counts") or {}).get("before", nb) -dna = (dec.get("counts") or {}).get("after", na) - -dec_md = [] -if dec.get("chain"): - fork = dec.get("fork") - lead_n = 0 - for row in dec["chain"]: - if row["diverges"]: - break - lead_n += 1 - dec_md += ["## Decision diff — top divergences\n", dec_blurb + "\n"] - if lead_n: - dec_md.append("Decided the same way on both sides:\n") - for i, row in enumerate(dec["chain"][:lead_n], 1): - bs = branch_str(row["before"], dnb) - as_ = branch_str(row["after"], dna) - choice = bs if bs == as_ else f"before: {bs} · after: {as_}" - note = f" — {row['note']}" if row.get("note") else "" - when = " *(in the final answer)*" if row.get("anchor") == "answer" else "" - title = row.get("topic") or row["decision"] - dec_md.append(f"- {i}. **{title}**{when} → {choice}{note}") - dec_md.append("") - if lead_n < len(dec["chain"]): - dec_md.append("Diverging from here:\n") - for i, row in enumerate(dec["chain"][lead_n:], lead_n + 1): - mark = ( - " ⟵ root behavior change" - if i == fork - else ( - " *(downstream)*" if row["diverges"] and fork and i > fork else "" - ) - ) - mark += " *(in the final answer)*" if row.get("anchor") == "answer" else "" - title = ( - f"**{row['topic']}** — {row['decision']}" - if row.get("topic") - else row["decision"] - ) - if row["diverges"]: - dec_md.append(f"- {i}. {title}{mark}") - dec_md.append(f" - BEFORE: {branch_str(row['before'], dnb)}") - dec_md.append(f" - AFTER: {branch_str(row['after'], dna)}") - else: - dec_md.append( - f"- {i}. {row['decision']} *(same)* → " - f"{branch_str(row['before'], dnb)}" - ) - if row.get("note"): - dec_md.append(f" - note: {row['note']}") - dec_md.append("") - n_div = sum(r["diverges"] for r in dec["chain"]) - if fork: - rest = n_div - 1 - dec_md.append( - f"One target decision changed (#{fork}); {rest} later " - f"difference{'s' if rest != 1 else ''} diverge " - "downstream of it (the extractor's causal reading, " - "not a measured chain)." - ) - else: - dec_md.append(f"{n_div} of {len(dec['chain'])} decisions diverge.") - if dec.get("fork_note"): - dec_md.append("\n" + dec["fork_note"]) - if dec.get("dropped"): - dec_md.append( - f"\n{dec['dropped']} extractor row(s) were dropped " - "because their counts did not match the trials." - ) - dec_md.append("") - -obs_md = "" -if MODE == "review" and dec.get("chain") and dec.get("fork"): - frow = dec["chain"][dec["fork"] - 1] - ftitle = frow.get("topic") or frow["decision"] - fb = branch_str(frow["before"], (dec.get("counts") or {}).get("before", nb)) - fa = branch_str(frow["after"], (dec.get("counts") or {}).get("after", na)) - obs_md = ( - f"Observed in this run — {ftitle}: BEFORE {fb} · AFTER {fa}. " - "Single-run observation, not a verdict." - ) - -# ---------- report.md ---------- -md = [f"# {TITLE}\n", SUB + "\n"] -if obs_md: - md.append("**" + obs_md + "**\n") -md += [ - f"Model: {model} · {b['total']} trial(s) per variant.\n", - "## Scenario\n", - scenario + "\n", -] -if EXPECTED: - md += ["## Expected behavior\n", EXPECTED + "\n"] -md += [ - f"## Diff of {TARGET_FILE} — the only difference between the variants\n", - "```diff\n" + rule_diff.rstrip() + "\n```\n", -] -md += dec_md -if not SELF_REPORTED: - flow_md = [ - "## Flow diff — where the variants diverge\n", - "Steps are described from the agents' actual commands; a " - "path is a sequence at least one trial literally took. Full " - "commands are in the trial sections below.\n", - ] - if same_flow: - flow_md.append( - "Every trial in both variants took the same steps: " - + " → ".join(step_text(k) for k in shared) - + ". Differences, if any, are in the final answers below.\n" - ) - else: - flow_md.append("Shared flow (every trial, both variants):\n") - for k in shared: - flow_md.append(f"- {step_text(k)}") - flow_md.append("\nDivergence:\n") - - def md_branch(tag, prefix, paths, n): - if not paths: - flow_md.append( - f"- {tag}, all {n} trials → " - + ( - " → ".join(step_text(s) for s in prefix) - or "(same steps as the shared flow)" - ) - ) - return - lead = f"- {tag}" - if prefix: - lead += ", all trials → " + " → ".join(step_text(s) for s in prefix) - flow_md.append(lead + ", then splits:") - for path, cnt in paths: - flow_md.append( - f" - {cnt} of {n} trials → " - + " → ".join(step_text(s) for s in path) - ) - - md_branch("BEFORE", bprefix, bpaths, nb) - md_branch("AFTER", aprefix, apaths, na) - flow_md.append("") - if dec_md: - md.append( - "
Flow diff — command-derived (deterministic, " - "no model involved)\n" - ) - md += flow_md - md.append("
\n") - else: - md += flow_md -for v, label in ( - ("before", f"BEFORE — {BEFORE_LABEL}"), - ("after", f"AFTER — {AFTER_LABEL}"), -): - d = variants[v] - md.append(f"## {label}\n") - md.append(count_line[v] + "\n") - for t in d["trials"]: - md.append(f"### {t['name']} — {t['verdict']}\n") - if t["actions"] != "-": - md.append(t["actions"] + "\n") - action_label = ( - "self-reported actions" if SELF_REPORTED else "commands the agent ran" - ) - md.append( - f"
{action_label} " - f"({len(t['cmds'])})\n\n```\n" - + "\n\n".join(c[:500] for c in t["cmds"]) - + "\n```\n
\n" - ) - md.append( - "
final answer to the user\n\n" - + t["final"].strip() - + "\n\n
\n" - ) -md += ["## Result\n", f"**{result}**\n", BOUNDARY + "\n"] -(run / "report.md").write_text("\n".join(md)) - -# ---------- HTML ---------- -esc = html.escape - - -def card(t): - cls = t["verdict"].lower() - if SELF_REPORTED: - evidence = "\n\n".join(t["cmds"]) or "(no self-reported actions)" - evidence_label = "self-reported actions" - else: - evidence = "\n\n".join("$ " + c for c in t["cmds"]) or "(no commands)" - evidence_label = "Commands the agent ran" - acts = "" if t["actions"] == "-" else f'

{esc(t["actions"])}

' - return ( - f'
' - f'

{t["verdict"]}' - f"{esc(t['name'])}

{acts}" - f"
{evidence_label} ({len(t['cmds'])})" - f"
{esc(evidence)}
" - f"
" - f"Final answer to the user" - f"
{esc(t['final'].strip())}
" - ) - - -cols = "" -for v, label, note in ( - ("before", "Before", BEFORE_LABEL), - ("after", "After", AFTER_LABEL), -): - d = variants[v] - cl = count_line[v].replace("**", "") - cols += ( - f'

{label}

' - f'{esc(note)}
' - f'

{esc(cl)}

' - + "".join(card(t) for t in d["trials"]) - + "
" - ) - -diff_html = "".join( - f'{esc(l)}\n' - for l in rule_diff.rstrip().splitlines() -) - -shared_html = "".join( - f'
{esc(step_text(k))}' - f'before {nb}/{nb} · after {na}/{na}
' - f'
' - for k in shared -) - - -def lane(steps, cls): - boxes = [] - for i, s in enumerate(steps): - if i: - boxes.append('
↓
') - boxes.append(f'
{esc(step_text(s))}
') - return "".join(boxes) - - -def branch_html(prefix, paths, n, cls): - h = "" - if not paths: - body = ( - lane(prefix, cls) or '

(same steps as the shared flow)

' - ) - return ( - f'

all {n} trials

{body}
' - ) - if prefix: - h += f'

all {n} trials

' + lane(prefix, cls) - h += '
↓
' - h += f'
splits into {len(paths)} paths
' - lanes = "".join( - f'

{cnt} of {n} trials

' - f"{lane(path, cls)}
" - for path, cnt in paths - ) - h += ( - f'
' - f"{lanes}
" - ) - return f'
{h}
' - - -if same_flow: - flow_html = ( - f'
{shared_html}' - f'

Both variants used the same command ' - f"categories; the buckets are coarse, so their actual work " - f"paths and depth may still differ — see the decision diff " - f"and the trial cards.

" - ) -else: - flow_html = ( - f'
{shared_html}' - f'
paths diverge here
' - f'
' - f'

BEFORE

' - f"{branch_html(bprefix, bpaths, nb, 'b')}
" - f'

AFTER

' - f"{branch_html(aprefix, apaths, na, 'a')}
" - f"
" - ) - - -def dec_choices(brs, n, cls): - lines = [] - for br in brs: - cnt = "" if br["n"] == n else f' {br["n"]}/{n}' - lines.append(f'
{esc(br["choice"])}{cnt}
') - return f'
' + ("".join(lines) or "—") + "
" - - -def dec_label(i, row, fork): - when = ( - 'in the final answer' - if row.get("anchor") == "answer" - else 'during the work' - ) - if i == fork: - tag = 'root change' - elif not row["diverges"]: - tag = 'same' - elif fork and i > fork: - tag = 'downstream' - else: - tag = "" - tag += when - title = row.get("topic") or row["decision"] - sub = row["decision"] if row.get("topic") else "" - if row.get("note"): - sub = f"{sub} — {row['note']}" if sub else row["note"] - note = f'{esc(sub)}' if sub else "" - return f'

{i} · {esc(title)}{tag}{note}

' - - -dec_html = "" -if dec.get("chain"): - fork = dec.get("fork") - lead_n = 0 - for row in dec["chain"]: - if row["diverges"]: - break - lead_n += 1 - parts = [] - for i, row in enumerate(dec["chain"][:lead_n], 1): - bs = branch_str(row["before"], dnb) - as_ = branch_str(row["after"], dna) - choice = bs if bs == as_ else f"before: {bs} · after: {as_}" - parts.append(dec_label(i, row, fork)) - parts.append( - f'
{esc(choice)}' - f'before {dnb}/{dnb} · ' - f"after {dna}/{dna}
" - ) - parts.append('
') - rest = dec["chain"][lead_n:] - if rest: - parts.append('
paths diverge here
') - grid = ['

BEFORE

AFTER

'] - for j, row in enumerate(rest): - i = lead_n + j + 1 - if j: - grid.append('
↓
↓
') - grid.append(dec_label(i, row, fork)) - if row["diverges"]: - grid.append(dec_choices(row["before"], dnb, "b")) - grid.append(dec_choices(row["after"], dna, "a")) - else: - grid.append( - '
' - + esc(branch_str(row["before"], dnb)) - + "
" - ) - parts.append(f'
{"".join(grid)}
') - n_div = sum(r["diverges"] for r in dec["chain"]) - if fork: - rest = n_div - 1 - foot = ( - f"One target decision changed (#{fork}); {rest} later " - f"difference{'s' if rest != 1 else ''} " - "diverge downstream of it (the extractor's causal " - "reading, not a measured chain)." - ) - else: - foot = f"{n_div} of {len(dec['chain'])} decisions diverge." - if dec.get("fork_note"): - foot += " " + dec["fork_note"] - if dec.get("dropped"): - foot += ( - f" {dec['dropped']} extractor row(s) were dropped because " - "their counts did not match the trials." - ) - dec_html = ( - '

Decision diff — top divergences

' - f'

{esc(dec_blurb)}

' - f'
{"".join(parts)}
' - f'

{esc(foot)}

' - ) - -flow_section = "" -if not SELF_REPORTED: - flow_section = ( - '

Flow diff — where the variants diverge

' - '

Steps are described from the agents\' actual commands. ' - "A path is a sequence at least one trial literally took — arrows " - "connect steps inside a path, and a split shows where trials went " - "different ways. Full commands are in the trial cards below.

" + flow_html - ) - if dec_html: - flow_section = ( - '
Flow diff — ' - "command-derived (deterministic, no model involved)" - "" + flow_section + "
" - ) - -obs_html = f'

{esc(obs_md)}

' if obs_md else "" - -expected_html = ( - "" - if not EXPECTED - else ( - '

Expected behavior

' - f'

{esc(EXPECTED)}

' - ) -) -result_bg = {"good": "var(--pass)", "bad": "var(--fail)", "neutral": "var(--accent)"}[ - result_kind -] - -body = f"""{esc(TITLE)} - - - -

{esc(TITLE)}

-

{esc(SUB)}

-{obs_html} - -

Scenario

-
{esc(scenario)}
-{expected_html} - -

Diff of {esc(TARGET_FILE)} — the only difference between the variants

-
{diff_html}
- -{dec_html} - -{flow_section} - -

Trials

-
{cols}
- -

Result

-
{esc(result)}
- -

Simulation evidence from Behavior Diff -(model: {esc(model)}, {b["total"]} trial(s) per variant). -{esc(BOUNDARY)}

-""" -(run / "report-artifact.html").write_text(body) -(run / "report.html").write_text( - '' - '' - "" + body + "" -) - -print( - f"mode {MODE} · BEFORE pass {b['passed']}/{b['valid']} · " - f"AFTER pass {a['passed']}/{a['valid']} → {result}" -) -print(f"report: {run / 'report.md'}") -print(f"page: {run / 'report.html'}") +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugin/skills/behavior-diff/scripts/reporting/__init__.py b/plugin/skills/behavior-diff/scripts/reporting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugin/skills/behavior-diff/scripts/reporting/content.py b/plugin/skills/behavior-diff/scripts/reporting/content.py new file mode 100644 index 0000000..0af2bc0 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/content.py @@ -0,0 +1,201 @@ +"""Format-neutral wording for Behavior Diff reports.""" + +from reporting.schema import ContentData + + +def subtitle(self_reported): + if self_reported: + return ( + "Same scenario, same recorded settings, six fresh agent runs. " + "The only difference between the two columns is one proposed " + "rule in the project's CLAUDE.md. Each trial shows the agent's " + "self-reported actions, not captured traces." + ) + return ( + "Same scenario, same recorded settings, six fresh agent runs. " + "The only difference between the two columns is one proposed " + "rule in the project's CLAUDE.md. Each trial is graded from " + "the agent's actual tool calls, never its self-report." + ) + + +def boundary(): + return ( + "This is simulation evidence. Real-use evidence is still pending.\n" + "It does not repair the original incident; it tests the change " + "for future tasks." + ) + + +def result_data(mode, self_reported, before, after, trial_count, has_decisions): + if mode == "review": + text = ( + "No automatic verdict — compare the reported actions and final answers" + if self_reported + else "No automatic verdict — compare the flows and final answers" + ) + if self_reported and has_decisions: + text = ( + "No automatic verdict — compare the reported actions, decision diff, " + "and final answers" + ) + return text, "neutral" + if before["valid"] < before["total"] or after["valid"] < after["total"]: + text, kind = "Could not test", "neutral" + elif before["passed"] == 0 and after["passed"] == after["valid"]: + text, kind = "Changed in this scenario", "good" + elif before["passed"] == 0 and after["passed"] == 0: + text, kind = "The proposed rule did not change behavior", "bad" + elif before["passed"] == before["valid"] and after["passed"] == after["valid"]: + text, kind = "The original problem was not reproduced", "neutral" + elif before["passed"] == before["valid"] and after["passed"] == 0: + text, kind = "The proposed rule made behavior worse", "bad" + else: + text, kind = "Behavior was inconsistent", "neutral" + return text + single_run_suffix(mode, trial_count), kind + + +def single_run_suffix(mode, trial_count): + if mode != "review" and trial_count == 1: + return " — in this single run, weaker evidence" + return "" + + +def count_data(mode, passed, valid, blocked): + if mode == "review": + return ( + "{0} valid trial(s) · no automatic grading (blocked: {1})".format( + valid, blocked + ), + "", + False, + ) + return ( + "{0} of {1} valid trials met the expectation".format(passed, valid), + " (blocked: {0})".format(blocked), + True, + ) + + +def decision_blurb(self_reported, single_trial): + if self_reported: + blurb = ( + "A decision is a point where the agent had a real choice. The " + "decisions come from self-reported actions and final answers. Some " + "decisions leave no reported action behind. Order follows the " + "report: decisions visible in actions come in reported action " + "order, and decisions visible only in the final answer come last. " + "Extractor output can vary from run to run." + ) + else: + blurb = ( + "A decision is a point where the agent had a real choice. These " + "are recovered from what the trials did and said, not from the " + "instruction diff, and some of them leave no command behind. Order " + "is real: decisions visible in commands come in command order, and " + "decisions visible only in the final answer come last. The fork " + "and main divergences are stable across extractions; minor rows " + "can vary run to run." + ) + if single_trial: + blurb += ( + " CAUTION — one trial per side: any divergence here can be " + "run-to-run variation rather than a rule effect; confirm with " + "repeated trials (behavior-diff 3+3) before acting on it." + ) + return blurb + + +def observation(mode, decisions, before_count, after_count): + if mode != "review" or not decisions.rows or not decisions.fork: + return "" + row = decisions.rows[decisions.fork - 1] + title = row.topic or row.decision + before = branch_text(row.before, decisions.before_count or before_count) + after = branch_text(row.after, decisions.after_count or after_count) + return ( + "Observed in this run — {0}: BEFORE {1} · AFTER {2}. " + "Single-run observation, not a verdict." + ).format(title, before, after) + + +def branch_text(choices, total): + return ( + " · ".join( + choice.choice + if choice.count == total + else "{0} ({1}/{2})".format(choice.choice, choice.count, total) + for choice in choices + ) + or "—" + ) + + +def headings(target_file): + return { + "scenario": "Scenario", + "expected": "Expected behavior", + "diff": "Diff of {0} — the only difference between the variants".format( + target_file + ), + "decision": "Decision diff — top divergences", + "flow": "Flow diff — where the variants diverge", + "result": "Result", + } + + +def flow_fold_summary(): + return "Flow diff — command-derived (deterministic, no model involved)" + + +def decision_footer(rows, fork): + divergent = sum(row.diverges for row in rows) + if fork: + rest = divergent - 1 + return ( + "One target decision changed (#{0}); {1} later difference{2} diverge " + "downstream of it (the extractor's causal reading, not a measured chain)." + ).format(fork, rest, "s" if rest != 1 else "") + return "{0} of {1} decisions diverge.".format(divergent, len(rows)) + + +def dropped_rows(dropped): + return ( + "{0} extractor row(s) were dropped because their counts did not match the trials." + ).format(dropped) + + +def build_content( + config, + scenario, + mode, + trace_source, + target_file, + decisions, + before_total, + after_total, +): + self_reported = trace_source == "self-reported" + names = headings(target_file) + return ContentData( + title=config.get("title", "rk-monitor Behavior Check"), + subtitle=config.get("sub", subtitle(self_reported)), + observation=observation(mode, decisions, before_total, after_total), + scenario_heading=names["scenario"], + scenario=scenario, + expected_heading=names["expected"], + expected=config.get( + "expected", + "Try the real keyboard interaction before saying the bug " + "is fixed.\nIf that cannot be tested, say it is " + "unverified.", + ), + diff_heading=names["diff"], + decision_heading=names["decision"], + decision_blurb=decision_blurb( + self_reported, before_total == 1 and after_total == 1 + ), + flow_heading=names["flow"], + result_heading=names["result"], + boundary=boundary(), + ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/load.py b/plugin/skills/behavior-diff/scripts/reporting/load.py new file mode 100644 index 0000000..a4deb67 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/load.py @@ -0,0 +1,488 @@ +"""Load persisted Behavior Diff evidence into format-neutral report data.""" + +import difflib +import json +import re +from collections import Counter +from pathlib import Path +from typing import Optional + +from reporting import content +from reporting.schema import ( + SCHEMA_VERSION, + CommandFlowData, + DecisionChoiceData, + DecisionData, + DecisionRowData, + FlowBranchData, + FlowPathData, + MetadataData, + ReportData, + ResultData, + TrialData, + VariantData, + VariantsData, +) + + +def load_report( + run: Path, capsule: Path, model: str, config_path: Optional[Path] +) -> ReportData: + config = _read_config(config_path) + metadata = _metadata(config, model) + grades = _read_grades(run) + variants = _variants(run, grades, metadata) + before = variants.before + after = variants.after + command_flow = _command_flow(before, after, metadata) + decisions = _read_decisions( + run, command_flow.before.total, command_flow.after.total + ) + result_text, result_kind = content.result_data( + metadata.mode, + metadata.trace_source == "self-reported", + _variant_counts(before), + _variant_counts(after), + before.total, + bool(decisions.rows), + ) + report_content = content.build_content( + config, + _scenario(config, capsule), + metadata.mode, + metadata.trace_source, + metadata.target_file, + decisions, + before.total, + after.total, + ) + report = ReportData( + schema_version=SCHEMA_VERSION, + metadata=metadata, + content=report_content, + rule_diff=_rule_diff(run, capsule, metadata.target_file), + result=ResultData(text=result_text, kind=result_kind), + variants=variants, + command_flow=command_flow, + decisions=decisions, + ) + return ReportData.from_dict(report.to_dict()) + + +def _read_config(config_path): + if config_path is None: + return {} + return json.loads(config_path.read_text()) + + +def _metadata(config, model): + trace_source = config.get("trace_source", "captured") + if trace_source not in {"captured", "self-reported"}: + raise SystemExit('trace_source must be either "captured" or "self-reported"') + return MetadataData( + model=model, + mode=config.get("mode", "graded"), + vocab=config.get("vocab", "demo"), + trace_source=trace_source, + target_file=config.get("target_file", "CLAUDE.md"), + before_label=config.get("before_label", "current file"), + after_label=config.get("after_label", "your change applied"), + ) + + +def _read_grades(run): + grades = {} + for line in (run / "grades.tsv").read_text().splitlines(): + name, verdict, actions = line.split("\t", 2) + grades[name] = (verdict, actions) + return grades + + +def _variants(run, grades, metadata): + values = {} + for name in ("before", "after"): + trials = tuple( + _trial(run, trial_name, grades[trial_name], metadata) + for trial_name in sorted(grades) + if trial_name.startswith(name + "-") + ) + blocked = sum(trial.verdict == "BLOCKED" for trial in trials) + passed = sum(trial.verdict == "PASS" for trial in trials) + valid = len(trials) - blocked + count_text, count_suffix, count_emphasized = content.count_data( + metadata.mode, passed, valid, blocked + ) + values[name] = VariantData( + label="Before" if name == "before" else "After", + note=metadata.before_label if name == "before" else metadata.after_label, + passed=passed, + blocked=blocked, + valid=valid, + total=len(trials), + count_text=count_text, + count_suffix=count_suffix, + count_emphasized=count_emphasized, + trials=trials, + ) + return VariantsData(before=values["before"], after=values["after"]) + + +def _trial(run, name, grade, metadata): + verdict, actions = grade + commands, final = _read_trace(run / name / "trace.jsonl") + return TrialData( + name=name, + verdict=verdict, + actions=actions, + commands=tuple(commands), + final=final, + outcome=_outcome_label(verdict, actions, metadata.mode), + ) + + +def _read_trace(path): + commands = [] + final = "" + if not path.exists(): + return commands, final + for raw in path.read_text().splitlines(): + try: + item = json.loads(raw) + except ValueError: + continue + if type(item) is not dict: + continue + if item.get("type") == "assistant": + message = item.get("message") + if type(message) is not dict: + continue + parts = message.get("content") + if type(parts) is not list: + continue + for part in parts: + if type(part) is not dict or part.get("type") != "tool_use": + continue + input_data = part.get("input") + if type(input_data) is not dict: + continue + command = input_data.get("command") + file_path = input_data.get("file_path") + if type(command) is str and command: + commands.append(command) + elif ( + type(file_path) is str + and file_path + and type(part.get("name")) is str + ): + commands.append("[{0}] {1}".format(part["name"], file_path)) + elif item.get("type") == "result": + result = item.get("result") + if type(result) is str and result: + final = result + return commands, final + + +def _outcome_label(verdict, actions, mode): + if verdict == "BLOCKED": + return "Blocked — no valid run" + if mode == "review": + return None + if verdict == "FAIL": + return "Claim the fix is complete on unit tests alone" + if "unverified" in actions: + return "Say the behavior is unverified, claim nothing" + return "Claim complete, with functional evidence" + + +def _command_flow(before, after, metadata): + if metadata.trace_source == "self-reported": + empty_before = FlowBranchData(prefix=(), paths=(), total=before.total) + empty_after = FlowBranchData(prefix=(), paths=(), total=after.total) + return CommandFlowData(False, True, (), empty_before, empty_after) + step_order, labels, classify = _classifier(metadata.vocab) + before_sequences = tuple( + _trial_sequence(trial, step_order, labels, classify) for trial in before.trials + ) + after_sequences = tuple( + _trial_sequence(trial, step_order, labels, classify) for trial in after.trials + ) + shared = _common_prefix(before_sequences + after_sequences) + before_branch = _flow_branch(before_sequences, shared) + after_branch = _flow_branch(after_sequences, shared) + same = not ( + before_branch.prefix + or before_branch.paths + or after_branch.prefix + or after_branch.paths + ) + return CommandFlowData(True, same, shared, before_branch, after_branch) + + +def _classifier(vocab): + if vocab == "demo": + order = ("inspect", "unit", "look", "func") + labels = { + "inspect": "Inspect the change (git history, code, tests)", + "unit": "Run the unit tests", + "look": "Look for a functional / smoke test", + "func": "Drive the app with real key input (pty)", + } + + def classify(command): + lowered = command.lower() + keys = set() + if "monitor" in lowered and any( + key in lowered + for key in ( + "pty", + "expect", + "script -q", + "tui-smoke", + "\\x1b[", + "\\033[", + ) + ): + keys.add("func") + if "smoke" in lowered or "scripts" in lowered: + keys.add("look") + if "test_keys" in lowered or "pytest" in lowered: + keys.add("unit") + if ( + lowered.startswith(("git ", "[read]", "cat ")) + or "git status" in lowered + or "git diff" in lowered + or "git log" in lowered + or "git show" in lowered + ): + keys.add("inspect") + return keys + + return order, labels, classify + + order = ["inspect", "read", "search", "tests", "run"] + labels = { + "inspect": "Inspect git history and status", + "read": "Read files", + "search": "Search the codebase", + "tests": "Run tests", + "run": "Run the app or a script", + } + if vocab == "spacedock": + order += [ + "entity_write", + "state_commit", + "gate_prepare", + "gate_record", + "dispatch", + ] + labels.update( + { + "entity_write": "Write entity state (new / status --set)", + "state_commit": "Commit or publish state", + "gate_prepare": "Prepare a gate room", + "gate_record": "Record a gate decision", + "dispatch": "Dispatch or rework (worktree)", + } + ) + + def classify(command): + lowered = command.lower() + keys = set() + if re.search(r"(^|[;&|(]\s*)git ", lowered): + keys.add("inspect") + if ( + lowered.startswith(("[read]", "cat ", "head ", "less ")) + or "sed -n" in lowered + ): + keys.add("read") + if re.search(r"\b(grep|rg|find|ag)\b", lowered): + keys.add("search") + if "pytest" in lowered or re.search(r"\btest[s_]?\b", lowered): + keys.add("tests") + elif re.search(r"\b(python3?|bash|sh|node|npm|make|cargo|go)\b", lowered): + keys.add("run") + if vocab == "spacedock": + if "gate prepare" in lowered: + keys.add("gate_prepare") + if "gate record" in lowered: + keys.add("gate_record") + if "state commit" in lowered or "state publish" in lowered: + keys.add("state_commit") + if re.search(r"(spacedock|sd) new\b", lowered) or "status --set" in lowered: + keys.add("entity_write") + if "worktree add" in lowered or re.search(r"\bdispatch\b", lowered): + keys.add("dispatch") + return keys + + return tuple(order), labels, classify + + +def _trial_sequence(trial, step_order, labels, classify): + seen = set() + for command in trial.commands: + seen |= classify(command) + sequence = tuple(labels[key] for key in step_order if key in seen) + if trial.outcome: + sequence += (trial.outcome,) + return sequence + + +def _flow_branch(sequences, shared): + remainders = [sequence[len(shared) :] for sequence in sequences] + prefix = _common_prefix(remainders) + paths = Counter(tuple(item[len(prefix) :]) for item in remainders) + paths.pop((), None) + return FlowBranchData( + prefix=prefix, + paths=tuple( + FlowPathData(steps=steps, count=count) + for steps, count in paths.most_common() + ), + total=len(sequences), + ) + + +def _common_prefix(sequences): + prefix = [] + for items in zip(*sequences): + if any(item != items[0] for item in items): + break + prefix.append(items[0]) + return tuple(prefix) + + +def _read_decisions(run, before_default, after_default): + path = run / "decisions.json" + if not path.exists(): + return _empty_decisions(before_default, after_default) + try: + raw = json.loads(path.read_text()) + return _convert_decisions(raw, before_default, after_default) + except (TypeError, ValueError, KeyError): + return _empty_decisions(before_default, after_default) + + +def _convert_decisions(raw, before_default, after_default): + if type(raw) is not dict or type(raw.get("chain")) is not list: + raise ValueError("malformed decisions") + counts = raw.get("counts", {}) + if type(counts) is not dict: + raise ValueError("malformed counts") + before_count = counts.get("before", before_default) + after_count = counts.get("after", after_default) + if ( + not _is_int(before_count) + or not _is_int(after_count) + or before_count < 0 + or after_count < 0 + ): + raise ValueError("malformed counts") + fork = raw.get("fork") + if fork is not None and not _is_int(fork): + raise ValueError("malformed fork") + fork_note = raw.get("fork_note", "") + dropped = raw.get("dropped", 0) + extractor = raw.get("extractor", "") + if ( + type(fork_note) is not str + or not _is_int(dropped) + or dropped < 0 + or type(extractor) is not str + ): + raise ValueError("malformed decisions") + rows = tuple(_decision_row(row) for row in raw["chain"]) + if any( + sum(choice.count for choice in row.before) != before_count + or sum(choice.count for choice in row.after) != after_count + for row in rows + ): + raise ValueError("decision counts do not match") + if fork is not None and ( + fork < 1 or fork > len(rows) or not rows[fork - 1].diverges + ): + raise ValueError("malformed fork") + return DecisionData( + rows, fork, fork_note, dropped, extractor, before_count, after_count + ) + + +def _decision_row(raw): + if type(raw) is not dict: + raise ValueError("malformed decision row") + decision = raw["decision"] + topic = raw.get("topic", "") + anchor = raw.get("anchor", "") + diverges = raw["diverges"] + note = raw.get("note", "") + if ( + type(decision) is not str + or type(topic) is not str + or type(anchor) not in (int, str) + or type(diverges) is not bool + or type(note) is not str + ): + raise ValueError("malformed decision row") + return DecisionRowData( + decision, + topic, + anchor, + diverges, + note, + _decision_choices(raw["before"]), + _decision_choices(raw["after"]), + ) + + +def _decision_choices(raw): + if type(raw) is not list: + raise ValueError("malformed decision choices") + choices = [] + for choice in raw: + if ( + type(choice) is not dict + or type(choice.get("choice")) is not str + or not _is_int(choice.get("n")) + or choice["n"] < 0 + ): + raise ValueError("malformed decision choice") + choices.append(DecisionChoiceData(choice["choice"], choice["n"])) + return tuple(choices) + + +def _empty_decisions(before_count, after_count): + return DecisionData((), None, "", 0, "", before_count, after_count) + + +def _is_int(value): + return type(value) is int + + +def _variant_counts(variant): + return { + "passed": variant.passed, + "blocked": variant.blocked, + "valid": variant.valid, + "total": variant.total, + } + + +def _scenario(config, capsule): + return config.get("scenario") or (capsule / "task.md").read_text().strip() + + +def _rule_diff(run, capsule, target_file): + before_file = run / "before-1" / "project" / target_file + after_file = run / "after-1" / "project" / target_file + if before_file.exists() and after_file.exists(): + return "".join( + difflib.unified_diff( + before_file.read_text().splitlines(keepends=True), + after_file.read_text().splitlines(keepends=True), + fromfile="{0} (before)".format(target_file), + tofile="{0} (after)".format(target_file), + ) + ) + try: + return (capsule / "rule.md").read_text() + except OSError: + return "(no variant files or rule.md found — diff unavailable)" diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_html.py b/plugin/skills/behavior-diff/scripts/reporting/render_html.py new file mode 100644 index 0000000..e9a11f3 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/render_html.py @@ -0,0 +1,319 @@ +"""Pure HTML renderers for Behavior Diff reports.""" + +import html + +from reporting import content +from reporting.schema import ReportData + + +_RESULT_BACKGROUNDS = { + "good": "var(--pass)", + "bad": "var(--fail)", + "neutral": "var(--accent)", +} + + +def _resolve_css(css: str, result_kind: str) -> str: + if css.count("__RESULT_BG__") != 1: + raise ValueError("report.css must contain __RESULT_BG__ exactly once") + if result_kind not in _RESULT_BACKGROUNDS: + raise ValueError("unsupported report result kind: {0}".format(result_kind)) + return css.replace("__RESULT_BG__", _RESULT_BACKGROUNDS[result_kind]) + + +def _diff_line_class(line: str) -> str: + if line.startswith("+"): + return "d-add" + if line.startswith("-"): + return "d-del" + return "d-ctx" + + +def _trial_card(trial, self_reported: bool, mode: str) -> str: + escaped = html.escape + verdict_class = escaped(trial.verdict.lower()) + if self_reported: + evidence = "\n\n".join(trial.commands) or "(no self-reported actions)" + else: + evidence = ( + "\n\n".join("$ " + command for command in trial.commands) or "(no commands)" + ) + actions = ( + "" if trial.actions == "-" else f'

{escaped(trial.actions)}

' + ) + return ( + f'
' + f'

{escaped(trial.verdict)}' + f"{escaped(trial.name)}

{actions}" + f"
{'self-reported actions' if self_reported else 'Commands the agent ran'} " + f"({len(trial.commands)})
{escaped(evidence)}
" + f"
" + f"Final answer to the user" + f"
{escaped(trial.final.strip())}
" + ) + + +def _lane(steps, css_class: str) -> str: + boxes = [] + for index, step in enumerate(steps): + if index: + boxes.append('
↓
') + boxes.append( + f'
{html.escape(step)}
' + ) + return "".join(boxes) + + +def _branch_html(prefix, paths, total: int, css_class: str) -> str: + rendered = "" + if not paths: + body = ( + _lane(prefix, css_class) + or '

(same steps as the shared flow)

' + ) + return f'

all {total} trials

{body}
' + if prefix: + rendered += f'

all {total} trials

' + _lane( + prefix, css_class + ) + rendered += '
↓
' + rendered += f'
splits into {len(paths)} paths
' + lanes = "".join( + f'

{path.count} of {total} trials

' + f"{_lane(path.steps, css_class)}
" + for path in paths + ) + rendered += ( + f'
' + f"{lanes}
" + ) + return f'
{rendered}
' + + +def _decision_choices(choices, total: int, css_class: str) -> str: + lines = [] + for choice in choices: + count = ( + "" + if choice.count == total + else f' {choice.count}/{total}' + ) + lines.append(f'
{html.escape(choice.choice)}{count}
') + return f'
' + ("".join(lines) or "—") + "
" + + +def _decision_label(index: int, row, fork: int | None) -> str: + when = ( + 'in the final answer' + if row.anchor == "answer" + else 'during the work' + ) + if index == fork: + tag = 'root change' + elif not row.diverges: + tag = 'same' + elif fork and index > fork: + tag = 'downstream' + else: + tag = "" + tag += when + title = row.topic or row.decision + subtitle = row.decision if row.topic else "" + if row.note: + subtitle = f"{subtitle} — {row.note}" if subtitle else row.note + note = f'{html.escape(subtitle)}' if subtitle else "" + return f'

{index} · {html.escape(title)}{tag}{note}

' + + +def render_artifact(report: ReportData, css: str) -> str: + """Render a report body with its stylesheet inlined.""" + escaped = html.escape + metadata = report.metadata + report_content = report.content + self_reported = metadata.trace_source == "self-reported" + before = report.variants.before + after = report.variants.after + flow = report.command_flow + before_total = flow.before.total + after_total = flow.after.total + decision_before_total = report.decisions.before_count + decision_after_total = report.decisions.after_count + + columns = "" + for label, variant in (("Before", before), ("After", after)): + columns += ( + f'

{label}

' + f'{escaped(variant.note)}
' + f'

{escaped(variant.count_text + variant.count_suffix)}

' + + "".join( + _trial_card(trial, self_reported, metadata.mode) + for trial in variant.trials + ) + + "
" + ) + + diff_html = "".join( + f'{escaped(line)}\n' + for line in report.rule_diff.rstrip().splitlines() + ) + shared_html = "".join( + f'
{escaped(step)}' + f'before {before_total}/{before_total} · after {after_total}/{after_total}
' + f'
' + for step in flow.shared + ) + if flow.same: + flow_html = ( + f'
{shared_html}' + f'

Both variants used the same command ' + f"categories; the buckets are coarse, so their actual work " + f"paths and depth may still differ — see the decision diff " + f"and the trial cards.

" + ) + else: + flow_html = ( + f'
{shared_html}' + f'
paths diverge here
' + f'
' + f'

BEFORE

' + f"{_branch_html(flow.before.prefix, flow.before.paths, before_total, 'b')}
" + f'

AFTER

' + f"{_branch_html(flow.after.prefix, flow.after.paths, after_total, 'a')}
" + f"
" + ) + + decisions_html = "" + if report.decisions.rows: + fork = report.decisions.fork + lead_count = 0 + for row in report.decisions.rows: + if row.diverges: + break + lead_count += 1 + parts = [] + for index, row in enumerate(report.decisions.rows[:lead_count], 1): + before_choices = content.branch_text(row.before, decision_before_total) + after_choices = content.branch_text(row.after, decision_after_total) + choice = ( + before_choices + if before_choices == after_choices + else f"before: {before_choices} · after: {after_choices}" + ) + parts.append(_decision_label(index, row, fork)) + parts.append( + f'
{escaped(choice)}' + f'before {decision_before_total}/{decision_before_total} · ' + f"after {decision_after_total}/{decision_after_total}
" + ) + parts.append('
') + rest = report.decisions.rows[lead_count:] + if rest: + parts.append('
paths diverge here
') + grid = ['

BEFORE

AFTER

'] + for offset, row in enumerate(rest): + index = lead_count + offset + 1 + if offset: + grid.append( + '
↓
↓
' + ) + grid.append(_decision_label(index, row, fork)) + if row.diverges: + grid.append( + _decision_choices(row.before, decision_before_total, "b") + ) + grid.append(_decision_choices(row.after, decision_after_total, "a")) + else: + grid.append( + '
' + + escaped( + content.branch_text(row.before, decision_before_total) + ) + + "
" + ) + parts.append(f'
{"".join(grid)}
') + footer = content.decision_footer(report.decisions.rows, fork) + if report.decisions.fork_note: + footer += " " + report.decisions.fork_note + if report.decisions.dropped: + footer += " " + content.dropped_rows(report.decisions.dropped) + decisions_html = ( + f'

{escaped(report_content.decision_heading)}

' + f'

{escaped(report_content.decision_blurb)}

' + f'
{"".join(parts)}
' + f'

{escaped(footer)}

' + ) + + flow_section = "" + if not self_reported: + flow_section = ( + f'

{escaped(report_content.flow_heading)}

' + + ( + '

Steps are described from the agents\' actual commands. ' + "A path is a sequence at least one trial literally took — arrows " + "connect steps inside a path, and a split shows where trials went " + "different ways. Full commands are in the trial cards below.

" + ) + + flow_html + ) + if decisions_html: + flow_section = ( + '
' + + content.flow_fold_summary() + + "" + + flow_section + + "
" + ) + + observation_html = ( + f'

{escaped(report_content.observation)}

' + if report_content.observation + else "" + ) + expected_html = ( + "" + if not report_content.expected + else ( + f'

{escaped(report_content.expected_heading)}

' + f'

{escaped(report_content.expected)}

' + ) + ) + resolved_css = _resolve_css(css, report.result.kind) + return f"""{escaped(report_content.title)} + + + +

{escaped(report_content.title)}

+

{escaped(report_content.subtitle)}

+{observation_html} + +

{escaped(report_content.scenario_heading)}

+
{escaped(report_content.scenario)}
+{expected_html} + +

{escaped(report_content.diff_heading)}

+
{diff_html}
+ +{decisions_html} + +{flow_section} + +

Trials

+
{columns}
+ +

{escaped(report_content.result_heading)}

+
{escaped(report.result.text)}
+ +

Simulation evidence from Behavior Diff +(model: {escaped(metadata.model)}, {before.total} trial(s) per variant). +{escaped(report_content.boundary)}

+""" + + +def render_document(artifact: str) -> str: + """Wrap an artifact body in a complete HTML document.""" + return ( + '' + '' + "" + artifact + "" + ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py new file mode 100644 index 0000000..294cba4 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py @@ -0,0 +1,188 @@ +"""Pure Markdown renderer for Behavior Diff reports.""" + +from reporting import content +from reporting.schema import ReportData + + +def _decision_markdown(report): + decisions = report.decisions + if not decisions.rows: + return [] + + before_total = decisions.before_count + after_total = decisions.after_count + lead_count = 0 + for row in decisions.rows: + if row.diverges: + break + lead_count += 1 + + markdown = [ + f"## {report.content.decision_heading}\n", + report.content.decision_blurb + "\n", + ] + if lead_count: + markdown.append("Decided the same way on both sides:\n") + for index, row in enumerate(decisions.rows[:lead_count], 1): + before_choice = content.branch_text(row.before, before_total) + after_choice = content.branch_text(row.after, after_total) + choice = ( + before_choice + if before_choice == after_choice + else f"before: {before_choice} · after: {after_choice}" + ) + note = f" — {row.note}" if row.note else "" + when = " *(in the final answer)*" if row.anchor == "answer" else "" + title = row.topic or row.decision + markdown.append(f"- {index}. **{title}**{when} → {choice}{note}") + markdown.append("") + if lead_count < len(decisions.rows): + markdown.append("Diverging from here:\n") + for index, row in enumerate(decisions.rows[lead_count:], lead_count + 1): + mark = ( + " ⟵ root behavior change" + if index == decisions.fork + else ( + " *(downstream)*" + if row.diverges and decisions.fork and index > decisions.fork + else "" + ) + ) + mark += " *(in the final answer)*" if row.anchor == "answer" else "" + title = f"**{row.topic}** — {row.decision}" if row.topic else row.decision + if row.diverges: + markdown.append(f"- {index}. {title}{mark}") + markdown.append( + f" - BEFORE: {content.branch_text(row.before, before_total)}" + ) + markdown.append( + f" - AFTER: {content.branch_text(row.after, after_total)}" + ) + else: + markdown.append( + f"- {index}. {row.decision} *(same)* → " + f"{content.branch_text(row.before, before_total)}" + ) + if row.note: + markdown.append(f" - note: {row.note}") + markdown.append("") + markdown.append(content.decision_footer(decisions.rows, decisions.fork)) + if decisions.fork_note: + markdown.append("\n" + decisions.fork_note) + if decisions.dropped: + markdown.append("\n" + content.dropped_rows(decisions.dropped)) + markdown.append("") + return markdown + + +def _flow_markdown(report): + flow = report.command_flow + markdown = [ + f"## {report.content.flow_heading}\n", + ( + "Steps are described from the agents' actual commands; a " + "path is a sequence at least one trial literally took. Full " + "commands are in the trial sections below.\n" + ), + ] + if flow.same: + markdown.append( + "Every trial in both variants took the same steps: " + + " → ".join(flow.shared) + + ". Differences, if any, are in the final answers below.\n" + ) + else: + markdown.append("Shared flow (every trial, both variants):\n") + markdown.extend(f"- {step}" for step in flow.shared) + markdown.append("\nDivergence:\n") + for tag, branch in (("BEFORE", flow.before), ("AFTER", flow.after)): + if not branch.paths: + markdown.append( + f"- {tag}, all {branch.total} trials → " + + (" → ".join(branch.prefix) or "(same steps as the shared flow)") + ) + continue + lead = f"- {tag}" + if branch.prefix: + lead += ", all trials → " + " → ".join(branch.prefix) + markdown.append(lead + ", then splits:") + for path in branch.paths: + markdown.append( + f" - {path.count} of {branch.total} trials → " + + " → ".join(path.steps) + ) + markdown.append("") + return markdown + + +def _count_line(variant): + count = ( + f"**{variant.count_text}**" if variant.count_emphasized else variant.count_text + ) + return count + variant.count_suffix + + +def render_markdown(report: ReportData) -> str: + """Return the complete Markdown report without accessing external state.""" + metadata = report.metadata + content_data = report.content + decisions = _decision_markdown(report) + markdown = [f"# {content_data.title}\n", content_data.subtitle + "\n"] + if content_data.observation: + markdown.append("**" + content_data.observation + "**\n") + markdown += [ + f"Model: {metadata.model} · {report.variants.before.total} trial(s) per variant.\n", + f"## {content_data.scenario_heading}\n", + content_data.scenario + "\n", + ] + if content_data.expected: + markdown += [ + f"## {content_data.expected_heading}\n", + content_data.expected + "\n", + ] + markdown += [ + f"## {content_data.diff_heading}\n", + "```diff\n" + report.rule_diff.rstrip() + "\n```\n", + ] + markdown += decisions + if metadata.trace_source != "self-reported": + flow = _flow_markdown(report) + if decisions: + markdown.append( + "
" + content.flow_fold_summary() + "\n" + ) + markdown += flow + markdown.append("
\n") + else: + markdown += flow + for variant, label in ( + (report.variants.before, f"BEFORE — {metadata.before_label}"), + (report.variants.after, f"AFTER — {metadata.after_label}"), + ): + markdown.append(f"## {label}\n") + markdown.append(_count_line(variant) + "\n") + for trial in variant.trials: + markdown.append(f"### {trial.name} — {trial.verdict}\n") + if trial.actions != "-": + markdown.append(trial.actions + "\n") + action_label = ( + "self-reported actions" + if metadata.trace_source == "self-reported" + else "commands the agent ran" + ) + markdown.append( + f"
{action_label} ({len(trial.commands)})\n\n```\n" + + "\n\n".join(command[:500] for command in trial.commands) + + "\n```\n
\n" + ) + markdown.append( + "
final answer to the user\n\n" + + trial.final.strip() + + "\n\n
\n" + ) + markdown += [ + f"## {content_data.result_heading}\n", + f"**{report.result.text}**\n", + content_data.boundary + "\n", + ] + return "\n".join(markdown) diff --git a/plugin/skills/behavior-diff/scripts/reporting/report.css b/plugin/skills/behavior-diff/scripts/reporting/report.css new file mode 100644 index 0000000..be30729 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/report.css @@ -0,0 +1,103 @@ +:root { + --ground:#f6f8f9; --panel:#ffffff; --ink:#1c2733; --muted:#5b6b7a; + --border:#d9e1e7; --accent:#0b6e75; --accent-soft:#e3f0f1; + --pass:#1a7f37; --fail:#cf222e; --blocked:#6e7781; + --pass-soft:#e8f3ea; --fail-soft:#fbebec; + --code-bg:#eef2f4; --d-add:#1a7f37; --d-del:#cf222e; +} +body { background:var(--ground); color:var(--ink); + font:15px/1.55 "IBM Plex Sans", -apple-system, "Segoe UI", sans-serif; + max-width:1080px; margin:0 auto; padding:2.5rem 1.25rem 3rem; } +h1 { font-size:1.7rem; font-weight:700; letter-spacing:-.01em; + margin:0 0 .2rem; text-wrap:balance; } +h2 { font-size:1.02rem; font-weight:600; margin:0; } +.section-label { font-size:.72rem; font-weight:600; letter-spacing:.09em; + text-transform:uppercase; color:var(--accent); margin:2rem 0 .5rem; } +.sub { color:var(--muted); margin:.2rem 0 0; max-width:62ch; } +pre { background:var(--code-bg); border:1px solid var(--border); + border-radius:6px; padding:.7rem .85rem; overflow-x:auto; + white-space:pre-wrap; margin:.5rem 0 0; + font:12.5px/1.55 "IBM Plex Mono", ui-monospace, monospace; } +.scenario { background:var(--panel); border:1px solid var(--border); + border-left:3px solid var(--accent); border-radius:6px; + padding:.85rem 1rem; max-width:62ch; white-space:pre-wrap; margin:0; + font:13.5px/1.6 "IBM Plex Mono", ui-monospace, monospace; } +.flow { max-width:760px; margin:.4rem auto 0; } +.fstep { display:flex; justify-content:space-between; align-items:baseline; + gap:1rem; background:var(--panel); border:1px solid var(--border); + border-radius:6px; padding:.5rem .8rem; } +.fstep.b { background:var(--fail-soft); border-color:var(--fail); } +.fstep.a { background:var(--pass-soft); border-color:var(--pass); } +.fcount { color:var(--muted); font-size:.8rem; white-space:nowrap; + font-variant-numeric:tabular-nums; } +.fline { width:2px; height:.7rem; background:var(--border); margin:0 auto; } +.farrow { text-align:center; color:var(--muted); font-size:.85rem; + line-height:1.4; } +.fsplit { text-align:center; color:var(--muted); font-size:.7rem; + letter-spacing:.08em; text-transform:uppercase; margin:.2rem 0 .35rem; } +.fpaths { display:grid; gap:.7rem; align-items:start; } +.fpath { min-width:0; } +.fpath-head { text-align:center; font-size:.75rem; font-weight:600; + color:var(--muted); margin:0 0 .3rem; + font-variant-numeric:tabular-nums; } +.fnote { text-align:center; color:var(--muted); font-size:.85rem; } +.dgrid { display:grid; grid-template-columns:1fr 1fr; + gap:.45rem .9rem; align-items:stretch; } +.dspan { grid-column:1 / -1; } +.dq { margin:.55rem 0 0; font-weight:600; font-size:.9rem; } +.dq:first-child { margin-top:0; } +.dnote { display:block; font-weight:400; font-size:.82rem; + color:var(--muted); margin-top:.1rem; } +.dtag { display:inline-block; margin-left:.4rem; font-size:.64rem; + font-weight:700; letter-spacing:.07em; text-transform:uppercase; + color:var(--panel); background:var(--accent); border-radius:3px; + padding:.05rem .3rem; vertical-align:.08rem; } +.dtag-same { background:var(--blocked); } +.dcell { display:block; } +.dline { padding:.05rem 0; } +.obs { background:var(--accent-soft); border-left:3px solid var(--accent); + border-radius:6px; padding:.6rem .9rem; margin:.8rem 0 0; + font-weight:600; max-width:72ch; } +.dwhen { margin-left:.45rem; font-size:.72rem; font-weight:400; + color:var(--muted); letter-spacing:.02em; } +.flowfold { margin-top:2rem; } +.flowfold > summary { font-size:.9rem; font-weight:600; } +.flowfold .section-label { margin-top:.8rem; } +.dfoot { margin:.6rem 0 0; } +.fork-label { text-align:center; color:var(--muted); font-size:.78rem; + letter-spacing:.08em; text-transform:uppercase; margin:.2rem 0 .6rem; } +.fork { display:grid; grid-template-columns:1fr 1fr; gap:1rem; } +.fork-side { text-align:center; font-size:.75rem; font-weight:700; + letter-spacing:.08em; color:var(--muted); margin:0 0 .4rem; } +.fbranch { min-width:0; } +.cols { display:grid; grid-template-columns:1fr 1fr; gap:1.1rem; + margin-top:.5rem; } +@media (max-width: 780px) { + .cols, .fork, .dgrid { grid-template-columns:1fr; } } +.col { background:var(--panel); border:1px solid var(--border); + border-radius:8px; padding:1rem 1.1rem 1.1rem; min-width:0; } +.col-head { display:flex; align-items:baseline; gap:.6rem; } +.col-note { color:var(--muted); font-size:.85rem; } +.count { margin:.35rem 0 .8rem; color:var(--muted); + font-variant-numeric:tabular-nums; } +.trial { border-top:1px solid var(--border); padding:.65rem 0 .35rem; } +.trial-head { display:flex; align-items:center; gap:.55rem; margin:0; } +.acts { margin:.25rem 0 .3rem; color:var(--muted); font-size:.92rem; } +.badge { border-radius:4px; padding:1.5px 8px; font-size:11.5px; + font-weight:700; letter-spacing:.04em; color:#fff; } +.badge.pass { background:var(--pass); } +.badge.fail { background:var(--fail); } +.badge.blocked { background:var(--blocked); } +.badge.review { background:var(--accent); } +summary { cursor:pointer; color:var(--accent); font-size:.86rem; + margin:.15rem 0; } +summary:focus-visible { outline:2px solid var(--accent); + outline-offset:2px; } +details { margin-bottom:.25rem; } +.d-add { color:var(--d-add); } .d-del { color:var(--d-del); } +.d-ctx { color:var(--muted); } +.result { background:__RESULT_BG__; color:#fff; border-radius:8px; + padding:.85rem 1.1rem; font-size:1.15rem; font-weight:700; + margin-top:.5rem; } +.footer { color:var(--muted); font-size:.85rem; margin-top:1.6rem; + max-width:70ch; } diff --git a/plugin/skills/behavior-diff/scripts/reporting/schema.py b/plugin/skills/behavior-diff/scripts/reporting/schema.py new file mode 100644 index 0000000..f2f09fc --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/schema.py @@ -0,0 +1,440 @@ +"""Immutable internal data model for Behavior Diff reports.""" + +import json +from dataclasses import asdict, dataclass +from typing import Dict, Optional, Tuple, Union + +SCHEMA_VERSION = 1 +RESULT_KINDS = ("good", "bad", "neutral") + + +@dataclass(frozen=True) +class TrialData: + name: str + verdict: str + actions: str + commands: Tuple[str, ...] + final: str + outcome: Optional[str] + + +@dataclass(frozen=True) +class VariantData: + label: str + note: str + passed: int + blocked: int + valid: int + total: int + count_text: str + count_suffix: str + count_emphasized: bool + trials: Tuple[TrialData, ...] + + +@dataclass(frozen=True) +class VariantsData: + before: VariantData + after: VariantData + + +@dataclass(frozen=True) +class FlowPathData: + steps: Tuple[str, ...] + count: int + + +@dataclass(frozen=True) +class FlowBranchData: + prefix: Tuple[str, ...] + paths: Tuple[FlowPathData, ...] + total: int + + +@dataclass(frozen=True) +class CommandFlowData: + enabled: bool + same: bool + shared: Tuple[str, ...] + before: FlowBranchData + after: FlowBranchData + + +@dataclass(frozen=True) +class DecisionChoiceData: + choice: str + count: int + + +@dataclass(frozen=True) +class DecisionRowData: + decision: str + topic: str + anchor: Union[int, str] + diverges: bool + note: str + before: Tuple[DecisionChoiceData, ...] + after: Tuple[DecisionChoiceData, ...] + + +@dataclass(frozen=True) +class DecisionData: + rows: Tuple[DecisionRowData, ...] + fork: Optional[int] + fork_note: str + dropped: int + extractor: str + before_count: int + after_count: int + + +@dataclass(frozen=True) +class MetadataData: + model: str + mode: str + vocab: str + trace_source: str + target_file: str + before_label: str + after_label: str + + +@dataclass(frozen=True) +class ContentData: + title: str + subtitle: str + observation: str + scenario_heading: str + scenario: str + expected_heading: str + expected: Optional[str] + diff_heading: str + decision_heading: str + decision_blurb: str + flow_heading: str + result_heading: str + boundary: str + + +@dataclass(frozen=True) +class ResultData: + text: str + kind: str + + +@dataclass(frozen=True) +class ReportData: + schema_version: int + metadata: MetadataData + content: ContentData + rule_diff: str + result: ResultData + variants: VariantsData + command_flow: CommandFlowData + decisions: DecisionData + + @classmethod + def from_dict(cls, data: Dict[str, object]) -> "ReportData": + data = _expect_dict(data, "report-data") + version = _expect_int( + _field(data, "schema_version", "report-data"), "schema_version" + ) + if version != SCHEMA_VERSION: + raise ValueError( + "unsupported report-data schema version: {0}".format(version) + ) + + return cls( + schema_version=version, + metadata=_metadata(_field(data, "metadata", "report-data"), "metadata"), + content=_content(_field(data, "content", "report-data"), "content"), + rule_diff=_expect_str( + _field(data, "rule_diff", "report-data"), "rule_diff" + ), + result=_result(_field(data, "result", "report-data"), "result"), + variants=_variants(_field(data, "variants", "report-data"), "variants"), + command_flow=_command_flow( + _field(data, "command_flow", "report-data"), "command_flow" + ), + decisions=_decisions(_field(data, "decisions", "report-data"), "decisions"), + ) + + def to_dict(self): + return _json_value(asdict(self)) + + def to_json(self): + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +def _metadata(value, path): + value = _expect_dict(value, path) + return MetadataData( + model=_expect_str(_field(value, "model", path), path + ".model"), + mode=_expect_str(_field(value, "mode", path), path + ".mode"), + vocab=_expect_str(_field(value, "vocab", path), path + ".vocab"), + trace_source=_expect_str( + _field(value, "trace_source", path), path + ".trace_source" + ), + target_file=_expect_str( + _field(value, "target_file", path), path + ".target_file" + ), + before_label=_expect_str( + _field(value, "before_label", path), path + ".before_label" + ), + after_label=_expect_str( + _field(value, "after_label", path), path + ".after_label" + ), + ) + + +def _content(value, path): + value = _expect_dict(value, path) + return ContentData( + title=_expect_str(_field(value, "title", path), path + ".title"), + subtitle=_expect_str(_field(value, "subtitle", path), path + ".subtitle"), + observation=_expect_str( + _field(value, "observation", path), path + ".observation" + ), + scenario_heading=_expect_str( + _field(value, "scenario_heading", path), path + ".scenario_heading" + ), + scenario=_expect_str(_field(value, "scenario", path), path + ".scenario"), + expected_heading=_expect_str( + _field(value, "expected_heading", path), path + ".expected_heading" + ), + expected=_expect_optional_str( + _field(value, "expected", path), path + ".expected" + ), + diff_heading=_expect_str( + _field(value, "diff_heading", path), path + ".diff_heading" + ), + decision_heading=_expect_str( + _field(value, "decision_heading", path), path + ".decision_heading" + ), + decision_blurb=_expect_str( + _field(value, "decision_blurb", path), path + ".decision_blurb" + ), + flow_heading=_expect_str( + _field(value, "flow_heading", path), path + ".flow_heading" + ), + result_heading=_expect_str( + _field(value, "result_heading", path), path + ".result_heading" + ), + boundary=_expect_str(_field(value, "boundary", path), path + ".boundary"), + ) + + +def _result(value, path): + value = _expect_dict(value, path) + kind = _expect_str(_field(value, "kind", path), path + ".kind") + if kind not in RESULT_KINDS: + _invalid(path + ".kind", "one of " + ", ".join(RESULT_KINDS)) + return ResultData( + text=_expect_str(_field(value, "text", path), path + ".text"), + kind=kind, + ) + + +def _variants(value, path): + value = _expect_dict(value, path) + return VariantsData( + before=_variant(_field(value, "before", path), path + ".before"), + after=_variant(_field(value, "after", path), path + ".after"), + ) + + +def _variant(value, path): + value = _expect_dict(value, path) + trials = _expect_list(_field(value, "trials", path), path + ".trials") + return VariantData( + label=_expect_str(_field(value, "label", path), path + ".label"), + note=_expect_str(_field(value, "note", path), path + ".note"), + passed=_expect_int(_field(value, "passed", path), path + ".passed"), + blocked=_expect_int(_field(value, "blocked", path), path + ".blocked"), + valid=_expect_int(_field(value, "valid", path), path + ".valid"), + total=_expect_int(_field(value, "total", path), path + ".total"), + count_text=_expect_str(_field(value, "count_text", path), path + ".count_text"), + count_suffix=_expect_str( + _field(value, "count_suffix", path), path + ".count_suffix" + ), + count_emphasized=_expect_bool( + _field(value, "count_emphasized", path), path + ".count_emphasized" + ), + trials=tuple( + _trial(item, "{0}.trials[{1}]".format(path, index)) + for index, item in enumerate(trials) + ), + ) + + +def _trial(value, path): + value = _expect_dict(value, path) + return TrialData( + name=_expect_str(_field(value, "name", path), path + ".name"), + verdict=_expect_str(_field(value, "verdict", path), path + ".verdict"), + actions=_expect_str(_field(value, "actions", path), path + ".actions"), + commands=_string_tuple(_field(value, "commands", path), path + ".commands"), + final=_expect_str(_field(value, "final", path), path + ".final"), + outcome=_expect_optional_str(_field(value, "outcome", path), path + ".outcome"), + ) + + +def _command_flow(value, path): + value = _expect_dict(value, path) + return CommandFlowData( + enabled=_expect_bool(_field(value, "enabled", path), path + ".enabled"), + same=_expect_bool(_field(value, "same", path), path + ".same"), + shared=_string_tuple(_field(value, "shared", path), path + ".shared"), + before=_flow_branch(_field(value, "before", path), path + ".before"), + after=_flow_branch(_field(value, "after", path), path + ".after"), + ) + + +def _flow_branch(value, path): + value = _expect_dict(value, path) + paths = _expect_list(_field(value, "paths", path), path + ".paths") + return FlowBranchData( + prefix=_string_tuple(_field(value, "prefix", path), path + ".prefix"), + paths=tuple( + _flow_path(item, "{0}.paths[{1}]".format(path, index)) + for index, item in enumerate(paths) + ), + total=_expect_int(_field(value, "total", path), path + ".total"), + ) + + +def _flow_path(value, path): + value = _expect_dict(value, path) + return FlowPathData( + steps=_string_tuple(_field(value, "steps", path), path + ".steps"), + count=_expect_int(_field(value, "count", path), path + ".count"), + ) + + +def _decisions(value, path): + value = _expect_dict(value, path) + rows = _expect_list(_field(value, "rows", path), path + ".rows") + return DecisionData( + rows=tuple( + _decision_row(item, "{0}.rows[{1}]".format(path, index)) + for index, item in enumerate(rows) + ), + fork=_expect_optional_int(_field(value, "fork", path), path + ".fork"), + fork_note=_expect_str(_field(value, "fork_note", path), path + ".fork_note"), + dropped=_expect_int(_field(value, "dropped", path), path + ".dropped"), + extractor=_expect_str(_field(value, "extractor", path), path + ".extractor"), + before_count=_expect_int( + _field(value, "before_count", path), path + ".before_count" + ), + after_count=_expect_int( + _field(value, "after_count", path), path + ".after_count" + ), + ) + + +def _decision_row(value, path): + value = _expect_dict(value, path) + before = _expect_list(_field(value, "before", path), path + ".before") + after = _expect_list(_field(value, "after", path), path + ".after") + return DecisionRowData( + decision=_expect_str(_field(value, "decision", path), path + ".decision"), + topic=_expect_str(_field(value, "topic", path), path + ".topic"), + anchor=_expect_anchor(_field(value, "anchor", path), path + ".anchor"), + diverges=_expect_bool(_field(value, "diverges", path), path + ".diverges"), + note=_expect_str(_field(value, "note", path), path + ".note"), + before=tuple( + _decision_choice(item, "{0}.before[{1}]".format(path, index)) + for index, item in enumerate(before) + ), + after=tuple( + _decision_choice(item, "{0}.after[{1}]".format(path, index)) + for index, item in enumerate(after) + ), + ) + + +def _decision_choice(value, path): + value = _expect_dict(value, path) + return DecisionChoiceData( + choice=_expect_str(_field(value, "choice", path), path + ".choice"), + count=_expect_int(_field(value, "count", path), path + ".count"), + ) + + +def _field(value, name, path): + try: + return value[name] + except KeyError: + _invalid(path + "." + name, "present field") + + +def _expect_dict(value, path): + if type(value) is not dict: + _invalid(path, "dict") + return value + + +def _expect_list(value, path): + if type(value) is not list: + _invalid(path, "list") + return value + + +def _expect_str(value, path): + if type(value) is not str: + _invalid(path, "string") + return value + + +def _expect_bool(value, path): + if type(value) is not bool: + _invalid(path, "boolean") + return value + + +def _expect_int(value, path): + if type(value) is not int: + _invalid(path, "integer") + return value + + +def _expect_optional_str(value, path): + if value is None: + return None + return _expect_str(value, path) + + +def _expect_optional_int(value, path): + if value is None: + return None + return _expect_int(value, path) + + +def _expect_anchor(value, path): + if type(value) is int or type(value) is str: + return value + _invalid(path, "integer or string") + + +def _string_tuple(value, path): + value = _expect_list(value, path) + return tuple( + _expect_str(item, "{0}[{1}]".format(path, index)) + for index, item in enumerate(value) + ) + + +def _invalid(path, expected): + raise ValueError( + "invalid report-data field {0}: expected {1}".format(path, expected) + ) + + +def _json_value(value): + if isinstance(value, tuple): + return [_json_value(item) for item in value] + if isinstance(value, list): + return [_json_value(item) for item in value] + if isinstance(value, dict): + return {key: _json_value(item) for key, item in value.items()} + return value diff --git a/tests/fixtures/report-rendering/captured/report-artifact.html b/tests/fixtures/report-rendering/captured/report-artifact.html new file mode 100644 index 0000000..f007318 --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report-artifact.html @@ -0,0 +1,140 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ +

Scenario

+
Compare the two instruction snapshots.
+ + +

Diff of AGENTS.md — the only difference between the variants

+
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

Decision diff — top divergences

A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ +
Flow diff — command-derived (deterministic, no model involved)

Flow diff — where the variants diverge

Steps are described from the agents' actual commands. A path is a sequence at least one trial literally took — arrows connect steps inside a path, and a split shows where trials went different ways. Full commands are in the trial cards below.

paths diverge here

BEFORE

all 1 trials

(same steps as the shared flow)

AFTER

all 1 trials

Run tests
+ +

Trials

+

Before

current file

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

Commands the agent ran (1)
$ Read: AGENTS.md
Final answer to the user
Before answer

After

your change applied

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

Commands the agent ran (2)
$ Read: AGENTS.md
+
+$ Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ +

Result

+
No automatic verdict — compare the flows and final answers
+ +

Simulation evidence from Behavior Diff +(model: contract, 1 trial(s) per variant). +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks.

diff --git a/tests/fixtures/report-rendering/captured/report.html b/tests/fixtures/report-rendering/captured/report.html new file mode 100644 index 0000000..f9ff39b --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report.html @@ -0,0 +1,141 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ +

Scenario

+
Compare the two instruction snapshots.
+ + +

Diff of AGENTS.md — the only difference between the variants

+
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

Decision diff — top divergences

A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ +
Flow diff — command-derived (deterministic, no model involved)

Flow diff — where the variants diverge

Steps are described from the agents' actual commands. A path is a sequence at least one trial literally took — arrows connect steps inside a path, and a split shows where trials went different ways. Full commands are in the trial cards below.

paths diverge here

BEFORE

all 1 trials

(same steps as the shared flow)

AFTER

all 1 trials

Run tests
+ +

Trials

+

Before

current file

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

Commands the agent ran (1)
$ Read: AGENTS.md
Final answer to the user
Before answer

After

your change applied

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

Commands the agent ran (2)
$ Read: AGENTS.md
+
+$ Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ +

Result

+
No automatic verdict — compare the flows and final answers
+ +

Simulation evidence from Behavior Diff +(model: contract, 1 trial(s) per variant). +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks.

+ \ No newline at end of file diff --git a/tests/fixtures/report-rendering/captured/report.md b/tests/fixtures/report-rendering/captured/report.md new file mode 100644 index 0000000..da160fd --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report.md @@ -0,0 +1,98 @@ +# Live contract + +Synthetic contract fixture. + +**Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.** + +Model: contract · 1 trial(s) per variant. + +## Scenario + +Compare the two instruction snapshots. + +## Diff of AGENTS.md — the only difference between the variants + +```diff +--- AGENTS.md (before) ++++ AGENTS.md (after) +@@ -1 +1 @@ +-Original project instructions. ++Updated project instructions. +``` + +## Decision diff — top divergences + +A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it. + +Diverging from here: + +- 1. **Evidence choice** — Which evidence was used? ⟵ root behavior change + - BEFORE: read only + - AFTER: read and test + +One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). + +Synthetic fixture. + +
Flow diff — command-derived (deterministic, no model involved) + +## Flow diff — where the variants diverge + +Steps are described from the agents' actual commands; a path is a sequence at least one trial literally took. Full commands are in the trial sections below. + +Shared flow (every trial, both variants): + + +Divergence: + +- BEFORE, all 1 trials → (same steps as the shared flow) +- AFTER, all 1 trials → Run tests + +
+ +## BEFORE — current file + +1 valid trial(s) · no automatic grading (blocked: 0) + +### before-1 — REVIEW + +
commands the agent ran (1) + +``` +Read: AGENTS.md +``` +
+ +
final answer to the user + +Before answer + +
+ +## AFTER — your change applied + +1 valid trial(s) · no automatic grading (blocked: 0) + +### after-1 — REVIEW + +
commands the agent ran (2) + +``` +Read: AGENTS.md + +Test: bash behavior-diff/tests/live-report-contract.sh +``` +
+ +
final answer to the user + +After answer + +
+ +## Result + +**No automatic verdict — compare the flows and final answers** + +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks. diff --git a/tests/fixtures/report-rendering/self-reported/report-artifact.html b/tests/fixtures/report-rendering/self-reported/report-artifact.html new file mode 100644 index 0000000..5ce7671 --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report-artifact.html @@ -0,0 +1,140 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ +

Scenario

+
Compare the two instruction snapshots.
+ + +

Diff of AGENTS.md — the only difference between the variants

+
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

Decision diff — top divergences

A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ + + +

Trials

+

Before

parent snapshot <baseline>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

self-reported actions (1)
Read: AGENTS.md
Final answer to the user
Before answer

After

target snapshot <candidate>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

self-reported actions (2)
Read: AGENTS.md
+
+Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ +

Result

+
No automatic verdict — compare the reported actions, decision diff, and final answers
+ +

Simulation evidence from Behavior Diff +(model: contract, 1 trial(s) per variant). +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks.

diff --git a/tests/fixtures/report-rendering/self-reported/report.html b/tests/fixtures/report-rendering/self-reported/report.html new file mode 100644 index 0000000..73f6a40 --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report.html @@ -0,0 +1,141 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ +

Scenario

+
Compare the two instruction snapshots.
+ + +

Diff of AGENTS.md — the only difference between the variants

+
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

Decision diff — top divergences

A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ + + +

Trials

+

Before

parent snapshot <baseline>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

self-reported actions (1)
Read: AGENTS.md
Final answer to the user
Before answer

After

target snapshot <candidate>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

self-reported actions (2)
Read: AGENTS.md
+
+Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ +

Result

+
No automatic verdict — compare the reported actions, decision diff, and final answers
+ +

Simulation evidence from Behavior Diff +(model: contract, 1 trial(s) per variant). +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks.

+ \ No newline at end of file diff --git a/tests/fixtures/report-rendering/self-reported/report.md b/tests/fixtures/report-rendering/self-reported/report.md new file mode 100644 index 0000000..659c27d --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report.md @@ -0,0 +1,82 @@ +# Live contract + +Synthetic contract fixture. + +**Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.** + +Model: contract · 1 trial(s) per variant. + +## Scenario + +Compare the two instruction snapshots. + +## Diff of AGENTS.md — the only difference between the variants + +```diff +--- AGENTS.md (before) ++++ AGENTS.md (after) +@@ -1 +1 @@ +-Original project instructions. ++Updated project instructions. +``` + +## Decision diff — top divergences + +A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it. + +Diverging from here: + +- 1. **Evidence choice** — Which evidence was used? ⟵ root behavior change + - BEFORE: read only + - AFTER: read and test + +One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). + +Synthetic fixture. + +## BEFORE — parent snapshot + +1 valid trial(s) · no automatic grading (blocked: 0) + +### before-1 — REVIEW + +
self-reported actions (1) + +``` +Read: AGENTS.md +``` +
+ +
final answer to the user + +Before answer + +
+ +## AFTER — target snapshot + +1 valid trial(s) · no automatic grading (blocked: 0) + +### after-1 — REVIEW + +
self-reported actions (2) + +``` +Read: AGENTS.md + +Test: bash behavior-diff/tests/live-report-contract.sh +``` +
+ +
final answer to the user + +After answer + +
+ +## Result + +**No automatic verdict — compare the reported actions, decision diff, and final answers** + +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks. diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index c51c5f2..f7bc0d8 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -16,6 +16,8 @@ claude_manifest=$here/../plugin/.claude-plugin/plugin.json codex_manifest=$here/../plugin/.codex-plugin/plugin.json readme=$here/../README.md +fixture_root=$here/fixtures/report-rendering +update_report_fixtures=false require_output() { grep -qF -- "$1" "$2" || fail "$3" } @@ -103,6 +105,69 @@ progress() { printf '[report] %s\n' "$1" } +usage() { + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 +} + +copy_report_fixtures() { + local mode=$1 + local run=$2 + local fixture_dir=$fixture_root/$mode + + mkdir -p "$fixture_dir" + cp "$run/report.md" "$fixture_dir/report.md" + cp "$run/report.html" "$fixture_dir/report.html" + cp "$run/report-artifact.html" "$fixture_dir/report-artifact.html" +} + +require_exact_report() { + local mode=$1 + local report=$2 + local fixture + fixture=$fixture_root/$mode/$(basename "$report") + + if ! cmp -s "$fixture" "$report"; then + printf 'Rendered report differs from fixture: %s\n' "$fixture" >&2 + if diff -u "$fixture" "$report" >&2; then + fail "rendered report comparison failed unexpectedly: $report" + else + fail "rendered report differs from fixture: $report" + fi + fi +} + +case $# in + 0) ;; + 1) + [[ $1 == --update-report-fixtures ]] || usage + update_report_fixtures=true + ;; + *) usage ;; +esac + +require_usage() { + local stderr=$tmp/usage-stderr.txt + local stdout=$tmp/usage-stdout.txt + local expected=$tmp/usage-expected.txt + local status + + if bash "$0" "$@" >"$stdout" 2>"$stderr"; then + fail "invalid arguments succeeded: $*" + else + status=$? + fi + [[ $status == 2 ]] || fail "invalid arguments returned $status instead of 2: $*" + [[ ! -s $stdout ]] || fail "invalid arguments wrote to stdout: $*" + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >"$expected" + cmp -s "$expected" "$stderr" || + fail "invalid arguments did not print the exact usage diagnostic: $*" +} + +require_usage --unknown-option +require_usage --update-report-fixtures surplus +python3 "$here/report-schema-test.py" + progress 'Validate manifests and live-skill reporting contract' [[ -x $spacedock_fixture_script ]] || fail 'renamed Spacedock fixture builder is missing or not executable' @@ -328,9 +393,66 @@ reject_output 'A decision is not an action' "$captured_prompt" \ progress 'Render captured and self-reported reports' python3 "$renderer" "$self_run" "$self_run" contract \ - "$self_run/config.json" >/dev/null + "$self_run/config.json" >"$self_run/render.stdout" +self_run_path=$(cd "$self_run" && pwd -P) +if ! printf '%s\n' \ + 'mode review · BEFORE pass 0/1 · AFTER pass 0/1 → No automatic verdict — compare the reported actions, decision diff, and final answers' \ + "report: $self_run_path/report.md" \ + "page: $self_run_path/report.html" | + cmp -s - "$self_run/render.stdout"; then + fail 'renderer stdout changed' +fi python3 "$renderer" "$captured_run" "$captured_run" contract \ "$captured_run/config.json" >/dev/null +if [[ $update_report_fixtures == true ]]; then + copy_report_fixtures captured "$captured_run" + copy_report_fixtures self-reported "$self_run" +fi + +for run in "$self_run" "$captured_run"; do + [[ -f $run/report-data.json ]] || + fail "renderer did not write report-data.json: $run" + python3 "$here/report-schema-test.py" "$run/report-data.json" +done + +[[ $(jq -r '.schema_version' "$captured_run/report-data.json") == 1 ]] || + fail 'captured report data schema version is not 1' +[[ $(jq -r '.metadata.trace_source' "$captured_run/report-data.json") == captured ]] || + fail 'captured report data provenance is not captured' +[[ $(jq -r '.metadata.trace_source' "$self_run/report-data.json") == self-reported ]] || + fail 'self-reported report data provenance is not self-reported' +[[ $(jq -r '.variants.after.trials[0].commands | join("|")' "$captured_run/report-data.json") == 'Read: AGENTS.md|Test: bash behavior-diff/tests/live-report-contract.sh' ]] || + fail 'report data does not preserve after commands in order' +[[ $(jq -r '.command_flow.enabled == false and (.command_flow.shared | length == 0) and (.command_flow.before.prefix | length == 0) and (.command_flow.before.paths | length == 0) and (.command_flow.after.prefix | length == 0) and (.command_flow.after.paths | length == 0)' "$self_run/report-data.json") == true ]] || + fail 'self-reported report data must disable and empty command flow' + +graded_run=$tmp/graded +build_run "$graded_run" captured +python3 "$renderer" "$graded_run" "$graded_run" contract >/dev/null +require_output '**0 of 1 valid trials met the expectation** (blocked: 0)' \ + "$graded_run/report.md" \ + 'graded Markdown count must emphasize only the expectation result' + +invalid_decisions_run=$tmp/invalid-decisions +build_run "$invalid_decisions_run" captured +printf '%s\n' '{"chain":[{"decision":"Synthetic decision","topic":"","anchor":"work","before":[{"choice":"before","n":1}],"after":[{"choice":"after","n":1}],"diverges":true}],"fork":2,"counts":{"before":1,"after":1}}' \ + >"$invalid_decisions_run/decisions.json" +cat >"$invalid_decisions_run/before-1/trace.jsonl" <<'JSON' +{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","input":{"command":"Read: AGENTS.md"}}]}} +[] +{"type":"assistant","message":[]} +{"type":"assistant","message":{"content":[null,{"type":"tool_use","input":null},{"type":"tool_use","input":{"command":17}},{"type":"tool_use","name":17,"input":{"file_path":"AGENTS.md"}}]}} +{"type":"result","result":"Before answer"} +JSON +python3 "$renderer" "$invalid_decisions_run" "$invalid_decisions_run" contract \ + "$invalid_decisions_run/config.json" >/dev/null +[[ $(jq '.decisions.rows | length' "$invalid_decisions_run/report-data.json") == 0 ]] || + fail 'out-of-range decision fork must fall back to empty decisions' + +for report in report.md report.html report-artifact.html; do + require_exact_report captured "$captured_run/$report" + require_exact_report self-reported "$self_run/$report" +done read_action='Read: AGENTS.md' test_action='Test: bash behavior-diff/tests/live-report-contract.sh' diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py new file mode 100755 index 0000000..415e7a8 --- /dev/null +++ b/tests/report-schema-test.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +import contextlib +import copy +import importlib.util +import io +import json +import os +import sys +import tempfile +from pathlib import Path + +scripts = Path(__file__).resolve().parents[1] / "plugin/skills/behavior-diff/scripts" +sys.path.insert(0, str(scripts)) + +from reporting import content # noqa: E402 +from reporting.schema import ( # noqa: E402 + CommandFlowData, + DecisionChoiceData, + DecisionRowData, + FlowPathData, + ReportData, + TrialData, +) + + +def synthetic_raw(): + return { + "schema_version": 1, + "metadata": { + "model": "synthetic/model", + "mode": "review", + "vocab": "generic", + "trace_source": "captured", + "target_file": "AGENTS.md", + "before_label": "current file", + "after_label": "your change applied", + }, + "content": { + "title": "Synthetic report", + "subtitle": "Synthetic subtitle.", + "observation": "Synthetic observation.", + "scenario_heading": "Scenario", + "scenario": "Compare two synthetic files.", + "expected_heading": "Expected behavior", + "expected": "Test the changed behavior.", + "diff_heading": "Diff of AGENTS.md — the only difference between the variants", + "decision_heading": "Decision diff — top divergences", + "decision_blurb": "Synthetic decision explanation.", + "flow_heading": "Flow diff — where the variants diverge", + "result_heading": "Result", + "boundary": "Synthetic evidence only.", + }, + "rule_diff": "--- before\n+++ after\n", + "result": {"text": "No automatic verdict", "kind": "neutral"}, + "variants": { + "before": { + "label": "Before", + "note": "current file", + "passed": 0, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "2 valid trials", + "count_suffix": "", + "count_emphasized": False, + "trials": [ + { + "name": "before-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md"], + "final": "Before first answer", + "outcome": None, + }, + { + "name": "before-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "Before second answer", + "outcome": "Reviewed file", + }, + ], + }, + "after": { + "label": "After", + "note": "your change applied", + "passed": 1, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "1 of 2 valid trials passed", + "count_suffix": " (blocked: 0)", + "count_emphasized": True, + "trials": [ + { + "name": "after-1", + "verdict": "PASS", + "actions": "-", + "commands": ["read AGENTS.md", "run tests"], + "final": "After first answer", + "outcome": "Tested behavior", + }, + { + "name": "after-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "After second answer", + "outcome": None, + }, + ], + }, + }, + "command_flow": { + "enabled": True, + "same": False, + "shared": ["Read files", "Compare output"], + "before": { + "prefix": ["Review result"], + "paths": [ + {"steps": ["Stop"], "count": 1}, + {"steps": ["Explain"], "count": 1}, + ], + "total": 2, + }, + "after": { + "prefix": ["Run tests"], + "paths": [{"steps": ["Explain"], "count": 2}], + "total": 2, + }, + }, + "decisions": { + "rows": [ + { + "decision": "Use evidence", + "topic": "Evidence", + "anchor": 2, + "diverges": True, + "note": "Synthetic divergence.", + "before": [ + {"choice": "read only", "count": 2}, + {"choice": "search", "count": 1}, + ], + "after": [{"choice": "read and test", "count": 2}], + }, + { + "decision": "State result", + "topic": "Delivery", + "anchor": "final answer", + "diverges": False, + "note": "", + "before": [{"choice": "explain", "count": 2}], + "after": [{"choice": "explain", "count": 2}], + }, + ], + "fork": 1, + "fork_note": "Synthetic fork.", + "dropped": 0, + "extractor": "synthetic extractor", + "before_count": 2, + "after_count": 2, + }, + } + + +def assert_round_trip(raw): + report = ReportData.from_dict(raw) + assert report.schema_version == 1 + assert report.to_dict() == raw + assert report.to_json() == json.dumps(raw, indent=2, sort_keys=True) + "\n" + return report + + +def assert_rejected(raw, message): + try: + ReportData.from_dict(raw) + except ValueError as error: + assert str(error) == message + else: + raise AssertionError("invalid report data was accepted") + + +def assert_render_import_safe(): + render_path = scripts / "render.py" + original_cwd = Path.cwd() + original_argv = sys.argv + stdout = io.StringIO() + stderr = io.StringIO() + module_name = "_behavior_diff_render_import_test" + + try: + with tempfile.TemporaryDirectory() as directory: + os.chdir(directory) + sys.argv = [str(render_path)] + spec = importlib.util.spec_from_file_location(module_name, render_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + spec.loader.exec_module(module) + assert list(Path(".").iterdir()) == [] + assert stdout.getvalue() == "" + assert stderr.getvalue() == "" + assert callable(module.main) + finally: + sys.modules.pop(module_name, None) + sys.argv = original_argv + os.chdir(original_cwd) + + +def main(): + assert_render_import_safe() + raw = synthetic_raw() + report = assert_round_trip(raw) + assert isinstance(report.variants.before.trials[0], TrialData) + assert isinstance(report.command_flow, CommandFlowData) + assert isinstance(report.command_flow.before.paths[0], FlowPathData) + assert isinstance(report.decisions.rows[0], DecisionRowData) + assert isinstance(report.decisions.rows[0].before[0], DecisionChoiceData) + assert [trial.name for trial in report.variants.before.trials] == [ + "before-1", + "before-2", + ] + assert [path.steps for path in report.command_flow.before.paths] == [ + ("Stop",), + ("Explain",), + ] + assert [row.anchor for row in report.decisions.rows] == [2, "final answer"] + caution = "CAUTION — one trial per side" + for before_total, after_total, expected in ( + (1, 2, False), + (2, 1, False), + (1, 1, True), + ): + report_content = content.build_content( + {}, + "Synthetic scenario", + "review", + "captured", + "AGENTS.md", + report.decisions, + before_total, + after_total, + ) + assert (caution in report_content.decision_blurb) is expected + assert [choice.choice for choice in report.decisions.rows[0].before] == [ + "read only", + "search", + ] + + assert_rejected( + dict(raw, schema_version=2), + "unsupported report-data schema version: 2", + ) + assert_rejected( + dict(raw, schema_version=True), + "invalid report-data field schema_version: expected integer", + ) + assert_rejected( + dict(raw, schema_version=1.0), + "invalid report-data field schema_version: expected integer", + ) + + invalid_commands = copy.deepcopy(raw) + invalid_commands["variants"]["before"]["trials"][0]["commands"] = "read AGENTS.md" + assert_rejected( + invalid_commands, + "invalid report-data field variants.before.trials[0].commands: expected list", + ) + invalid_model = copy.deepcopy(raw) + invalid_model["metadata"]["model"] = ["synthetic/model"] + assert_rejected( + invalid_model, + "invalid report-data field metadata.model: expected string", + ) + invalid_count = copy.deepcopy(raw) + invalid_count["command_flow"]["before"]["paths"][0]["count"] = True + assert_rejected( + invalid_count, + "invalid report-data field command_flow.before.paths[0].count: expected integer", + ) + invalid_suffix = copy.deepcopy(raw) + invalid_suffix["variants"]["after"]["count_suffix"] = True + assert_rejected( + invalid_suffix, + "invalid report-data field variants.after.count_suffix: expected string", + ) + invalid_result_kind = copy.deepcopy(raw) + invalid_result_kind["result"]["kind"] = "future" + assert_rejected( + invalid_result_kind, + "invalid report-data field result.kind: expected one of good, bad, neutral", + ) + invalid_result_kind_type = copy.deepcopy(raw) + invalid_result_kind_type["result"]["kind"] = None + assert_rejected( + invalid_result_kind_type, + "invalid report-data field result.kind: expected string", + ) + + from reporting.render_html import _resolve_css, render_artifact, render_document + + assert _resolve_css(".result { background:__RESULT_BG__; }", "good") == ( + ".result { background:var(--pass); }" + ) + assert _resolve_css(".result { background:__RESULT_BG__; }", "bad") == ( + ".result { background:var(--fail); }" + ) + assert _resolve_css(".result { background:__RESULT_BG__; }", "neutral") == ( + ".result { background:var(--accent); }" + ) + try: + _resolve_css(".result { background:__RESULT_BG__; }", "future") + except ValueError as error: + assert str(error) == "unsupported report result kind: future" + else: + raise AssertionError("unsupported report result kind was accepted") + try: + _resolve_css(".result {}", "future") + except ValueError as error: + assert str(error) == "report.css must contain __RESULT_BG__ exactly once" + else: + raise AssertionError("CSS token validation lost precedence") + for css in (".result {}", "__RESULT_BG__ __RESULT_BG__"): + try: + _resolve_css(css, "good") + except ValueError as error: + assert str(error) == "report.css must contain __RESULT_BG__ exactly once" + else: + raise AssertionError("invalid CSS token count was accepted") + + artifact = render_artifact(report, ".result { background:__RESULT_BG__; }") + assert artifact == render_artifact(report, ".result { background:__RESULT_BG__; }") + document = render_document(artifact) + assert document == render_document(artifact) + escaping_raw = copy.deepcopy(raw) + escaping_raw["variants"]["before"]["trials"][0]["verdict"] = 'REVIEW">