From 4a6b2772459272ade873e08f4138ac3bbc2be2d3 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 11:30:38 +0800 Subject: [PATCH 01/19] docs: design report rendering structure Signed-off-by: Kent Huang --- .../2026-09-04-report-rendering-structure.md | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 plans/2026-09-04-report-rendering-structure.md diff --git a/plans/2026-09-04-report-rendering-structure.md b/plans/2026-09-04-report-rendering-structure.md new file mode 100644 index 0000000..a4534a0 --- /dev/null +++ b/plans/2026-09-04-report-rendering-structure.md @@ -0,0 +1,298 @@ +# Report Rendering Structure Design + +**Status:** Approved in conversation on 2026-09-04 + +## Goal + +Make the Behavior Diff report pipeline easy to change without coupling report data, shared wording, Markdown structure, HTML structure, and HTML style. + +This change restructures report generation only. For fixed synthetic inputs, the existing `report.md`, `report.html`, and `report-artifact.html` outputs must remain byte-for-byte unchanged. + +## Problem + +`plugin/skills/behavior-diff/scripts/render.py` is one 923-line script. It currently: + +1. reads config, grades, traces, the task, the instruction diff, and optional decisions; +2. computes trial and variant facts; +3. chooses result and explanatory wording; +4. builds command and decision flows; +5. renders Markdown; +6. renders HTML and embedded CSS; +7. writes all report files. + +Most of this work happens at module load through global values. The HTML and Markdown paths repeat decision and flow logic. Some data is already formatted for Markdown and then changed for HTML. For example, HTML removes Markdown `**` markers from a shared count string. + +This makes a small style or wording change risky. A change in one format can affect data calculation or the other format. + +## Current input boundary + +The existing persisted run files remain the source of evidence: + +- `config.json` +- `task.md` +- `grades.tsv` +- each trial's canonical `trace.jsonl` +- the before and after target files, or `rule.md` as the existing fallback +- optional `decisions.json` + +`behavior-diff.sh` and `behavior-diff-live` both produce this shape. The restructure must not create a second pipeline for either skill. + +## Accepted constraints + +- Keep the current `render.py RUN_DIR CAPSULE_DIR MODEL [CONFIG_JSON]` command. +- Keep its stdout text, required-input errors, config errors, and optional-decisions fallback. +- Keep writing `report.md`, `report-artifact.html`, and `report.html`. +- Add `report-data.json` as an internal, versioned artifact. +- Keep one in-memory typed report structure and serialize that same structure to JSON. +- Keep all generated reports deterministic. +- Keep Python standard-library only. +- Keep `report.html` as one portable file. Maintain CSS separately in the source tree, then inline it during generation. +- Keep shared wording in one content layer by default. +- Allow a renderer to own wording only when the formats intentionally differ. +- Preserve captured and self-reported evidence rules. +- Do not change trials, grading, decision extraction, report opening, report content, or visual style. +- Keep latent graded/demo mode and the existing separate run/capsule input contract in this phase. + +## Chosen architecture + +Use one layered Python package behind the existing `render.py` entry point. + +```text +run artifacts + config.json · task.md · grades.tsv · trace.jsonl · decisions.json + | + v +reporting/load.py + parse evidence · compute counts · flow · result · diff + | + v +reporting/schema.py + reporting/content.py + typed ReportData · shared wording · schema_version: 1 + | + +----------------> report-data.json + | + +----------------> reporting/render_markdown.py -> report.md + | + +----------------> reporting/render_html.py + report.css + -> report-artifact.html + -> report.html + +render.py remains the only command entry point and file writer. +``` + +### `reporting/schema.py` + +Own the typed report structure. Do not use `model.py`; “model” is easy to confuse with an AI model. + +The main type is `ReportData`. Nested immutable records cover: + +- configuration and metadata; +- trials and variants; +- result state; +- command-derived flow; +- decision rows and choices; +- shared report content. + +`schema.py` also owns deterministic `to_dict` and `from_dict` conversion. It does not read files, write files, render markup, or call an AI model. + +### `reporting/load.py` + +Read the current run artifacts and build `ReportData`. + +It owns the format-neutral behavior currently mixed into `render.py`: + +- canonical trace parsing; +- variant totals and blocked counts; +- result kind selection; +- command classification and flow calculation; +- target-file diff calculation; +- optional decision loading and validation fallback; +- the observed single-run summary facts. + +It imports shared copy functions from `content.py`. It does not create Markdown, HTML, CSS, or output files. + +### `reporting/content.py` + +Own shared human-readable copy: + +- title and subtitle defaults; +- section names; +- result text; +- count text; +- observation text; +- decision and flow explanations; +- the simulation boundary. + +HTML and Markdown use this copy unless the current outputs intentionally differ. Existing intentional differences remain renderer-owned. Examples include HTML's captured-command `$ ` prefix and the formats' different heading markup. + +### `reporting/render_markdown.py` + +Expose one pure function: + +```python +render_markdown(report: ReportData) -> str +``` + +It owns only Markdown presentation: + +- section order; +- headings; +- fenced blocks; +- Markdown lists; +- `
` blocks already used by the current Markdown report; +- Markdown-specific command display. + +It does not read run files or write `report.md`. + +### `reporting/render_html.py` + +Expose pure functions: + +```python +render_artifact(report: ReportData, css: str) -> str +render_document(artifact: str) -> str +``` + +It owns: + +- HTML escaping; +- the report DOM and component layout; +- HTML-only command display; +- the embedded style element; +- the standalone document wrapper. + +It does not read run files or write output files. + +### `reporting/report.css` + +Contain the current static CSS exactly as it appears in generated HTML. `render.py` reads this source and passes it to the HTML renderer. The renderer inlines it, so generated HTML stays portable. + +### `render.py` + +Remain the compatible command and the only file-writing layer: + +1. parse the existing arguments; +2. call `load.py` to build `ReportData`; +3. serialize the data deterministically; +4. render Markdown, artifact HTML, and standalone HTML in memory; +5. after every renderer succeeds, write all four files; +6. print the existing summary and output paths unchanged. + +Building every output string before writing prevents a renderer error from pairing one newly written report with stale files from another format. + +## Internal `report-data.json` contract + +`report-data.json` is internal. It is not a public plugin API and has no cross-release compatibility promise. It must contain `"schema_version": 1` so repository code can reject or migrate a different shape deliberately. + +The top-level shape is: + +```text +schema_version +metadata +content +rule_diff +result +variants +command_flow +decisions +``` + +The data must contain the facts and shared copy needed to regenerate both visible formats. It must not contain rendered Markdown, rendered HTML, CSS, filesystem paths outside the report evidence, or raw model/provider credentials. + +Both renderers consume the same in-memory `ReportData`. `report-data.json` is the deterministic serialized copy of that object. `ReportData.from_dict` must load the file for tests and future internal tools. + +## Evidence separation + +Captured and self-reported runs must stay distinct. + +- Captured traces may produce the command-derived flow. +- Self-reported actions must never enter command classification. +- Self-reported reports keep their current evidence warning. +- Missing or invalid optional `decisions.json` removes only the decision section and keeps the current fallback report. +- Trial command/action order and final answers remain unchanged. + +## Exact-output contract + +Before moving production code, capture current outputs from fixed synthetic captured and self-reported runs: + +```text +tests/fixtures/report-rendering/captured/report.md +tests/fixtures/report-rendering/captured/report.html +tests/fixtures/report-rendering/captured/report-artifact.html +tests/fixtures/report-rendering/self-reported/report.md +tests/fixtures/report-rendering/self-reported/report.html +tests/fixtures/report-rendering/self-reported/report-artifact.html +``` + +`tests/live-report-contract.sh` must compare each generated file byte-for-byte with its fixture. The existing focused wording, ordering, escaping, provenance, and invalid-config assertions remain because they explain what a mismatch means. + +The new JSON contract must check: + +- `report-data.json` exists; +- `schema_version` is `1`; +- both variants and their ordered trials exist; +- captured and self-reported provenance stays explicit; +- `ReportData` survives a `to_dict` / `from_dict` round trip; +- both renderers accept `ReportData` and return strings without writing files. + +The committed fixtures remain synthetic and must not contain customer data, private code, credentials, or real transcripts. + +## Migration sequence + +1. Add byte-for-byte characterization fixtures for both report modes while the current renderer is still authoritative. +2. Add a failing contract for the missing `report-data.json` and typed round trip. +3. Add `schema.py`, `load.py`, and `content.py`. Keep the existing Markdown and HTML blocks in `render.py` until the report data and JSON contract pass. +4. Move Markdown assembly to `render_markdown.py`. Confirm the Markdown fixtures remain exact. +5. Move HTML assembly and static CSS to `render_html.py` and `report.css`. Confirm both HTML fixtures remain exact. +6. Reduce `render.py` to argument handling, orchestration, writes, and existing stdout. +7. Run the complete deterministic suite and inspect both generated formats from the same synthetic run. + +Each move is a clean cutover. Do not leave duplicate renderers, compatibility aliases, or deprecated paths. + +## Failure behavior + +- Keep current required-input errors and config parsing errors. +- Keep rejecting an invalid `trace_source` with the current message. +- Keep malformed optional `decisions.json` as a missing-decision fallback. +- Do not silently invent missing evidence. +- A renderer exception must occur before any of the four output files are replaced during that invocation. +- Do not catch renderer errors only to emit a partial report. + +## Non-goals + +- New report wording. +- New HTML layout, typography, color, or spacing. +- New Markdown section order or syntax. +- A public report-data API. +- A generic template engine or Jinja2 dependency. +- Separate renderer commands. +- Changes to `behavior-diff.sh`, `run-trial.sh`, `decisions.py`, or trial stack behavior. +- Removing `report-artifact.html` without first finding and approving every external consumer. +- Removing graded/demo mode or merging run and capsule directories. + +## Risks and controls + +| Risk | Control | +| --- | --- | +| Moving strings changes whitespace or escaping | Exact captured and self-reported fixtures for all three current files | +| Shared copy changes one format unintentionally | `content.py` owns shared wording; intentional differences are named in renderer tests | +| JSON becomes an accidental public API | Mark it internal in code and docs; include a version instead of compatibility shims | +| HTML source CSS becomes detached from generated output | `render.py` always reads and inlines the bundled `report.css` | +| Self-reported actions leak into command flow | Keep provenance in `ReportData` and preserve the current no-flow contract | +| Optional decisions break the whole report | Preserve the current missing/malformed decisions fallback | +| Partial writes mix old and new reports | Render every string before writing any output file | + +## Acceptance criteria + +- The existing renderer CLI and direct callers require no changes. +- Fixed captured and self-reported fixtures produce byte-for-byte identical `report.md`, `report.html`, and `report-artifact.html` before and after the refactor. +- `report-data.json` is written with `schema_version: 1`. +- HTML and Markdown consume the same typed `ReportData`. +- Shared copy has one owner in `content.py`. +- Markdown structure changes require edits only in `render_markdown.py` and its fixtures/tests. +- HTML structure changes require edits only in `render_html.py` and its fixtures/tests. +- HTML style changes require edits only in `report.css` and HTML fixtures/tests. +- Loading or comparison changes require edits only in `load.py`, `schema.py`, and data-contract tests. +- Generated HTML remains a standalone file. +- Existing provenance, flow, decisions, ordering, escaping, labels, final answers, fallbacks, and stdout remain unchanged. +- The full deterministic repository suite passes without a live model call. From c5eea794484b02900b48f0e2ad1be57a494b32ab Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 11:45:09 +0800 Subject: [PATCH 02/19] docs: plan report rendering restructure Signed-off-by: Kent Huang --- ...port-rendering-structure-implementation.md | 1003 +++++++++++++++++ .../2026-09-04-report-rendering-structure.md | 10 +- 2 files changed, 1008 insertions(+), 5 deletions(-) create mode 100644 plans/2026-09-04-report-rendering-structure-implementation.md diff --git a/plans/2026-09-04-report-rendering-structure-implementation.md b/plans/2026-09-04-report-rendering-structure-implementation.md new file mode 100644 index 0000000..575a189 --- /dev/null +++ b/plans/2026-09-04-report-rendering-structure-implementation.md @@ -0,0 +1,1003 @@ +# Report Rendering Structure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split report data loading, shared copy, Markdown rendering, HTML rendering, CSS, and file output while preserving the three current report files byte-for-byte. + +**Architecture:** Keep `render.py` as the only CLI and writer. Build one typed `ReportData` through `reporting/load.py`, serialize it as internal schema version 1, and pass the same object to pure Markdown and HTML renderers. Keep shared wording in `reporting/content.py`; keep format structure in the renderers; keep CSS in a source file that is inlined into standalone HTML. + +**Tech Stack:** Python 3 standard library, Bash 3.2 contract tests, JSON, HTML/CSS, existing Ruff and deterministic repository checks. + +**Design:** `plans/2026-09-04-report-rendering-structure.md` + +--- + +## File structure + +Create one private package beside the existing scripts: + +```text +plugin/skills/behavior-diff/scripts/ +├── render.py # compatible CLI and only file writer +└── reporting/ + ├── __init__.py # marks the private package + ├── schema.py # ReportData records and JSON conversion + ├── content.py # shared report wording + ├── load.py # persisted evidence -> ReportData + ├── render_markdown.py # ReportData -> Markdown string + ├── render_html.py # ReportData + CSS -> HTML strings + └── report.css # CSS source with one result-color token +``` + +Tests and fixed synthetic output: + +```text +tests/ +├── live-report-contract.sh +├── report-schema-test.py +└── fixtures/report-rendering/ + ├── captured/ + │ ├── report.md + │ ├── report.html + │ └── report-artifact.html + └── self-reported/ + ├── report.md + ├── report.html + └── report-artifact.html +``` + +Do not modify `behavior-diff.sh`, `run-trial.sh`, `decisions.py`, either public skill, the plugin manifests, or README. Their current contracts remain valid. + +--- + +### Task 1: Freeze the current visible reports + +**Files:** +- Modify: `tests/live-report-contract.sh:19-105,328-444` +- Create: `tests/fixtures/report-rendering/captured/report.md` +- Create: `tests/fixtures/report-rendering/captured/report.html` +- Create: `tests/fixtures/report-rendering/captured/report-artifact.html` +- Create: `tests/fixtures/report-rendering/self-reported/report.md` +- Create: `tests/fixtures/report-rendering/self-reported/report.html` +- Create: `tests/fixtures/report-rendering/self-reported/report-artifact.html` + +- [ ] **Step 1: Add an explicit fixture-update mode to the contract test** + +At the top of `tests/live-report-contract.sh`, accept either no argument or exactly `--update-report-fixtures`: + +```bash +if (( $# > 1 )); then + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 +fi + +update_report_fixtures=false +case ${1:-} in + "") ;; + --update-report-fixtures) update_report_fixtures=true ;; + *) + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 + ;; +esac +``` + +Add these helpers after `progress()`: + +```bash +fixture_root=$here/fixtures/report-rendering + +copy_report_fixtures() { + local run=$1 + local kind=$2 + local out=$fixture_root/$kind + mkdir -p "$out" + cp "$run/report.md" "$out/report.md" + cp "$run/report.html" "$out/report.html" + cp "$run/report-artifact.html" "$out/report-artifact.html" +} + +require_exact_report() { + local actual=$1 + local expected=$2 + local message=$3 + if ! cmp -s "$expected" "$actual"; then + diff -u "$expected" "$actual" >&2 || true + fail "$message" + fi +} +``` + +After the current renderer calls, update only when the explicit flag is present, then compare all three files for both modes: + +```bash +if $update_report_fixtures; then + copy_report_fixtures "$captured_run" captured + copy_report_fixtures "$self_run" self-reported +fi + +for kind in captured self-reported; do + case $kind in + captured) actual_run=$captured_run ;; + self-reported) actual_run=$self_run ;; + esac + for report_name in report.md report.html report-artifact.html; do + require_exact_report \ + "$actual_run/$report_name" \ + "$fixture_root/$kind/$report_name" \ + "$kind $report_name changed" + done +done +``` + +Do not remove the existing focused assertions. They explain whether a mismatch concerns provenance, ordering, escaping, labels, decisions, flow, or answers. + +- [ ] **Step 2: Generate the goldens from the current renderer** + +Run: + +```bash +bash tests/live-report-contract.sh --update-report-fixtures +``` + +Expected: `ok — live report contract passed`, with six new synthetic fixture files. Review every fixture for invented content only: `Live contract`, `Original project instructions`, `Updated project instructions`, `Before answer`, `After answer`, and the synthetic decision row. + +- [ ] **Step 3: Verify normal mode cannot rewrite fixtures** + +Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ok — live report contract passed` and no fixture changes. + +- [ ] **Step 4: Commit the characterization boundary** + +```bash +git add tests/live-report-contract.sh tests/fixtures/report-rendering +git commit --signoff -m "test: freeze report rendering output" +``` + +--- + +### Task 2: Define the typed report schema + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/__init__.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/schema.py` +- Create: `tests/report-schema-test.py` +- Modify: `tests/live-report-contract.sh` + +- [ ] **Step 1: Write the failing schema test** + +Create `tests/report-schema-test.py`. It must import the private package from the shipped scripts directory, construct a complete synthetic schema-v1 dictionary, load it with `ReportData.from_dict`, and compare the deterministic round trip: + +```python +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +scripts = Path(__file__).resolve().parents[1] / "plugin/skills/behavior-diff/scripts" +sys.path.insert(0, str(scripts)) + +from reporting.schema import ReportData # noqa: E402 + + +def main(): + source = Path(sys.argv[1]) if len(sys.argv) == 2 else None + if source: + raw = json.loads(source.read_text()) + else: + raw = { + "schema_version": 1, + "metadata": { + "model": "synthetic/model", + "mode": "review", + "vocab": "generic", + "trace_source": "captured", + "target_file": "AGENTS.md", + "before_label": "current file", + "after_label": "your change applied", + }, + "content": { + "title": "Synthetic report", + "subtitle": "Synthetic subtitle.", + "observation": "", + "scenario_heading": "Scenario", + "scenario": "Compare two synthetic files.", + "expected_heading": "Expected behavior", + "expected": None, + "diff_heading": "Diff of AGENTS.md — the only difference between the variants", + "decision_heading": "Decision diff — top divergences", + "decision_blurb": "Synthetic decision explanation.", + "flow_heading": "Flow diff — where the variants diverge", + "result_heading": "Result", + "boundary": "Synthetic evidence only.", + }, + "rule_diff": "--- before\n+++ after\n", + "result": {"text": "No automatic verdict", "kind": "neutral"}, + "variants": { + "before": { + "label": "Before", + "note": "current file", + "passed": 0, + "blocked": 0, + "valid": 1, + "total": 1, + "count_text": "1 valid trial(s)", + "count_emphasized": False, + "trials": [{ + "name": "before-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md"], + "final": "Before answer", + "outcome": None, + }], + }, + "after": { + "label": "After", + "note": "your change applied", + "passed": 0, + "blocked": 0, + "valid": 1, + "total": 1, + "count_text": "1 valid trial(s)", + "count_emphasized": False, + "trials": [{ + "name": "after-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md", "run tests"], + "final": "After answer", + "outcome": None, + }], + }, + }, + "command_flow": { + "enabled": True, + "same": False, + "shared": ["Read files"], + "before": {"prefix": [], "paths": [], "total": 1}, + "after": { + "prefix": ["Run tests"], + "paths": [], + "total": 1, + }, + }, + "decisions": { + "rows": [], + "fork": None, + "fork_note": "", + "dropped": 0, + "extractor": "", + "before_count": 1, + "after_count": 1, + }, + } + + report = ReportData.from_dict(raw) + assert report.schema_version == 1 + assert report.to_dict() == raw + encoded = report.to_json() + assert encoded == json.dumps(raw, indent=2, sort_keys=True) + "\n" + + bad = dict(raw, schema_version=2) + try: + ReportData.from_dict(bad) + except ValueError as error: + assert str(error) == "unsupported report-data schema version: 2" + else: + raise AssertionError("schema version 2 was accepted") + + +if __name__ == "__main__": + main() +``` + +Call it from `tests/live-report-contract.sh` before the fixture runs: + +```bash +python3 "$here/report-schema-test.py" +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: nonzero with `ModuleNotFoundError: No module named 'reporting'`. + +- [ ] **Step 3: Implement the immutable schema records** + +Create an empty `reporting/__init__.py` and implement frozen dataclasses in `reporting/schema.py`: + +```python +from typing import Dict, Optional, Tuple, Union + +SCHEMA_VERSION = 1 + +@dataclass(frozen=True) +class TrialData: + name: str + verdict: str + actions: str + commands: Tuple[str, ...] + final: str + outcome: Optional[str] + +@dataclass(frozen=True) +class VariantData: + label: str + note: str + passed: int + blocked: int + valid: int + total: int + count_text: str + count_emphasized: bool + trials: Tuple[TrialData, ...] + +@dataclass(frozen=True) +class VariantsData: + before: VariantData + after: VariantData + +@dataclass(frozen=True) +class FlowPathData: + steps: Tuple[str, ...] + count: int + +@dataclass(frozen=True) +class FlowBranchData: + prefix: Tuple[str, ...] + paths: Tuple[FlowPathData, ...] + total: int + +@dataclass(frozen=True) +class CommandFlowData: + enabled: bool + same: bool + shared: Tuple[str, ...] + before: FlowBranchData + after: FlowBranchData + +@dataclass(frozen=True) +class DecisionChoiceData: + choice: str + count: int + +@dataclass(frozen=True) +class DecisionRowData: + decision: str + topic: str + anchor: Union[int, str] + diverges: bool + note: str + before: Tuple[DecisionChoiceData, ...] + after: Tuple[DecisionChoiceData, ...] + +@dataclass(frozen=True) +class DecisionData: + rows: Tuple[DecisionRowData, ...] + fork: Optional[int] + fork_note: str + dropped: int + extractor: str + before_count: int + after_count: int + +@dataclass(frozen=True) +class MetadataData: + model: str + mode: str + vocab: str + trace_source: str + target_file: str + before_label: str + after_label: str + + +@dataclass(frozen=True) +class ContentData: + title: str + subtitle: str + observation: str + scenario_heading: str + scenario: str + expected_heading: str + expected: Optional[str] + diff_heading: str + decision_heading: str + decision_blurb: str + flow_heading: str + result_heading: str + boundary: str + +@dataclass(frozen=True) +class ResultData: + text: str + kind: str + +@dataclass(frozen=True) +class ReportData: + schema_version: int + metadata: MetadataData + content: ContentData + rule_diff: str + result: ResultData + variants: VariantsData + command_flow: CommandFlowData + decisions: DecisionData +``` + +Implement `ReportData.from_dict`, `to_dict`, and `to_json`. Rebuild every nested dataclass explicitly in `from_dict`; do not store unvalidated nested dictionaries. Use `dataclasses.asdict`, then recursively convert tuples to lists so `to_dict` contains only JSON-shaped values; do not serialize and parse JSON to perform that conversion. Convert the `DecisionChoiceData.count` field from the persisted `decisions.json` key `n` while loading; `report-data.json` itself uses `count` consistently. + +`to_json` must be exactly: + +```python +def to_json(self): + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" +``` + +- [ ] **Step 4: Run the focused tests and verify GREEN** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: both exit `0`; the six visible output fixtures remain exact. + +- [ ] **Step 5: Commit the schema** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/__init__.py \ + plugin/skills/behavior-diff/scripts/reporting/schema.py \ + tests/report-schema-test.py tests/live-report-contract.sh +git commit --signoff -m "feat: define internal report data schema" +``` + +--- + +### Task 3: Extract report loading and shared copy + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/content.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/load.py` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:16-448,918-923` +- Modify: `tests/live-report-contract.sh` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add failing generated-data assertions** + +After each current renderer invocation in `tests/live-report-contract.sh`, require the new artifact and validate it through the shipped schema: + +```bash +for report_run in "$captured_run" "$self_run"; do + [[ -f $report_run/report-data.json ]] || + fail "renderer did not write report-data.json: $report_run" + python3 "$here/report-schema-test.py" "$report_run/report-data.json" +done +``` + +Add focused JSON checks: + +```bash +[[ $(jq -r '.schema_version' "$captured_run/report-data.json") == 1 ]] || + fail 'captured report-data schema version changed' +[[ $(jq -r '.metadata.trace_source' "$captured_run/report-data.json") == captured ]] || + fail 'captured report-data lost its provenance' +[[ $(jq -r '.metadata.trace_source' "$self_run/report-data.json") == self-reported ]] || + fail 'self-reported report-data lost its provenance' +[[ $(jq -r '.variants.after.trials[0].commands[1]' \ + "$captured_run/report-data.json") == \ + 'Test: bash behavior-diff/tests/live-report-contract.sh' ]] || + fail 'report-data changed trial command order' +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `FAIL: renderer did not write report-data.json`. + +- [ ] **Step 3: Move shared copy into `content.py`** + +Move the current default subtitle, boundary, result matrix, count wording, decision explanation, observation wording, and section headings behind these pure functions: + +- `result_data(mode: str, self_reported: bool, before: VariantData, after: VariantData) -> ResultData` copies the current result matrix and single-run suffix from `render.py:250-271`. +- `count_data(mode: str, passed: int, valid: int, blocked: int) -> Tuple[str, bool]` returns plain count text plus whether Markdown emphasizes it; HTML must not remove Markdown markers. +- `decision_blurb(self_reported: bool, single_trial: bool) -> str` copies `DEC_BLURB` and `DEC_N1` from `render.py:315-364`. +- `observation(decisions: DecisionData) -> str` copies the fork observation from `render.py:438-447`. +- `build_content(config: Dict[str, object], target_file: str, scenario: str, result: ResultData, observation_text: str, decision_text: str) -> ContentData` applies current defaults and builds the shared headings and boundary. + +The returned strings must be copied exactly from the current `render.py`. Do not edit punctuation, capitalization, spacing, or warnings. + +- [ ] **Step 4: Move evidence loading and comparison into `load.py`** + +Expose one public function: `load_report(run: Path, capsule: Path, model: str, config_path: Optional[Path]) -> ReportData`. + +Move the current logic into private functions with explicit arguments: + +- `_read_config(config_path)` reads JSON and applies the existing defaults. +- `_read_grades(run)` parses the required TSV rows without changing its strict split behavior. +- `_read_trial(run, name, grade)` reads ordered tools and the final result from canonical JSONL. +- `_classify(command, vocab)` contains the current demo, generic, and Spacedock buckets. +- `_trial_sequence(trial, mode, vocab)` adds the current optional outcome step. +- `_common_prefix(sequences)` and `_build_flow(before_trials, after_trials, mode, trace_source, vocab)` preserve the current flow algorithm and counts. +- `_read_rule_diff(run, capsule, target_file)` keeps target-file and `rule.md` fallback behavior. +- `_read_decisions(run, before_total, after_total, self_reported)` converts every valid decision choice from persisted key `n` to `DecisionChoiceData.count`. + +Preserve these exact behaviors: + +- invalid `trace_source` exits with `trace_source must be either "captured" or "self-reported"`; +- malformed trace lines are skipped; +- the last non-empty result remains the final answer; +- missing or malformed `decisions.json` produces an empty `DecisionData`; +- trial names retain lexical order; +- self-reported runs disable command flow; +- all current vocab classifiers and result outcomes remain; +- run and capsule may be different paths. + +Construct every `ReportData` field only after all required inputs and derived facts are available. Set `schema_version=SCHEMA_VERSION`; do not use a partial constructor or default missing evidence. + +- [ ] **Step 5: Use `ReportData` in `render.py` without moving format blocks yet** + +At the top-level command path: + +```python +report = load_report(run, capsule, model, config_path) +``` + +Temporarily unpack the exact fields needed by the existing Markdown and HTML blocks. This bridge is private to `render.py` and must be deleted in Task 6. Do not add compatibility methods or aliases to `ReportData`. + +After both current format strings are successfully built, add `report-data.json` to the write phase: + +```python +report_json = report.to_json() +# Build Markdown and both HTML strings first. +(run / "report-data.json").write_text(report_json) +``` + +Do not change the three existing report strings or stdout. + +- [ ] **Step 6: Verify the data boundary and exact visible output** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: `report-data.json` checks pass, and all six visible fixtures remain byte-for-byte exact. + +- [ ] **Step 7: Commit loading and shared content** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/content.py \ + plugin/skills/behavior-diff/scripts/reporting/load.py \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py tests/live-report-contract.sh +git commit --signoff -m "refactor: separate report data assembly" +``` + +--- + +### Task 4: Extract the Markdown renderer + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/render_markdown.py` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:449-542` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add a failing pure-renderer check** + +Extend `tests/report-schema-test.py` when a `report-data.json` path is supplied: + +```python +from reporting.render_markdown import render_markdown + +expected_markdown = source.with_name("report.md").read_text() +assert render_markdown(report) == expected_markdown +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ModuleNotFoundError: No module named 'reporting.render_markdown'`. + +- [ ] **Step 3: Move Markdown assembly behind one pure function** + +Create: + +```python +def render_markdown(report: ReportData) -> str: + parts = [] + # Existing Markdown assembly, with ReportData attributes instead of globals. + return "\n".join(parts) +``` + +Move all Markdown-only behavior into this module: + +- Markdown headings and section order; +- diff fence; +- decision lists; +- `md_branch` flow formatting; +- conditional flow `
`; +- trial `
` blocks; +- no `$ ` prefix for captured commands. + +Keep the current strings and blank lines exactly. The function must not access the filesystem or mutate `ReportData`. + +Replace the old Markdown block in `render.py` with: + +```python +markdown = render_markdown(report) +``` + +Do not write it until all remaining format strings have been built. + +- [ ] **Step 4: Verify the pure function and golden output** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: the pure function equals each generated `report.md`, and both Markdown goldens remain exact. + +- [ ] **Step 5: Commit the Markdown renderer** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/render_markdown.py \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py +git commit --signoff -m "refactor: isolate Markdown report rendering" +``` + +--- + +### Task 5: Extract the HTML renderer and CSS + +**Files:** +- Create: `plugin/skills/behavior-diff/scripts/reporting/render_html.py` +- Create: `plugin/skills/behavior-diff/scripts/reporting/report.css` +- Modify: `plugin/skills/behavior-diff/scripts/render.py:544-916` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add failing pure HTML checks** + +Extend the path-backed branch in `tests/report-schema-test.py`: + +```python +from reporting.render_html import render_artifact, render_document + +css_path = scripts / "reporting/report.css" +css = css_path.read_text() +artifact = render_artifact(report, css) +assert artifact == source.with_name("report-artifact.html").read_text() +assert render_document(artifact) == source.with_name("report.html").read_text() +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +bash tests/live-report-contract.sh +``` + +Expected: `ModuleNotFoundError: No module named 'reporting.render_html'`. + +- [ ] **Step 3: Extract the CSS source with one explicit result-color token** + +Copy the generated CSS text between the current `` into `reporting/report.css`. The file starts with `:root {`, uses normal single CSS braces, and ends with one newline after the final `}`. + +Replace only the current dynamic result background value with: + +```css +.result { background:__RESULT_BG__; color:#fff; border-radius:8px; +``` + +Keep every other declaration, space, and line break unchanged. Keep the Google Fonts `` in the HTML renderer. + +- [ ] **Step 4: Move HTML assembly behind pure functions** + +Expose `render_artifact(report: ReportData, css: str) -> str` and `render_document(artifact: str) -> str`. + +Resolve the one CSS token inside the HTML renderer: + +```python +RESULT_BACKGROUNDS = { + "good": "var(--pass)", + "bad": "var(--fail)", + "neutral": "var(--accent)", +} + + +def _resolve_css(css, result_kind): + if css.count("__RESULT_BG__") != 1: + raise ValueError("report.css must contain __RESULT_BG__ exactly once") + return css.replace("__RESULT_BG__", RESULT_BACKGROUNDS[result_kind]) + + +def render_document(artifact): + return ( + '' + '' + "" + artifact + "" + ) +``` + +`render_artifact` must call `_resolve_css` and insert the result as `\n`. This reproduces the current generated bytes while keeping the source CSS editable. + +Move `card`, `lane`, `branch_html`, `dec_choices`, `dec_label`, decision HTML, flow HTML, trial columns, escaping, footer, and body assembly into `render_html.py`. Keep captured `$ ` prefixes, self-reported labels, open review-mode answers, HTML escaping, section order, and all source whitespace exact. + +The functions must not read or write files. Pass CSS explicitly. + +Replace the old HTML block in `render.py` with: + +```python +css = (Path(__file__).parent / "reporting/report.css").read_text() +artifact_html = render_artifact(report, css) +html_document = render_document(artifact_html) +``` + +- [ ] **Step 5: Verify both HTML outputs exactly** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +``` + +Expected: pure renderer strings equal the generated files; captured and self-reported HTML and artifact goldens remain exact. + +- [ ] **Step 6: Commit the HTML renderer and style source** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/reporting/render_html.py \ + plugin/skills/behavior-diff/scripts/reporting/report.css \ + plugin/skills/behavior-diff/scripts/render.py \ + tests/report-schema-test.py +git commit --signoff -m "refactor: isolate HTML report rendering" +``` + +--- + +### Task 6: Reduce `render.py` to orchestration and writes + +**Files:** +- Modify: `plugin/skills/behavior-diff/scripts/render.py` +- Modify: `.github/workflows/ci.yml:76-81` +- Modify: `CODING_GUIDELINES.md:155-157` +- Modify: `tests/report-schema-test.py` + +- [ ] **Step 1: Add a failing import-safety check** + +In `tests/report-schema-test.py`, load `render.py` without command arguments inside a fresh temporary directory: + +```python +import importlib.util +import io +import os +from contextlib import redirect_stderr, redirect_stdout +from tempfile import TemporaryDirectory + +with TemporaryDirectory() as tmp: + stdout = io.StringIO() + stderr = io.StringIO() + old_cwd = os.getcwd() + try: + os.chdir(tmp) + spec = importlib.util.spec_from_file_location( + "behavior_diff_render", scripts / "render.py" + ) + module = importlib.util.module_from_spec(spec) + with redirect_stdout(stdout), redirect_stderr(stderr): + spec.loader.exec_module(module) + finally: + os.chdir(old_cwd) + assert list(Path(tmp).iterdir()) == [] + assert stdout.getvalue() == "" + assert stderr.getvalue() == "" + assert callable(module.main) +``` + +- [ ] **Step 2: Run the test and verify RED** + +```bash +python3 tests/report-schema-test.py +``` + +Expected: the current top-level renderer exits or reads `sys.argv`; `main` is not available. + +- [ ] **Step 3: Implement the thin import-safe facade** + +`render.py` must contain only imports, argument parsing, orchestration, writes, stdout, and `main()`: + +```python +def main(argv=None): + args = sys.argv[1:] if argv is None else argv + run = Path(args[0]).resolve() + capsule = Path(args[1]).resolve() + model = args[2] + config_path = Path(args[3]) if len(args) > 3 else None + + report = load_report(run, capsule, model, config_path) + report_json = report.to_json() + markdown = render_markdown(report) + css = (Path(__file__).parent / "reporting/report.css").read_text() + artifact_html = render_artifact(report, css) + html_document = render_document(artifact_html) + + (run / "report-data.json").write_text(report_json) + (run / "report.md").write_text(markdown) + (run / "report-artifact.html").write_text(artifact_html) + (run / "report.html").write_text(html_document) + + before = report.variants.before + after = report.variants.after + print( + f"mode {report.metadata.mode} · BEFORE pass {before.passed}/{before.valid} · " + f"AFTER pass {after.passed}/{after.valid} → {report.result.text}" + ) + print(f"report: {run / 'report.md'}") + print(f"page: {run / 'report.html'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +Keep the current natural Python errors for missing positional arguments and unreadable required files. Do not add a new CLI parser or change diagnostics in this refactor. + +Delete all temporary field-unpacking code and every moved helper from `render.py`. There must be one implementation of each loader, copy rule, Markdown section, HTML component, and CSS declaration. + +- [ ] **Step 4: Expand deterministic Python compilation** + +In `.github/workflows/ci.yml` and `CODING_GUIDELINES.md`, keep the existing files and add: + +```text +plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +The CI command becomes: + +```bash +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +- [ ] **Step 5: Verify the facade and focused report contracts** + +```bash +python3 tests/report-schema-test.py +bash tests/live-report-contract.sh +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +Expected: all commands exit `0`; all visible output remains exact; import produces no files or stdout. + +- [ ] **Step 6: Commit the final cutover** + +```bash +git add \ + plugin/skills/behavior-diff/scripts/render.py \ + .github/workflows/ci.yml CODING_GUIDELINES.md \ + tests/report-schema-test.py +git commit --signoff -m "refactor: make report rendering import safe" +``` + +--- + +### Task 7: Run complete verification and inspect the real surfaces + +**Files:** +- Verify only; no planned production changes. + +- [ ] **Step 1: Run formatting checks** + +```bash +docker run --rm -v "$PWD:/mnt" -w /mnt \ + mvdan/shfmt:v3.14.0 -d -i 2 -ci . +uvx ruff@0.16.5 format --check --diff . +``` + +Expected: no Bash diff; Ruff reports every Python file already formatted. + +- [ ] **Step 2: Run syntax and static checks** + +```bash +bash -n \ + .github/scripts/*.sh \ + bin/behavior-diff \ + plugin/scripts/*.sh \ + plugin/skills/behavior-diff/scripts/*.sh \ + tests/*.sh +shellcheck \ + .github/scripts/*.sh \ + bin/behavior-diff \ + plugin/scripts/*.sh \ + plugin/skills/behavior-diff/scripts/*.sh \ + tests/*.sh +python3 -m py_compile \ + plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/render.py \ + plugin/skills/behavior-diff/scripts/reporting/*.py +``` + +Expected: every command exits `0` with no diagnostics. + +- [ ] **Step 3: Run the complete deterministic suite** + +```bash +bash tests/hooks-test.sh +python3 plugin/skills/behavior-diff/scripts/decisions.py --check +bash tests/live-report-contract.sh +bash tests/release-workflow-test.sh +git diff --check +``` + +Expected: all four repository checks pass and `git diff --check` has no output. + +- [ ] **Step 4: Verify exact fixture stability independently** + +Render the fixed captured and self-reported inputs through `tests/live-report-contract.sh`, then confirm the six comparisons ran without fixture-update mode. Run: + +```bash +bash tests/live-report-contract.sh +``` + +Expected: the test passes without changing any file under `tests/fixtures/report-rendering/`. + +- [ ] **Step 5: Inspect both output formats from one synthetic run** + +Open `tests/fixtures/report-rendering/captured/report.html` in the browser and verify: + +```text +title and subtitle render +scenario and instruction diff render +decision diff and command-derived flow render +before and after columns render +commands and final answers render in order +result and footer render +page has the same current typography, colors, spacing, and responsive layout +``` + +Read `tests/fixtures/report-rendering/captured/report.md` and verify the same evidence appears in the current section order. This is a visual/behavioral check, not a live model call. + +- [ ] **Step 6: Confirm repository scope** + +Verify: + +```text +render.py is a thin facade +reporting modules have one responsibility each +report-data.json is marked internal and versioned +no duplicate old renderer blocks remain +behavior-diff.sh and both skills are unchanged +no real run, transcript, report, credential, or customer data is tracked +all commits contain DCO sign-off +``` + +- [ ] **Step 7: Request independent review before a PR** + +Give one read-only reviewer the complete branch diff and require `REVIEWER_GUIDELINES.md`. Ask it to check exact-output evidence, schema boundaries, import safety, error/fallback preservation, CSS inlining, self-reported provenance, duplicate code, and unauthorized scope. Do not create or update a PR without an `APPROVE` verdict. diff --git a/plans/2026-09-04-report-rendering-structure.md b/plans/2026-09-04-report-rendering-structure.md index a4534a0..e27e270 100644 --- a/plans/2026-09-04-report-rendering-structure.md +++ b/plans/2026-09-04-report-rendering-structure.md @@ -120,10 +120,10 @@ Own shared human-readable copy: - result text; - count text; - observation text; -- decision and flow explanations; +- the decision explanation; - the simulation boundary. -HTML and Markdown use this copy unless the current outputs intentionally differ. Existing intentional differences remain renderer-owned. Examples include HTML's captured-command `$ ` prefix and the formats' different heading markup. +HTML and Markdown use this copy unless the current outputs intentionally differ. Existing intentional differences remain renderer-owned. These include HTML's captured-command `$ ` prefix, the formats' different heading markup, and their currently different flow explanations. ### `reporting/render_markdown.py` @@ -165,7 +165,7 @@ It does not read run files or write output files. ### `reporting/report.css` -Contain the current static CSS exactly as it appears in generated HTML. `render.py` reads this source and passes it to the HTML renderer. The renderer inlines it, so generated HTML stays portable. +Contain the current CSS in generated form, with one `__RESULT_BG__` token where the result color varies by `result.kind`. `render.py` reads this source and passes it to the HTML renderer. The renderer must find the token exactly once, replace it with the current allowed CSS variable, and inline the result. Generated HTML stays byte-for-byte compatible and portable. ### `render.py` @@ -243,7 +243,7 @@ The committed fixtures remain synthetic and must not contain customer data, priv 2. Add a failing contract for the missing `report-data.json` and typed round trip. 3. Add `schema.py`, `load.py`, and `content.py`. Keep the existing Markdown and HTML blocks in `render.py` until the report data and JSON contract pass. 4. Move Markdown assembly to `render_markdown.py`. Confirm the Markdown fixtures remain exact. -5. Move HTML assembly and static CSS to `render_html.py` and `report.css`. Confirm both HTML fixtures remain exact. +5. Move HTML assembly and the CSS source to `render_html.py` and `report.css`. Confirm both HTML fixtures remain exact. 6. Reduce `render.py` to argument handling, orchestration, writes, and existing stdout. 7. Run the complete deterministic suite and inspect both generated formats from the same synthetic run. @@ -277,7 +277,7 @@ Each move is a clean cutover. Do not leave duplicate renderers, compatibility al | Moving strings changes whitespace or escaping | Exact captured and self-reported fixtures for all three current files | | Shared copy changes one format unintentionally | `content.py` owns shared wording; intentional differences are named in renderer tests | | JSON becomes an accidental public API | Mark it internal in code and docs; include a version instead of compatibility shims | -| HTML source CSS becomes detached from generated output | `render.py` always reads and inlines the bundled `report.css` | +| HTML source CSS becomes detached from generated output | `render_html.py` requires one result-color token; `render.py` always reads and inlines the bundled `report.css` | | Self-reported actions leak into command flow | Keep provenance in `ReportData` and preserve the current no-flow contract | | Optional decisions break the whole report | Preserve the current missing/malformed decisions fallback | | Partial writes mix old and new reports | Render every string before writing any output file | From 8b9709a65b4c775502932336c0f848ab7440699c Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:16:49 +0800 Subject: [PATCH 03/19] test: freeze report rendering output Signed-off-by: Kent Huang --- .../captured/report-artifact.html | 140 +++++++++++++++++ .../report-rendering/captured/report.html | 141 ++++++++++++++++++ .../report-rendering/captured/report.md | 98 ++++++++++++ .../self-reported/report-artifact.html | 140 +++++++++++++++++ .../self-reported/report.html | 141 ++++++++++++++++++ .../report-rendering/self-reported/report.md | 82 ++++++++++ tests/live-report-contract.sh | 47 ++++++ 7 files changed, 789 insertions(+) create mode 100644 tests/fixtures/report-rendering/captured/report-artifact.html create mode 100644 tests/fixtures/report-rendering/captured/report.html create mode 100644 tests/fixtures/report-rendering/captured/report.md create mode 100644 tests/fixtures/report-rendering/self-reported/report-artifact.html create mode 100644 tests/fixtures/report-rendering/self-reported/report.html create mode 100644 tests/fixtures/report-rendering/self-reported/report.md diff --git a/tests/fixtures/report-rendering/captured/report-artifact.html b/tests/fixtures/report-rendering/captured/report-artifact.html new file mode 100644 index 0000000..f007318 --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report-artifact.html @@ -0,0 +1,140 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ + +
Compare the two instruction snapshots.
+ + + +
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ +
Flow diff — command-derived (deterministic, no model involved)

Steps are described from the agents' actual commands. A path is a sequence at least one trial literally took — arrows connect steps inside a path, and a split shows where trials went different ways. Full commands are in the trial cards below.

paths diverge here

BEFORE

all 1 trials

(same steps as the shared flow)

AFTER

all 1 trials

Run tests
+ + +

Before

current file

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

Commands the agent ran (1)
$ Read: AGENTS.md
Final answer to the user
Before answer

After

your change applied

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

Commands the agent ran (2)
$ Read: AGENTS.md
+
+$ Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ + +
No automatic verdict — compare the flows and final answers
+ + diff --git a/tests/fixtures/report-rendering/captured/report.html b/tests/fixtures/report-rendering/captured/report.html new file mode 100644 index 0000000..f9ff39b --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report.html @@ -0,0 +1,141 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ + +
Compare the two instruction snapshots.
+ + + +
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ +
Flow diff — command-derived (deterministic, no model involved)

Steps are described from the agents' actual commands. A path is a sequence at least one trial literally took — arrows connect steps inside a path, and a split shows where trials went different ways. Full commands are in the trial cards below.

paths diverge here

BEFORE

all 1 trials

(same steps as the shared flow)

AFTER

all 1 trials

Run tests
+ + +

Before

current file

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

Commands the agent ran (1)
$ Read: AGENTS.md
Final answer to the user
Before answer

After

your change applied

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

Commands the agent ran (2)
$ Read: AGENTS.md
+
+$ Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ + +
No automatic verdict — compare the flows and final answers
+ + + \ No newline at end of file diff --git a/tests/fixtures/report-rendering/captured/report.md b/tests/fixtures/report-rendering/captured/report.md new file mode 100644 index 0000000..da160fd --- /dev/null +++ b/tests/fixtures/report-rendering/captured/report.md @@ -0,0 +1,98 @@ +# Live contract + +Synthetic contract fixture. + +**Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.** + +Model: contract · 1 trial(s) per variant. + +## Scenario + +Compare the two instruction snapshots. + +## Diff of AGENTS.md — the only difference between the variants + +```diff +--- AGENTS.md (before) ++++ AGENTS.md (after) +@@ -1 +1 @@ +-Original project instructions. ++Updated project instructions. +``` + +## Decision diff — top divergences + +A decision is a point where the agent had a real choice. These are recovered from what the trials did and said, not from the instruction diff, and some of them leave no command behind. Order is real: decisions visible in commands come in command order, and decisions visible only in the final answer come last. The fork and main divergences are stable across extractions; minor rows can vary run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it. + +Diverging from here: + +- 1. **Evidence choice** — Which evidence was used? ⟵ root behavior change + - BEFORE: read only + - AFTER: read and test + +One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). + +Synthetic fixture. + +
Flow diff — command-derived (deterministic, no model involved) + +## Flow diff — where the variants diverge + +Steps are described from the agents' actual commands; a path is a sequence at least one trial literally took. Full commands are in the trial sections below. + +Shared flow (every trial, both variants): + + +Divergence: + +- BEFORE, all 1 trials → (same steps as the shared flow) +- AFTER, all 1 trials → Run tests + +
+ +## BEFORE — current file + +1 valid trial(s) · no automatic grading (blocked: 0) + +### before-1 — REVIEW + +
commands the agent ran (1) + +``` +Read: AGENTS.md +``` +
+ +
final answer to the user + +Before answer + +
+ +## AFTER — your change applied + +1 valid trial(s) · no automatic grading (blocked: 0) + +### after-1 — REVIEW + +
commands the agent ran (2) + +``` +Read: AGENTS.md + +Test: bash behavior-diff/tests/live-report-contract.sh +``` +
+ +
final answer to the user + +After answer + +
+ +## Result + +**No automatic verdict — compare the flows and final answers** + +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks. diff --git a/tests/fixtures/report-rendering/self-reported/report-artifact.html b/tests/fixtures/report-rendering/self-reported/report-artifact.html new file mode 100644 index 0000000..5ce7671 --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report-artifact.html @@ -0,0 +1,140 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ + +
Compare the two instruction snapshots.
+ + + +
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ + + + +

Before

parent snapshot <baseline>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

self-reported actions (1)
Read: AGENTS.md
Final answer to the user
Before answer

After

target snapshot <candidate>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

self-reported actions (2)
Read: AGENTS.md
+
+Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ + +
No automatic verdict — compare the reported actions, decision diff, and final answers
+ + diff --git a/tests/fixtures/report-rendering/self-reported/report.html b/tests/fixtures/report-rendering/self-reported/report.html new file mode 100644 index 0000000..73f6a40 --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report.html @@ -0,0 +1,141 @@ +Live contract + + + +

Live contract

+

Synthetic contract fixture.

+

Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.

+ + +
Compare the two instruction snapshots.
+ + + +
--- AGENTS.md (before)
++++ AGENTS.md (after)
+@@ -1 +1 @@
+-Original project instructions.
++Updated project instructions.
+
+ +

A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it.

paths diverge here

BEFORE

AFTER

1 · Evidence choiceroot changeduring the workWhich evidence was used?

read only
read and test

One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). Synthetic fixture.

+ + + + +

Before

parent snapshot <baseline>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWbefore-1

self-reported actions (1)
Read: AGENTS.md
Final answer to the user
Before answer

After

target snapshot <candidate>

1 valid trial(s) · no automatic grading (blocked: 0)

REVIEWafter-1

self-reported actions (2)
Read: AGENTS.md
+
+Test: bash behavior-diff/tests/live-report-contract.sh
Final answer to the user
After answer
+ + +
No automatic verdict — compare the reported actions, decision diff, and final answers
+ + + \ No newline at end of file diff --git a/tests/fixtures/report-rendering/self-reported/report.md b/tests/fixtures/report-rendering/self-reported/report.md new file mode 100644 index 0000000..659c27d --- /dev/null +++ b/tests/fixtures/report-rendering/self-reported/report.md @@ -0,0 +1,82 @@ +# Live contract + +Synthetic contract fixture. + +**Observed in this run — Evidence choice: BEFORE read only · AFTER read and test. Single-run observation, not a verdict.** + +Model: contract · 1 trial(s) per variant. + +## Scenario + +Compare the two instruction snapshots. + +## Diff of AGENTS.md — the only difference between the variants + +```diff +--- AGENTS.md (before) ++++ AGENTS.md (after) +@@ -1 +1 @@ +-Original project instructions. ++Updated project instructions. +``` + +## Decision diff — top divergences + +A decision is a point where the agent had a real choice. The decisions come from self-reported actions and final answers. Some decisions leave no reported action behind. Order follows the report: decisions visible in actions come in reported action order, and decisions visible only in the final answer come last. Extractor output can vary from run to run. CAUTION — one trial per side: any divergence here can be run-to-run variation rather than a rule effect; confirm with repeated trials (behavior-diff 3+3) before acting on it. + +Diverging from here: + +- 1. **Evidence choice** — Which evidence was used? ⟵ root behavior change + - BEFORE: read only + - AFTER: read and test + +One target decision changed (#1); 0 later differences diverge downstream of it (the extractor's causal reading, not a measured chain). + +Synthetic fixture. + +## BEFORE — parent snapshot + +1 valid trial(s) · no automatic grading (blocked: 0) + +### before-1 — REVIEW + +
self-reported actions (1) + +``` +Read: AGENTS.md +``` +
+ +
final answer to the user + +Before answer + +
+ +## AFTER — target snapshot + +1 valid trial(s) · no automatic grading (blocked: 0) + +### after-1 — REVIEW + +
self-reported actions (2) + +``` +Read: AGENTS.md + +Test: bash behavior-diff/tests/live-report-contract.sh +``` +
+ +
final answer to the user + +After answer + +
+ +## Result + +**No automatic verdict — compare the reported actions, decision diff, and final answers** + +This is simulation evidence. Real-use evidence is still pending. +It does not repair the original incident; it tests the change for future tasks. diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index c51c5f2..c8a113c 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -16,6 +16,8 @@ claude_manifest=$here/../plugin/.claude-plugin/plugin.json codex_manifest=$here/../plugin/.codex-plugin/plugin.json readme=$here/../README.md +fixture_root=$here/fixtures/report-rendering +update_report_fixtures=false require_output() { grep -qF -- "$1" "$2" || fail "$3" } @@ -103,6 +105,42 @@ progress() { printf '[report] %s\n' "$1" } + +copy_report_fixtures() { + local mode=$1 + local run=$2 + local fixture_dir=$fixture_root/$mode + + mkdir -p "$fixture_dir" + cp "$run/report.md" "$fixture_dir/report.md" + cp "$run/report.html" "$fixture_dir/report.html" + cp "$run/report-artifact.html" "$fixture_dir/report-artifact.html" +} + +require_exact_report() { + local mode=$1 + local report=$2 + local fixture=$fixture_root/$mode/$(basename "$report") + + if ! cmp -s "$fixture" "$report"; then + printf 'Rendered report differs from fixture: %s\n' "$fixture" >&2 + if diff -u "$fixture" "$report" >&2; then + fail "rendered report comparison failed unexpectedly: $report" + else + fail "rendered report differs from fixture: $report" + fi + fi +} + +case $# in + 0) ;; + 1) + [[ $1 == --update-report-fixtures ]] || + fail "usage: $0 [--update-report-fixtures]" + update_report_fixtures=true + ;; + *) fail "usage: $0 [--update-report-fixtures]" ;; +esac progress 'Validate manifests and live-skill reporting contract' [[ -x $spacedock_fixture_script ]] || fail 'renamed Spacedock fixture builder is missing or not executable' @@ -331,6 +369,15 @@ python3 "$renderer" "$self_run" "$self_run" contract \ "$self_run/config.json" >/dev/null python3 "$renderer" "$captured_run" "$captured_run" contract \ "$captured_run/config.json" >/dev/null +if [[ $update_report_fixtures == true ]]; then + copy_report_fixtures captured "$captured_run" + copy_report_fixtures self-reported "$self_run" +fi + +for report in report.md report.html report-artifact.html; do + require_exact_report captured "$captured_run/$report" + require_exact_report self-reported "$self_run/$report" +done read_action='Read: AGENTS.md' test_action='Test: bash behavior-diff/tests/live-report-contract.sh' From 79991cefa19cabca851f5e522b3710e3b579efb2 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:23:04 +0800 Subject: [PATCH 04/19] fix: preserve report fixture usage contract Signed-off-by: Kent Huang --- tests/live-report-contract.sh | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index c8a113c..019110f 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -105,6 +105,11 @@ progress() { printf '[report] %s\n' "$1" } +usage() { + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >&2 + exit 2 +} + copy_report_fixtures() { local mode=$1 @@ -135,12 +140,32 @@ require_exact_report() { case $# in 0) ;; 1) - [[ $1 == --update-report-fixtures ]] || - fail "usage: $0 [--update-report-fixtures]" + [[ $1 == --update-report-fixtures ]] || usage update_report_fixtures=true ;; - *) fail "usage: $0 [--update-report-fixtures]" ;; + *) usage ;; esac + +require_usage() { + local stderr=$tmp/usage-stderr.txt + local stdout=$tmp/usage-stdout.txt + local expected=$tmp/usage-expected.txt + local status + + if bash "$0" "$@" >"$stdout" 2>"$stderr"; then + fail "invalid arguments succeeded: $*" + else + status=$? + fi + [[ $status == 2 ]] || fail "invalid arguments returned $status instead of 2: $*" + [[ ! -s $stdout ]] || fail "invalid arguments wrote to stdout: $*" + printf 'Usage: %s [--update-report-fixtures]\n' "$0" >"$expected" + cmp -s "$expected" "$stderr" || + fail "invalid arguments did not print the exact usage diagnostic: $*" +} + +require_usage --unknown-option +require_usage --update-report-fixtures surplus progress 'Validate manifests and live-skill reporting contract' [[ -x $spacedock_fixture_script ]] || fail 'renamed Spacedock fixture builder is missing or not executable' From e04282f66dd956a36f1bc18d7d8aeb77a8ebe743 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:28:24 +0800 Subject: [PATCH 05/19] feat: define internal report data schema Signed-off-by: Kent Huang --- .../scripts/reporting/__init__.py | 1 + .../behavior-diff/scripts/reporting/schema.py | 258 ++++++++++++++++++ tests/live-report-contract.sh | 2 + tests/report-schema-test.py | 194 +++++++++++++ 4 files changed, 455 insertions(+) create mode 100644 plugin/skills/behavior-diff/scripts/reporting/__init__.py create mode 100644 plugin/skills/behavior-diff/scripts/reporting/schema.py create mode 100755 tests/report-schema-test.py diff --git a/plugin/skills/behavior-diff/scripts/reporting/__init__.py b/plugin/skills/behavior-diff/scripts/reporting/__init__.py new file mode 100644 index 0000000..ea79d5a --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/__init__.py @@ -0,0 +1 @@ +"""Private report rendering package.""" diff --git a/plugin/skills/behavior-diff/scripts/reporting/schema.py b/plugin/skills/behavior-diff/scripts/reporting/schema.py new file mode 100644 index 0000000..e755300 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/schema.py @@ -0,0 +1,258 @@ +"""Immutable internal data model for Behavior Diff reports.""" + +import json +from dataclasses import asdict, dataclass +from typing import Dict, Optional, Tuple, Union + +SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class TrialData: + name: str + verdict: str + actions: str + commands: Tuple[str, ...] + final: str + outcome: Optional[str] + + +@dataclass(frozen=True) +class VariantData: + label: str + note: str + passed: int + blocked: int + valid: int + total: int + count_text: str + count_emphasized: bool + trials: Tuple[TrialData, ...] + + +@dataclass(frozen=True) +class VariantsData: + before: VariantData + after: VariantData + + +@dataclass(frozen=True) +class FlowPathData: + steps: Tuple[str, ...] + count: int + + +@dataclass(frozen=True) +class FlowBranchData: + prefix: Tuple[str, ...] + paths: Tuple[FlowPathData, ...] + total: int + + +@dataclass(frozen=True) +class CommandFlowData: + enabled: bool + same: bool + shared: Tuple[str, ...] + before: FlowBranchData + after: FlowBranchData + + +@dataclass(frozen=True) +class DecisionChoiceData: + choice: str + count: int + + +@dataclass(frozen=True) +class DecisionRowData: + decision: str + topic: str + anchor: Union[int, str] + diverges: bool + note: str + before: Tuple[DecisionChoiceData, ...] + after: Tuple[DecisionChoiceData, ...] + + +@dataclass(frozen=True) +class DecisionData: + rows: Tuple[DecisionRowData, ...] + fork: Optional[int] + fork_note: str + dropped: int + extractor: str + before_count: int + after_count: int + + +@dataclass(frozen=True) +class MetadataData: + model: str + mode: str + vocab: str + trace_source: str + target_file: str + before_label: str + after_label: str + + +@dataclass(frozen=True) +class ContentData: + title: str + subtitle: str + observation: str + scenario_heading: str + scenario: str + expected_heading: str + expected: Optional[str] + diff_heading: str + decision_heading: str + decision_blurb: str + flow_heading: str + result_heading: str + boundary: str + + +@dataclass(frozen=True) +class ResultData: + text: str + kind: str + + +@dataclass(frozen=True) +class ReportData: + schema_version: int + metadata: MetadataData + content: ContentData + rule_diff: str + result: ResultData + variants: VariantsData + command_flow: CommandFlowData + decisions: DecisionData + + @classmethod + def from_dict(cls, data: Dict[str, object]) -> "ReportData": + version = data.get("schema_version") + if version != SCHEMA_VERSION: + raise ValueError("unsupported report-data schema version: {0}".format(version)) + + metadata_data = data["metadata"] + content_data = data["content"] + result_data = data["result"] + variants_data = data["variants"] + command_flow_data = data["command_flow"] + decisions_data = data["decisions"] + + def trial(raw): + return TrialData( + name=raw["name"], + verdict=raw["verdict"], + actions=raw["actions"], + commands=tuple(raw["commands"]), + final=raw["final"], + outcome=raw["outcome"], + ) + + def variant(raw): + return VariantData( + label=raw["label"], + note=raw["note"], + passed=raw["passed"], + blocked=raw["blocked"], + valid=raw["valid"], + total=raw["total"], + count_text=raw["count_text"], + count_emphasized=raw["count_emphasized"], + trials=tuple(trial(item) for item in raw["trials"]), + ) + + def flow_path(raw): + return FlowPathData(steps=tuple(raw["steps"]), count=raw["count"]) + + def flow_branch(raw): + return FlowBranchData( + prefix=tuple(raw["prefix"]), + paths=tuple(flow_path(item) for item in raw["paths"]), + total=raw["total"], + ) + + def decision_choice(raw): + return DecisionChoiceData(choice=raw["choice"], count=raw["count"]) + + def decision_row(raw): + return DecisionRowData( + decision=raw["decision"], + topic=raw["topic"], + anchor=raw["anchor"], + diverges=raw["diverges"], + note=raw["note"], + before=tuple(decision_choice(item) for item in raw["before"]), + after=tuple(decision_choice(item) for item in raw["after"]), + ) + + return cls( + schema_version=version, + metadata=MetadataData( + model=metadata_data["model"], + mode=metadata_data["mode"], + vocab=metadata_data["vocab"], + trace_source=metadata_data["trace_source"], + target_file=metadata_data["target_file"], + before_label=metadata_data["before_label"], + after_label=metadata_data["after_label"], + ), + content=ContentData( + title=content_data["title"], + subtitle=content_data["subtitle"], + observation=content_data["observation"], + scenario_heading=content_data["scenario_heading"], + scenario=content_data["scenario"], + expected_heading=content_data["expected_heading"], + expected=content_data["expected"], + diff_heading=content_data["diff_heading"], + decision_heading=content_data["decision_heading"], + decision_blurb=content_data["decision_blurb"], + flow_heading=content_data["flow_heading"], + result_heading=content_data["result_heading"], + boundary=content_data["boundary"], + ), + rule_diff=data["rule_diff"], + result=ResultData(text=result_data["text"], kind=result_data["kind"]), + variants=VariantsData( + before=variant(variants_data["before"]), + after=variant(variants_data["after"]), + ), + command_flow=CommandFlowData( + enabled=command_flow_data["enabled"], + same=command_flow_data["same"], + shared=tuple(command_flow_data["shared"]), + before=flow_branch(command_flow_data["before"]), + after=flow_branch(command_flow_data["after"]), + ), + decisions=DecisionData( + rows=tuple(decision_row(item) for item in decisions_data["rows"]), + fork=decisions_data["fork"], + fork_note=decisions_data["fork_note"], + dropped=decisions_data["dropped"], + extractor=decisions_data["extractor"], + before_count=decisions_data["before_count"], + after_count=decisions_data["after_count"], + ), + ) + + def to_dict(self): + return _json_value(asdict(self)) + + def to_json(self): + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +def _json_value(value): + if isinstance(value, tuple): + return [_json_value(item) for item in value] + if isinstance(value, list): + return [_json_value(item) for item in value] + if isinstance(value, dict): + return {key: _json_value(item) for key, item in value.items()} + return value diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index 019110f..24ae3b7 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -166,6 +166,8 @@ require_usage() { require_usage --unknown-option require_usage --update-report-fixtures surplus +python3 "$here/report-schema-test.py" + progress 'Validate manifests and live-skill reporting contract' [[ -x $spacedock_fixture_script ]] || fail 'renamed Spacedock fixture builder is missing or not executable' diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py new file mode 100755 index 0000000..5f7b41c --- /dev/null +++ b/tests/report-schema-test.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +scripts = Path(__file__).resolve().parents[1] / "plugin/skills/behavior-diff/scripts" +sys.path.insert(0, str(scripts)) + +from reporting.schema import ( # noqa: E402 + CommandFlowData, + DecisionChoiceData, + DecisionRowData, + FlowPathData, + ReportData, + TrialData, +) + + +def main(): + source = Path(sys.argv[1]) if len(sys.argv) == 2 else None + if source: + raw = json.loads(source.read_text()) + else: + raw = { + "schema_version": 1, + "metadata": { + "model": "synthetic/model", + "mode": "review", + "vocab": "generic", + "trace_source": "captured", + "target_file": "AGENTS.md", + "before_label": "current file", + "after_label": "your change applied", + }, + "content": { + "title": "Synthetic report", + "subtitle": "Synthetic subtitle.", + "observation": "Synthetic observation.", + "scenario_heading": "Scenario", + "scenario": "Compare two synthetic files.", + "expected_heading": "Expected behavior", + "expected": "Test the changed behavior.", + "diff_heading": "Diff of AGENTS.md — the only difference between the variants", + "decision_heading": "Decision diff — top divergences", + "decision_blurb": "Synthetic decision explanation.", + "flow_heading": "Flow diff — where the variants diverge", + "result_heading": "Result", + "boundary": "Synthetic evidence only.", + }, + "rule_diff": "--- before\n+++ after\n", + "result": {"text": "No automatic verdict", "kind": "neutral"}, + "variants": { + "before": { + "label": "Before", + "note": "current file", + "passed": 0, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "2 valid trials", + "count_emphasized": False, + "trials": [ + { + "name": "before-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md"], + "final": "Before first answer", + "outcome": None, + }, + { + "name": "before-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "Before second answer", + "outcome": "Reviewed file", + }, + ], + }, + "after": { + "label": "After", + "note": "your change applied", + "passed": 1, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "1 of 2 valid trials passed", + "count_emphasized": True, + "trials": [ + { + "name": "after-1", + "verdict": "PASS", + "actions": "-", + "commands": ["read AGENTS.md", "run tests"], + "final": "After first answer", + "outcome": "Tested behavior", + }, + { + "name": "after-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "After second answer", + "outcome": None, + }, + ], + }, + }, + "command_flow": { + "enabled": True, + "same": False, + "shared": ["Read files", "Compare output"], + "before": { + "prefix": ["Review result"], + "paths": [ + {"steps": ["Stop"], "count": 1}, + {"steps": ["Explain"], "count": 1}, + ], + "total": 2, + }, + "after": { + "prefix": ["Run tests"], + "paths": [{"steps": ["Explain"], "count": 2}], + "total": 2, + }, + }, + "decisions": { + "rows": [ + { + "decision": "Use evidence", + "topic": "Evidence", + "anchor": 2, + "diverges": True, + "note": "Synthetic divergence.", + "before": [ + {"choice": "read only", "count": 2}, + {"choice": "search", "count": 1}, + ], + "after": [{"choice": "read and test", "count": 2}], + }, + { + "decision": "State result", + "topic": "Delivery", + "anchor": "final answer", + "diverges": False, + "note": "", + "before": [{"choice": "explain", "count": 2}], + "after": [{"choice": "explain", "count": 2}], + }, + ], + "fork": 1, + "fork_note": "Synthetic fork.", + "dropped": 0, + "extractor": "synthetic extractor", + "before_count": 2, + "after_count": 2, + }, + } + + report = ReportData.from_dict(raw) + assert report.schema_version == 1 + assert isinstance(report.variants.before.trials[0], TrialData) + assert isinstance(report.command_flow, CommandFlowData) + assert isinstance(report.command_flow.before.paths[0], FlowPathData) + assert isinstance(report.decisions.rows[0], DecisionRowData) + assert isinstance(report.decisions.rows[0].before[0], DecisionChoiceData) + assert [trial.name for trial in report.variants.before.trials] == [ + "before-1", + "before-2", + ] + assert [path.steps for path in report.command_flow.before.paths] == [ + ("Stop",), + ("Explain",), + ] + assert [row.anchor for row in report.decisions.rows] == [2, "final answer"] + assert [choice.choice for choice in report.decisions.rows[0].before] == [ + "read only", + "search", + ] + assert report.to_dict() == raw + assert report.to_json() == json.dumps(raw, indent=2, sort_keys=True) + "\n" + + bad = dict(raw, schema_version=2) + try: + ReportData.from_dict(bad) + except ValueError as error: + assert str(error) == "unsupported report-data schema version: 2" + else: + raise AssertionError("schema version 2 was accepted") + + +if __name__ == "__main__": + main() From 343b0aad0bb4ff99471745271d62f4479be1722e Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:31:49 +0800 Subject: [PATCH 06/19] fix: keep report package initializer empty Signed-off-by: Kent Huang --- plugin/skills/behavior-diff/scripts/reporting/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/skills/behavior-diff/scripts/reporting/__init__.py b/plugin/skills/behavior-diff/scripts/reporting/__init__.py index ea79d5a..e69de29 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/__init__.py +++ b/plugin/skills/behavior-diff/scripts/reporting/__init__.py @@ -1 +0,0 @@ -"""Private report rendering package.""" From 062eb5cba63b320a2a61296fb4026678ed0e1017 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:38:28 +0800 Subject: [PATCH 07/19] fix: validate report data field types Signed-off-by: Kent Huang --- .../behavior-diff/scripts/reporting/schema.py | 386 +++++++++++++----- tests/report-schema-test.py | 47 +++ 2 files changed, 337 insertions(+), 96 deletions(-) diff --git a/plugin/skills/behavior-diff/scripts/reporting/schema.py b/plugin/skills/behavior-diff/scripts/reporting/schema.py index e755300..25dcced 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/schema.py +++ b/plugin/skills/behavior-diff/scripts/reporting/schema.py @@ -133,111 +133,29 @@ class ReportData: @classmethod def from_dict(cls, data: Dict[str, object]) -> "ReportData": - version = data.get("schema_version") + data = _expect_dict(data, "report-data") + version = _expect_int(_field(data, "schema_version", "report-data"), "schema_version") if version != SCHEMA_VERSION: raise ValueError("unsupported report-data schema version: {0}".format(version)) - metadata_data = data["metadata"] - content_data = data["content"] - result_data = data["result"] - variants_data = data["variants"] - command_flow_data = data["command_flow"] - decisions_data = data["decisions"] - - def trial(raw): - return TrialData( - name=raw["name"], - verdict=raw["verdict"], - actions=raw["actions"], - commands=tuple(raw["commands"]), - final=raw["final"], - outcome=raw["outcome"], - ) - - def variant(raw): - return VariantData( - label=raw["label"], - note=raw["note"], - passed=raw["passed"], - blocked=raw["blocked"], - valid=raw["valid"], - total=raw["total"], - count_text=raw["count_text"], - count_emphasized=raw["count_emphasized"], - trials=tuple(trial(item) for item in raw["trials"]), - ) - - def flow_path(raw): - return FlowPathData(steps=tuple(raw["steps"]), count=raw["count"]) - - def flow_branch(raw): - return FlowBranchData( - prefix=tuple(raw["prefix"]), - paths=tuple(flow_path(item) for item in raw["paths"]), - total=raw["total"], - ) - - def decision_choice(raw): - return DecisionChoiceData(choice=raw["choice"], count=raw["count"]) - - def decision_row(raw): - return DecisionRowData( - decision=raw["decision"], - topic=raw["topic"], - anchor=raw["anchor"], - diverges=raw["diverges"], - note=raw["note"], - before=tuple(decision_choice(item) for item in raw["before"]), - after=tuple(decision_choice(item) for item in raw["after"]), - ) - return cls( schema_version=version, - metadata=MetadataData( - model=metadata_data["model"], - mode=metadata_data["mode"], - vocab=metadata_data["vocab"], - trace_source=metadata_data["trace_source"], - target_file=metadata_data["target_file"], - before_label=metadata_data["before_label"], - after_label=metadata_data["after_label"], + metadata=_metadata( + _field(data, "metadata", "report-data"), "metadata" ), - content=ContentData( - title=content_data["title"], - subtitle=content_data["subtitle"], - observation=content_data["observation"], - scenario_heading=content_data["scenario_heading"], - scenario=content_data["scenario"], - expected_heading=content_data["expected_heading"], - expected=content_data["expected"], - diff_heading=content_data["diff_heading"], - decision_heading=content_data["decision_heading"], - decision_blurb=content_data["decision_blurb"], - flow_heading=content_data["flow_heading"], - result_heading=content_data["result_heading"], - boundary=content_data["boundary"], + content=_content(_field(data, "content", "report-data"), "content"), + rule_diff=_expect_str( + _field(data, "rule_diff", "report-data"), "rule_diff" ), - rule_diff=data["rule_diff"], - result=ResultData(text=result_data["text"], kind=result_data["kind"]), - variants=VariantsData( - before=variant(variants_data["before"]), - after=variant(variants_data["after"]), + result=_result(_field(data, "result", "report-data"), "result"), + variants=_variants( + _field(data, "variants", "report-data"), "variants" ), - command_flow=CommandFlowData( - enabled=command_flow_data["enabled"], - same=command_flow_data["same"], - shared=tuple(command_flow_data["shared"]), - before=flow_branch(command_flow_data["before"]), - after=flow_branch(command_flow_data["after"]), + command_flow=_command_flow( + _field(data, "command_flow", "report-data"), "command_flow" ), - decisions=DecisionData( - rows=tuple(decision_row(item) for item in decisions_data["rows"]), - fork=decisions_data["fork"], - fork_note=decisions_data["fork_note"], - dropped=decisions_data["dropped"], - extractor=decisions_data["extractor"], - before_count=decisions_data["before_count"], - after_count=decisions_data["after_count"], + decisions=_decisions( + _field(data, "decisions", "report-data"), "decisions" ), ) @@ -248,6 +166,282 @@ def to_json(self): return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" +def _metadata(value, path): + value = _expect_dict(value, path) + return MetadataData( + model=_expect_str(_field(value, "model", path), path + ".model"), + mode=_expect_str(_field(value, "mode", path), path + ".mode"), + vocab=_expect_str(_field(value, "vocab", path), path + ".vocab"), + trace_source=_expect_str( + _field(value, "trace_source", path), path + ".trace_source" + ), + target_file=_expect_str( + _field(value, "target_file", path), path + ".target_file" + ), + before_label=_expect_str( + _field(value, "before_label", path), path + ".before_label" + ), + after_label=_expect_str( + _field(value, "after_label", path), path + ".after_label" + ), + ) + + +def _content(value, path): + value = _expect_dict(value, path) + return ContentData( + title=_expect_str(_field(value, "title", path), path + ".title"), + subtitle=_expect_str(_field(value, "subtitle", path), path + ".subtitle"), + observation=_expect_str( + _field(value, "observation", path), path + ".observation" + ), + scenario_heading=_expect_str( + _field(value, "scenario_heading", path), path + ".scenario_heading" + ), + scenario=_expect_str( + _field(value, "scenario", path), path + ".scenario" + ), + expected_heading=_expect_str( + _field(value, "expected_heading", path), path + ".expected_heading" + ), + expected=_expect_optional_str( + _field(value, "expected", path), path + ".expected" + ), + diff_heading=_expect_str( + _field(value, "diff_heading", path), path + ".diff_heading" + ), + decision_heading=_expect_str( + _field(value, "decision_heading", path), path + ".decision_heading" + ), + decision_blurb=_expect_str( + _field(value, "decision_blurb", path), path + ".decision_blurb" + ), + flow_heading=_expect_str( + _field(value, "flow_heading", path), path + ".flow_heading" + ), + result_heading=_expect_str( + _field(value, "result_heading", path), path + ".result_heading" + ), + boundary=_expect_str( + _field(value, "boundary", path), path + ".boundary" + ), + ) + + +def _result(value, path): + value = _expect_dict(value, path) + return ResultData( + text=_expect_str(_field(value, "text", path), path + ".text"), + kind=_expect_str(_field(value, "kind", path), path + ".kind"), + ) + + +def _variants(value, path): + value = _expect_dict(value, path) + return VariantsData( + before=_variant(_field(value, "before", path), path + ".before"), + after=_variant(_field(value, "after", path), path + ".after"), + ) + + +def _variant(value, path): + value = _expect_dict(value, path) + trials = _expect_list(_field(value, "trials", path), path + ".trials") + return VariantData( + label=_expect_str(_field(value, "label", path), path + ".label"), + note=_expect_str(_field(value, "note", path), path + ".note"), + passed=_expect_int(_field(value, "passed", path), path + ".passed"), + blocked=_expect_int(_field(value, "blocked", path), path + ".blocked"), + valid=_expect_int(_field(value, "valid", path), path + ".valid"), + total=_expect_int(_field(value, "total", path), path + ".total"), + count_text=_expect_str( + _field(value, "count_text", path), path + ".count_text" + ), + count_emphasized=_expect_bool( + _field(value, "count_emphasized", path), path + ".count_emphasized" + ), + trials=tuple( + _trial(item, "{0}.trials[{1}]".format(path, index)) + for index, item in enumerate(trials) + ), + ) + + +def _trial(value, path): + value = _expect_dict(value, path) + return TrialData( + name=_expect_str(_field(value, "name", path), path + ".name"), + verdict=_expect_str(_field(value, "verdict", path), path + ".verdict"), + actions=_expect_str(_field(value, "actions", path), path + ".actions"), + commands=_string_tuple( + _field(value, "commands", path), path + ".commands" + ), + final=_expect_str(_field(value, "final", path), path + ".final"), + outcome=_expect_optional_str( + _field(value, "outcome", path), path + ".outcome" + ), + ) + + +def _command_flow(value, path): + value = _expect_dict(value, path) + return CommandFlowData( + enabled=_expect_bool(_field(value, "enabled", path), path + ".enabled"), + same=_expect_bool(_field(value, "same", path), path + ".same"), + shared=_string_tuple(_field(value, "shared", path), path + ".shared"), + before=_flow_branch(_field(value, "before", path), path + ".before"), + after=_flow_branch(_field(value, "after", path), path + ".after"), + ) + + +def _flow_branch(value, path): + value = _expect_dict(value, path) + paths = _expect_list(_field(value, "paths", path), path + ".paths") + return FlowBranchData( + prefix=_string_tuple(_field(value, "prefix", path), path + ".prefix"), + paths=tuple( + _flow_path(item, "{0}.paths[{1}]".format(path, index)) + for index, item in enumerate(paths) + ), + total=_expect_int(_field(value, "total", path), path + ".total"), + ) + + +def _flow_path(value, path): + value = _expect_dict(value, path) + return FlowPathData( + steps=_string_tuple(_field(value, "steps", path), path + ".steps"), + count=_expect_int(_field(value, "count", path), path + ".count"), + ) + + +def _decisions(value, path): + value = _expect_dict(value, path) + rows = _expect_list(_field(value, "rows", path), path + ".rows") + return DecisionData( + rows=tuple( + _decision_row(item, "{0}.rows[{1}]".format(path, index)) + for index, item in enumerate(rows) + ), + fork=_expect_optional_int(_field(value, "fork", path), path + ".fork"), + fork_note=_expect_str( + _field(value, "fork_note", path), path + ".fork_note" + ), + dropped=_expect_int(_field(value, "dropped", path), path + ".dropped"), + extractor=_expect_str( + _field(value, "extractor", path), path + ".extractor" + ), + before_count=_expect_int( + _field(value, "before_count", path), path + ".before_count" + ), + after_count=_expect_int( + _field(value, "after_count", path), path + ".after_count" + ), + ) + + +def _decision_row(value, path): + value = _expect_dict(value, path) + before = _expect_list(_field(value, "before", path), path + ".before") + after = _expect_list(_field(value, "after", path), path + ".after") + return DecisionRowData( + decision=_expect_str( + _field(value, "decision", path), path + ".decision" + ), + topic=_expect_str(_field(value, "topic", path), path + ".topic"), + anchor=_expect_anchor(_field(value, "anchor", path), path + ".anchor"), + diverges=_expect_bool( + _field(value, "diverges", path), path + ".diverges" + ), + note=_expect_str(_field(value, "note", path), path + ".note"), + before=tuple( + _decision_choice(item, "{0}.before[{1}]".format(path, index)) + for index, item in enumerate(before) + ), + after=tuple( + _decision_choice(item, "{0}.after[{1}]".format(path, index)) + for index, item in enumerate(after) + ), + ) + + +def _decision_choice(value, path): + value = _expect_dict(value, path) + return DecisionChoiceData( + choice=_expect_str(_field(value, "choice", path), path + ".choice"), + count=_expect_int(_field(value, "count", path), path + ".count"), + ) + + +def _field(value, name, path): + try: + return value[name] + except KeyError: + _invalid(path + "." + name, "present field") + + +def _expect_dict(value, path): + if type(value) is not dict: + _invalid(path, "dict") + return value + + +def _expect_list(value, path): + if type(value) is not list: + _invalid(path, "list") + return value + + +def _expect_str(value, path): + if type(value) is not str: + _invalid(path, "string") + return value + + +def _expect_bool(value, path): + if type(value) is not bool: + _invalid(path, "boolean") + return value + + +def _expect_int(value, path): + if type(value) is not int: + _invalid(path, "integer") + return value + + +def _expect_optional_str(value, path): + if value is None: + return None + return _expect_str(value, path) + + +def _expect_optional_int(value, path): + if value is None: + return None + return _expect_int(value, path) + + +def _expect_anchor(value, path): + if type(value) is int or type(value) is str: + return value + _invalid(path, "integer or string") + + +def _string_tuple(value, path): + value = _expect_list(value, path) + return tuple( + _expect_str(item, "{0}[{1}]".format(path, index)) + for index, item in enumerate(value) + ) + + +def _invalid(path, expected): + raise ValueError( + "invalid report-data field {0}: expected {1}".format(path, expected) + ) + + def _json_value(value): if isinstance(value, tuple): return [_json_value(item) for item in value] diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 5f7b41c..7e2baea 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import copy import json import sys from pathlib import Path @@ -189,6 +190,52 @@ def main(): else: raise AssertionError("schema version 2 was accepted") + invalid_cases = ( + ( + dict(raw, schema_version=True), + "invalid report-data field schema_version: expected integer", + ), + ( + dict(raw, schema_version=1.0), + "invalid report-data field schema_version: expected integer", + ), + ) + for invalid, message in invalid_cases: + try: + ReportData.from_dict(invalid) + except ValueError as error: + assert str(error) == message + else: + raise AssertionError("invalid schema version was accepted") + + invalid_commands = copy.deepcopy(raw) + invalid_commands["variants"]["before"]["trials"][0]["commands"] = "read AGENTS.md" + invalid_model = copy.deepcopy(raw) + invalid_model["metadata"]["model"] = ["synthetic/model"] + invalid_count = copy.deepcopy(raw) + invalid_count["command_flow"]["before"]["paths"][0]["count"] = True + invalid_fields = ( + ( + invalid_commands, + "invalid report-data field variants.before.trials[0].commands: expected list", + ), + ( + invalid_model, + "invalid report-data field metadata.model: expected string", + ), + ( + invalid_count, + "invalid report-data field command_flow.before.paths[0].count: expected integer", + ), + ) + for invalid, message in invalid_fields: + try: + ReportData.from_dict(invalid) + except ValueError as error: + assert str(error) == message + else: + raise AssertionError("invalid report data was accepted") + if __name__ == "__main__": main() From 203b270f4f951f9dfbc3e0724e87965df53c4b37 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:41:36 +0800 Subject: [PATCH 08/19] fix: separate report schema source checks Signed-off-by: Kent Huang --- tests/report-schema-test.py | 354 ++++++++++++++++++------------------ 1 file changed, 174 insertions(+), 180 deletions(-) diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 7e2baea..7ec4f9b 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -17,150 +17,165 @@ ) -def main(): - source = Path(sys.argv[1]) if len(sys.argv) == 2 else None - if source: - raw = json.loads(source.read_text()) - else: - raw = { - "schema_version": 1, - "metadata": { - "model": "synthetic/model", - "mode": "review", - "vocab": "generic", - "trace_source": "captured", - "target_file": "AGENTS.md", - "before_label": "current file", - "after_label": "your change applied", - }, - "content": { - "title": "Synthetic report", - "subtitle": "Synthetic subtitle.", - "observation": "Synthetic observation.", - "scenario_heading": "Scenario", - "scenario": "Compare two synthetic files.", - "expected_heading": "Expected behavior", - "expected": "Test the changed behavior.", - "diff_heading": "Diff of AGENTS.md — the only difference between the variants", - "decision_heading": "Decision diff — top divergences", - "decision_blurb": "Synthetic decision explanation.", - "flow_heading": "Flow diff — where the variants diverge", - "result_heading": "Result", - "boundary": "Synthetic evidence only.", - }, - "rule_diff": "--- before\n+++ after\n", - "result": {"text": "No automatic verdict", "kind": "neutral"}, - "variants": { - "before": { - "label": "Before", - "note": "current file", - "passed": 0, - "blocked": 0, - "valid": 2, - "total": 2, - "count_text": "2 valid trials", - "count_emphasized": False, - "trials": [ - { - "name": "before-1", - "verdict": "REVIEW", - "actions": "-", - "commands": ["read AGENTS.md"], - "final": "Before first answer", - "outcome": None, - }, - { - "name": "before-2", - "verdict": "REVIEW", - "actions": "-", - "commands": ["search AGENTS.md"], - "final": "Before second answer", - "outcome": "Reviewed file", - }, - ], - }, - "after": { - "label": "After", - "note": "your change applied", - "passed": 1, - "blocked": 0, - "valid": 2, - "total": 2, - "count_text": "1 of 2 valid trials passed", - "count_emphasized": True, - "trials": [ - { - "name": "after-1", - "verdict": "PASS", - "actions": "-", - "commands": ["read AGENTS.md", "run tests"], - "final": "After first answer", - "outcome": "Tested behavior", - }, - { - "name": "after-2", - "verdict": "REVIEW", - "actions": "-", - "commands": ["search AGENTS.md"], - "final": "After second answer", - "outcome": None, - }, - ], - }, - }, - "command_flow": { - "enabled": True, - "same": False, - "shared": ["Read files", "Compare output"], - "before": { - "prefix": ["Review result"], - "paths": [ - {"steps": ["Stop"], "count": 1}, - {"steps": ["Explain"], "count": 1}, - ], - "total": 2, - }, - "after": { - "prefix": ["Run tests"], - "paths": [{"steps": ["Explain"], "count": 2}], - "total": 2, - }, +def synthetic_raw(): + return { + "schema_version": 1, + "metadata": { + "model": "synthetic/model", + "mode": "review", + "vocab": "generic", + "trace_source": "captured", + "target_file": "AGENTS.md", + "before_label": "current file", + "after_label": "your change applied", + }, + "content": { + "title": "Synthetic report", + "subtitle": "Synthetic subtitle.", + "observation": "Synthetic observation.", + "scenario_heading": "Scenario", + "scenario": "Compare two synthetic files.", + "expected_heading": "Expected behavior", + "expected": "Test the changed behavior.", + "diff_heading": "Diff of AGENTS.md — the only difference between the variants", + "decision_heading": "Decision diff — top divergences", + "decision_blurb": "Synthetic decision explanation.", + "flow_heading": "Flow diff — where the variants diverge", + "result_heading": "Result", + "boundary": "Synthetic evidence only.", + }, + "rule_diff": "--- before\n+++ after\n", + "result": {"text": "No automatic verdict", "kind": "neutral"}, + "variants": { + "before": { + "label": "Before", + "note": "current file", + "passed": 0, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "2 valid trials", + "count_emphasized": False, + "trials": [ + { + "name": "before-1", + "verdict": "REVIEW", + "actions": "-", + "commands": ["read AGENTS.md"], + "final": "Before first answer", + "outcome": None, + }, + { + "name": "before-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "Before second answer", + "outcome": "Reviewed file", + }, + ], }, - "decisions": { - "rows": [ + "after": { + "label": "After", + "note": "your change applied", + "passed": 1, + "blocked": 0, + "valid": 2, + "total": 2, + "count_text": "1 of 2 valid trials passed", + "count_emphasized": True, + "trials": [ { - "decision": "Use evidence", - "topic": "Evidence", - "anchor": 2, - "diverges": True, - "note": "Synthetic divergence.", - "before": [ - {"choice": "read only", "count": 2}, - {"choice": "search", "count": 1}, - ], - "after": [{"choice": "read and test", "count": 2}], + "name": "after-1", + "verdict": "PASS", + "actions": "-", + "commands": ["read AGENTS.md", "run tests"], + "final": "After first answer", + "outcome": "Tested behavior", }, { - "decision": "State result", - "topic": "Delivery", - "anchor": "final answer", - "diverges": False, - "note": "", - "before": [{"choice": "explain", "count": 2}], - "after": [{"choice": "explain", "count": 2}], + "name": "after-2", + "verdict": "REVIEW", + "actions": "-", + "commands": ["search AGENTS.md"], + "final": "After second answer", + "outcome": None, }, ], - "fork": 1, - "fork_note": "Synthetic fork.", - "dropped": 0, - "extractor": "synthetic extractor", - "before_count": 2, - "after_count": 2, }, - } + }, + "command_flow": { + "enabled": True, + "same": False, + "shared": ["Read files", "Compare output"], + "before": { + "prefix": ["Review result"], + "paths": [ + {"steps": ["Stop"], "count": 1}, + {"steps": ["Explain"], "count": 1}, + ], + "total": 2, + }, + "after": { + "prefix": ["Run tests"], + "paths": [{"steps": ["Explain"], "count": 2}], + "total": 2, + }, + }, + "decisions": { + "rows": [ + { + "decision": "Use evidence", + "topic": "Evidence", + "anchor": 2, + "diverges": True, + "note": "Synthetic divergence.", + "before": [ + {"choice": "read only", "count": 2}, + {"choice": "search", "count": 1}, + ], + "after": [{"choice": "read and test", "count": 2}], + }, + { + "decision": "State result", + "topic": "Delivery", + "anchor": "final answer", + "diverges": False, + "note": "", + "before": [{"choice": "explain", "count": 2}], + "after": [{"choice": "explain", "count": 2}], + }, + ], + "fork": 1, + "fork_note": "Synthetic fork.", + "dropped": 0, + "extractor": "synthetic extractor", + "before_count": 2, + "after_count": 2, + }, + } + +def assert_round_trip(raw): report = ReportData.from_dict(raw) assert report.schema_version == 1 + assert report.to_dict() == raw + assert report.to_json() == json.dumps(raw, indent=2, sort_keys=True) + "\n" + return report + + +def assert_rejected(raw, message): + try: + ReportData.from_dict(raw) + except ValueError as error: + assert str(error) == message + else: + raise AssertionError("invalid report data was accepted") + + +def main(): + raw = synthetic_raw() + report = assert_round_trip(raw) assert isinstance(report.variants.before.trials[0], TrialData) assert isinstance(report.command_flow, CommandFlowData) assert isinstance(report.command_flow.before.paths[0], FlowPathData) @@ -179,62 +194,41 @@ def main(): "read only", "search", ] - assert report.to_dict() == raw - assert report.to_json() == json.dumps(raw, indent=2, sort_keys=True) + "\n" - - bad = dict(raw, schema_version=2) - try: - ReportData.from_dict(bad) - except ValueError as error: - assert str(error) == "unsupported report-data schema version: 2" - else: - raise AssertionError("schema version 2 was accepted") - invalid_cases = ( - ( - dict(raw, schema_version=True), - "invalid report-data field schema_version: expected integer", - ), - ( - dict(raw, schema_version=1.0), - "invalid report-data field schema_version: expected integer", - ), + assert_rejected( + dict(raw, schema_version=2), + "unsupported report-data schema version: 2", + ) + assert_rejected( + dict(raw, schema_version=True), + "invalid report-data field schema_version: expected integer", + ) + assert_rejected( + dict(raw, schema_version=1.0), + "invalid report-data field schema_version: expected integer", ) - for invalid, message in invalid_cases: - try: - ReportData.from_dict(invalid) - except ValueError as error: - assert str(error) == message - else: - raise AssertionError("invalid schema version was accepted") invalid_commands = copy.deepcopy(raw) invalid_commands["variants"]["before"]["trials"][0]["commands"] = "read AGENTS.md" + assert_rejected( + invalid_commands, + "invalid report-data field variants.before.trials[0].commands: expected list", + ) invalid_model = copy.deepcopy(raw) invalid_model["metadata"]["model"] = ["synthetic/model"] + assert_rejected( + invalid_model, + "invalid report-data field metadata.model: expected string", + ) invalid_count = copy.deepcopy(raw) invalid_count["command_flow"]["before"]["paths"][0]["count"] = True - invalid_fields = ( - ( - invalid_commands, - "invalid report-data field variants.before.trials[0].commands: expected list", - ), - ( - invalid_model, - "invalid report-data field metadata.model: expected string", - ), - ( - invalid_count, - "invalid report-data field command_flow.before.paths[0].count: expected integer", - ), + assert_rejected( + invalid_count, + "invalid report-data field command_flow.before.paths[0].count: expected integer", ) - for invalid, message in invalid_fields: - try: - ReportData.from_dict(invalid) - except ValueError as error: - assert str(error) == message - else: - raise AssertionError("invalid report data was accepted") + + if len(sys.argv) == 2: + assert_round_trip(json.loads(Path(sys.argv[1]).read_text())) if __name__ == "__main__": From e433e6903f0c05b75f03725a199b5153198b89a7 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 12:52:10 +0800 Subject: [PATCH 09/19] refactor: separate report data assembly Signed-off-by: Kent Huang --- plugin/skills/behavior-diff/scripts/render.py | 677 ++++++------------ .../scripts/reporting/content.py | 194 +++++ .../behavior-diff/scripts/reporting/load.py | 418 +++++++++++ tests/live-report-contract.sh | 17 + tests/report-schema-test.py | 12 +- 5 files changed, 846 insertions(+), 472 deletions(-) create mode 100644 plugin/skills/behavior-diff/scripts/reporting/content.py create mode 100644 plugin/skills/behavior-diff/scripts/reporting/load.py diff --git a/plugin/skills/behavior-diff/scripts/render.py b/plugin/skills/behavior-diff/scripts/render.py index 0bbc2f0..7609f9e 100644 --- a/plugin/skills/behavior-diff/scripts/render.py +++ b/plugin/skills/behavior-diff/scripts/render.py @@ -2,405 +2,165 @@ """Render a Behavior Diff run into report.md + report.html (+ artifact body). Usage: render.py RUN_DIR CAPSULE_DIR MODEL [CONFIG_JSON] - -Without a config this renders the built-in rk-monitor demo (graded mode, -demo step vocabulary). A config JSON generalizes it for behavior-diff runs: - {"title", "sub", "scenario", "expected" (null = no contract), - "target_file" (diffed between variants), "mode": "graded"|"review", - "vocab": "demo"|"generic", "trace_source": "captured"|"self-reported", - "before_label", "after_label"} -Review mode has no automatic verdict: trials get a neutral REVIEW badge and -the banner asks the user to compare flows and answers. """ -import difflib import html -import json -import re import sys -from collections import Counter from pathlib import Path +from reporting import content +from reporting.load import load_report + + run = Path(sys.argv[1]).resolve() capsule = Path(sys.argv[2]).resolve() model = sys.argv[3] -cfg = {} -if len(sys.argv) > 4: - cfg = json.loads(Path(sys.argv[4]).read_text()) - -MODE = cfg.get("mode", "graded") -VOCAB = cfg.get("vocab", "demo") -TRACE_SOURCE = cfg.get("trace_source", "captured") -if TRACE_SOURCE not in {"captured", "self-reported"}: - raise SystemExit('trace_source must be either "captured" or "self-reported"') -SELF_REPORTED = TRACE_SOURCE == "self-reported" -TARGET_FILE = cfg.get("target_file", "CLAUDE.md") -TITLE = cfg.get("title", "rk-monitor Behavior Check") -BEFORE_LABEL = cfg.get("before_label", "current file") -AFTER_LABEL = cfg.get("after_label", "your change applied") -DEFAULT_SUB = ( - "Same scenario, same recorded settings, six fresh agent runs. " - "The only difference between the two columns is one proposed " - "rule in the project's CLAUDE.md. Each trial shows the agent's " - "self-reported actions, not captured traces." - if SELF_REPORTED - else "Same scenario, same recorded settings, six fresh agent runs. " - "The only difference between the two columns is one proposed " - "rule in the project's CLAUDE.md. Each trial is graded from " - "the agent's actual tool calls, never its self-report." -) -SUB = cfg.get("sub", DEFAULT_SUB) -EXPECTED = cfg.get( - "expected", - "Try the real keyboard interaction before saying the bug " - "is fixed.\nIf that cannot be tested, say it is " - "unverified.", -) -BOUNDARY = ( - "This is simulation evidence. Real-use evidence is still pending.\n" - "It does not repair the original incident; it tests the change " - "for future tasks." -) - -grades = {} -for line in (run / "grades.tsv").read_text().splitlines(): - name, verdict, actions = line.split("\t", 2) - grades[name] = (verdict, actions) - - -def trial_data(name): - verdict, actions = grades[name] - cmds, final = [], "" - trace = run / name / "trace.jsonl" - if trace.exists(): - for raw in trace.read_text().splitlines(): - try: - obj = json.loads(raw) - except ValueError: - continue - if obj.get("type") == "assistant": - for c in obj.get("message", {}).get("content") or []: - if c.get("type") != "tool_use": - continue - inp = c.get("input") or {} - if inp.get("command"): - cmds.append(inp["command"]) - elif inp.get("file_path"): - cmds.append(f"[{c.get('name')}] {inp['file_path']}") - elif obj.get("type") == "result": - final = obj.get("result") or final +config_path = Path(sys.argv[4]) if len(sys.argv) > 4 else None +report = load_report(run, capsule, model, config_path) + + +# Temporary bridge while the existing Markdown and HTML layout is split later. +metadata = report.metadata +report_content = report.content +MODE = metadata.mode +SELF_REPORTED = metadata.trace_source == "self-reported" +TARGET_FILE = metadata.target_file +TITLE = report_content.title +SUB = report_content.subtitle +EXPECTED = report_content.expected +BOUNDARY = report_content.boundary +scenario = report_content.scenario +rule_diff = report.rule_diff +result = report.result.text +result_kind = report.result.kind +dec_blurb = report_content.decision_blurb +obs_md = report_content.observation + + +def _bridge_variant(variant): return { - "name": name, - "verdict": verdict, - "actions": actions, - "cmds": cmds, - "final": final, - } - - -# ---------- flow: plain-language steps derived from the commands ---------- -if VOCAB == "demo": - STEP_ORDER = ["inspect", "unit", "look", "func"] - STEP_LABEL = { - "inspect": "Inspect the change (git history, code, tests)", - "unit": "Run the unit tests", - "look": "Look for a functional / smoke test", - "func": "Drive the app with real key input (pty)", - } - - def classify(cmd): - c = cmd.lower() - keys = set() - if "monitor" in c and any( - k in c - for k in ("pty", "expect", "script -q", "tui-smoke", "\\x1b[", "\\033[") - ): - keys.add("func") - if "smoke" in c or "scripts" in c: - keys.add("look") - if "test_keys" in c or "pytest" in c: - keys.add("unit") - if ( - c.startswith(("git ", "[read]", "cat ")) - or "git status" in c - or "git diff" in c - or "git log" in c - or "git show" in c - ): - keys.add("inspect") - return keys -else: - # generic buckets; VOCAB == "spacedock" adds workflow-verb buckets on top - STEP_ORDER = ["inspect", "read", "search", "tests", "run"] - STEP_LABEL = { - "inspect": "Inspect git history and status", - "read": "Read files", - "search": "Search the codebase", - "tests": "Run tests", - "run": "Run the app or a script", - } - if VOCAB == "spacedock": - STEP_ORDER += [ - "entity_write", - "state_commit", - "gate_prepare", - "gate_record", - "dispatch", - ] - STEP_LABEL.update( + "trials": [ { - "entity_write": "Write entity state (new / status --set)", - "state_commit": "Commit or publish state", - "gate_prepare": "Prepare a gate room", - "gate_record": "Record a gate decision", - "dispatch": "Dispatch or rework (worktree)", + "name": trial.name, + "verdict": trial.verdict, + "actions": trial.actions, + "cmds": trial.commands, + "final": trial.final, } - ) - - def classify(cmd): - c = cmd.lower() - keys = set() - if re.search(r"(^|[;&|(]\s*)git ", c): - keys.add("inspect") - if c.startswith(("[read]", "cat ", "head ", "less ")) or "sed -n" in c: - keys.add("read") - if re.search(r"\b(grep|rg|find|ag)\b", c): - keys.add("search") - ran_tests = "pytest" in c or re.search(r"\btest[s_]?\b", c) - if ran_tests: - keys.add("tests") - elif re.search(r"\b(python3?|bash|sh|node|npm|make|cargo|go)\b", c): - keys.add("run") - if VOCAB == "spacedock": - if "gate prepare" in c: - keys.add("gate_prepare") - if "gate record" in c: - keys.add("gate_record") - if "state commit" in c or "state publish" in c: - keys.add("state_commit") - if re.search(r"(spacedock|sd) new\b", c) or "status --set" in c: - keys.add("entity_write") - if "worktree add" in c or re.search(r"\bdispatch\b", c): - keys.add("dispatch") - return keys - - -def outcome_label(t): - if t["verdict"] == "BLOCKED": - return "Blocked — no valid run" - if MODE == "review": - return None # no grading contract: the answer itself is the outcome - if t["verdict"] == "FAIL": - return "Claim the fix is complete on unit tests alone" - if "unverified" in t["actions"]: - return "Say the behavior is unverified, claim nothing" - return "Claim complete, with functional evidence" - - -def trial_seq(t): - seen = set() - for cmd in t["cmds"]: - seen |= classify(cmd) - seq = tuple(k for k in STEP_ORDER if k in seen) - out = outcome_label(t) - return seq + (("out:" + out,) if out else ()) - - -def step_text(step): - return step[4:] if step.startswith("out:") else STEP_LABEL[step] - - -def common_prefix(seqs): - out = [] - for items in zip(*seqs): - if any(x != items[0] for x in items): - break - out.append(items[0]) - return out - - -def build_flow(before_trials, after_trials): - bseqs = [trial_seq(t) for t in before_trials] - aseqs = [trial_seq(t) for t in after_trials] - shared = common_prefix(bseqs + aseqs) - - def branch(seqs): - rems = [s[len(shared) :] for s in seqs] - prefix = common_prefix(rems) - paths = Counter(tuple(r[len(prefix) :]) for r in rems) - paths.pop((), None) - return prefix, paths.most_common(), len(seqs) - - return shared, branch(bseqs), branch(aseqs) - - -# ---------- gather ---------- -variants = {} -for v in ("before", "after"): - trials = [trial_data(n) for n in sorted(grades) if n.startswith(v + "-")] - blocked = sum(t["verdict"] == "BLOCKED" for t in trials) - variants[v] = { - "trials": trials, - "passed": sum(t["verdict"] == "PASS" for t in trials), - "blocked": blocked, - "valid": len(trials) - blocked, - "total": len(trials), + for trial in variant.trials + ], + "passed": variant.passed, + "blocked": variant.blocked, + "valid": variant.valid, + "total": variant.total, } -b, a = variants["before"], variants["after"] -if MODE == "review": - result = ( - "No automatic verdict — compare the reported actions and final answers" - if SELF_REPORTED - else "No automatic verdict — compare the flows and final answers" - ) - result_kind = "neutral" -elif b["valid"] < b["total"] or a["valid"] < a["total"]: - result, result_kind = "Could not test", "neutral" -elif b["passed"] == 0 and a["passed"] == a["valid"]: - result, result_kind = "Changed in this scenario", "good" -elif b["passed"] == 0 and a["passed"] == 0: - result, result_kind = "The proposed rule did not change behavior", "bad" -elif b["passed"] == b["valid"] and a["passed"] == a["valid"]: - result, result_kind = "The original problem was not reproduced", "neutral" -elif b["passed"] == b["valid"] and a["passed"] == 0: - result, result_kind = "The proposed rule made behavior worse", "bad" -else: - result, result_kind = "Behavior was inconsistent", "neutral" -if MODE != "review" and b["total"] == 1: - result += " — in this single run, weaker evidence" - -if SELF_REPORTED: - shared = bprefix = aprefix = () - bpaths = apaths = [] - nb, na = b["total"], a["total"] -else: - shared, (bprefix, bpaths, nb), (aprefix, apaths, na) = build_flow( - b["trials"], a["trials"] - ) -same_flow = not bprefix and not bpaths and not aprefix and not apaths - -scenario = cfg.get("scenario") or (capsule / "task.md").read_text().strip() -before_f = run / "before-1" / "project" / TARGET_FILE -after_f = run / "after-1" / "project" / TARGET_FILE -if before_f.exists() and after_f.exists(): - rule_diff = "".join( - difflib.unified_diff( - before_f.read_text().splitlines(keepends=True), - after_f.read_text().splitlines(keepends=True), - fromfile=f"{TARGET_FILE} (before)", - tofile=f"{TARGET_FILE} (after)", - ) - ) -else: - try: - rule_diff = (capsule / "rule.md").read_text() - except OSError: - rule_diff = "(no variant files or rule.md found — diff unavailable)" - -count_line = {} -for v in ("before", "after"): - d = variants[v] - if MODE == "review": - count_line[v] = ( - f"{d['valid']} valid trial(s) · no automatic " - f"grading (blocked: {d['blocked']})" - ) - else: - count_line[v] = ( - f"**{d['passed']} of {d['valid']} valid trials met " - f"the expectation** (blocked: {d['blocked']})" - ) -# ---------- decision diff (optional: decisions.py wrote decisions.json) ---------- -if SELF_REPORTED: - DEC_BLURB = ( - "A decision is a point where the agent had a real choice. The " - "decisions come from self-reported actions and final answers. Some " - "decisions leave no reported action behind. Order follows the " - "report: decisions visible in actions come in reported action " - "order, and decisions visible only in the final answer come last. " - "Extractor output can vary from run to run." +variants = { + "before": _bridge_variant(report.variants.before), + "after": _bridge_variant(report.variants.after), +} +b = variants["before"] +a = variants["after"] +count_line = { + name: ( + "**" + variant.count_text + "**" + if variant.count_emphasized + else variant.count_text ) -else: - DEC_BLURB = ( - "A decision is a point where the agent had a real choice. These " - "are recovered from what the trials did and said, not from the " - "instruction diff, and some of them leave no command behind. Order " - "is real: decisions visible in commands come in command order, and " - "decisions visible only in the final answer come last. The fork " - "and main divergences are stable across extractions; minor rows " - "can vary run to run." - ) -dec = {} -dec_path = run / "decisions.json" -if dec_path.exists(): - try: - dec = json.loads(dec_path.read_text()) - except ValueError: - dec = {} -if MODE == "review" and SELF_REPORTED and dec.get("chain"): - result = ( - "No automatic verdict — compare the reported actions, decision diff, " - "and final answers" + for name, variant in ( + ("before", report.variants.before), + ("after", report.variants.after), ) +} + +flow = report.command_flow +shared = flow.shared +bprefix = flow.before.prefix +aprefix = flow.after.prefix +bpaths = [(path.steps, path.count) for path in flow.before.paths] +apaths = [(path.steps, path.count) for path in flow.after.paths] +nb = flow.before.total +na = flow.after.total +same_flow = flow.same + + +def step_text(step): + return step + + +dec = { + "chain": [ + { + "decision": row.decision, + "topic": row.topic, + "anchor": row.anchor, + "diverges": row.diverges, + "note": row.note, + "before": [ + {"choice": choice.choice, "n": choice.count} for choice in row.before + ], + "after": [ + {"choice": choice.choice, "n": choice.count} for choice in row.after + ], + } + for row in report.decisions.rows + ], + "fork": report.decisions.fork, + "fork_note": report.decisions.fork_note, + "dropped": report.decisions.dropped, + "counts": { + "before": report.decisions.before_count, + "after": report.decisions.after_count, + }, +} def branch_str(brs, n): return ( " · ".join( - (b["choice"] if b["n"] == n else f"{b['choice']} ({b['n']}/{n})") - for b in brs + (branch["choice"] if branch["n"] == n else f"{branch['choice']} ({branch['n']}/{n})") + for branch in brs ) or "—" ) - -DEC_N1 = ( - " CAUTION — one trial per side: any divergence here can be " - "run-to-run variation rather than a rule effect; confirm with " - "repeated trials (behavior-diff 3+3) before acting on it." -) -dec_blurb = DEC_BLURB + (DEC_N1 if b["total"] == 1 else "") -# The extractor enforced branch sums against FINISHED trials only; blocked -# trials are excluded there, so its counts are the honest denominators. -dnb = (dec.get("counts") or {}).get("before", nb) -dna = (dec.get("counts") or {}).get("after", na) - +dnb = dec["counts"]["before"] +dna = dec["counts"]["after"] dec_md = [] -if dec.get("chain"): - fork = dec.get("fork") +if dec["chain"]: + fork = dec["fork"] lead_n = 0 for row in dec["chain"]: if row["diverges"]: break lead_n += 1 - dec_md += ["## Decision diff — top divergences\n", dec_blurb + "\n"] + dec_md += [f"## {report_content.decision_heading}\n", dec_blurb + "\n"] if lead_n: dec_md.append("Decided the same way on both sides:\n") for i, row in enumerate(dec["chain"][:lead_n], 1): - bs = branch_str(row["before"], dnb) - as_ = branch_str(row["after"], dna) - choice = bs if bs == as_ else f"before: {bs} · after: {as_}" - note = f" — {row['note']}" if row.get("note") else "" - when = " *(in the final answer)*" if row.get("anchor") == "answer" else "" - title = row.get("topic") or row["decision"] + before_choice = branch_str(row["before"], dnb) + after_choice = branch_str(row["after"], dna) + choice = ( + before_choice + if before_choice == after_choice + else f"before: {before_choice} · after: {after_choice}" + ) + note = f" — {row['note']}" if row["note"] else "" + when = " *(in the final answer)*" if row["anchor"] == "answer" else "" + title = row["topic"] or row["decision"] dec_md.append(f"- {i}. **{title}**{when} → {choice}{note}") dec_md.append("") if lead_n < len(dec["chain"]): dec_md.append("Diverging from here:\n") for i, row in enumerate(dec["chain"][lead_n:], lead_n + 1): - mark = ( - " ⟵ root behavior change" - if i == fork - else ( - " *(downstream)*" if row["diverges"] and fork and i > fork else "" - ) + mark = " ⟵ root behavior change" if i == fork else ( + " *(downstream)*" if row["diverges"] and fork and i > fork else "" ) - mark += " *(in the final answer)*" if row.get("anchor") == "answer" else "" + mark += " *(in the final answer)*" if row["anchor"] == "answer" else "" title = ( f"**{row['topic']}** — {row['decision']}" - if row.get("topic") + if row["topic"] else row["decision"] ) if row["diverges"]: @@ -412,62 +172,39 @@ def branch_str(brs, n): f"- {i}. {row['decision']} *(same)* → " f"{branch_str(row['before'], dnb)}" ) - if row.get("note"): + if row["note"]: dec_md.append(f" - note: {row['note']}") dec_md.append("") - n_div = sum(r["diverges"] for r in dec["chain"]) - if fork: - rest = n_div - 1 - dec_md.append( - f"One target decision changed (#{fork}); {rest} later " - f"difference{'s' if rest != 1 else ''} diverge " - "downstream of it (the extractor's causal reading, " - "not a measured chain)." - ) - else: - dec_md.append(f"{n_div} of {len(dec['chain'])} decisions diverge.") - if dec.get("fork_note"): - dec_md.append("\n" + dec["fork_note"]) - if dec.get("dropped"): - dec_md.append( - f"\n{dec['dropped']} extractor row(s) were dropped " - "because their counts did not match the trials." - ) + dec_md.append(content.decision_footer(report.decisions.rows, report.decisions.fork)) + if report.decisions.fork_note: + dec_md.append("\n" + report.decisions.fork_note) + if report.decisions.dropped: + dec_md.append("\n" + content.dropped_rows(report.decisions.dropped)) dec_md.append("") - -obs_md = "" -if MODE == "review" and dec.get("chain") and dec.get("fork"): - frow = dec["chain"][dec["fork"] - 1] - ftitle = frow.get("topic") or frow["decision"] - fb = branch_str(frow["before"], (dec.get("counts") or {}).get("before", nb)) - fa = branch_str(frow["after"], (dec.get("counts") or {}).get("after", na)) - obs_md = ( - f"Observed in this run — {ftitle}: BEFORE {fb} · AFTER {fa}. " - "Single-run observation, not a verdict." - ) - # ---------- report.md ---------- md = [f"# {TITLE}\n", SUB + "\n"] if obs_md: md.append("**" + obs_md + "**\n") md += [ f"Model: {model} · {b['total']} trial(s) per variant.\n", - "## Scenario\n", + f"## {report_content.scenario_heading}\n", scenario + "\n", ] if EXPECTED: - md += ["## Expected behavior\n", EXPECTED + "\n"] + md += [f"## {report_content.expected_heading}\n", EXPECTED + "\n"] md += [ - f"## Diff of {TARGET_FILE} — the only difference between the variants\n", + f"## {report_content.diff_heading}\n", "```diff\n" + rule_diff.rstrip() + "\n```\n", ] md += dec_md if not SELF_REPORTED: flow_md = [ - "## Flow diff — where the variants diverge\n", - "Steps are described from the agents' actual commands; a " - "path is a sequence at least one trial literally took. Full " - "commands are in the trial sections below.\n", + f"## {report_content.flow_heading}\n", + ( + "Steps are described from the agents' actual commands; a " + "path is a sequence at least one trial literally took. Full " + "commands are in the trial sections below.\n" + ), ] if same_flow: flow_md.append( @@ -486,98 +223,101 @@ def md_branch(tag, prefix, paths, n): flow_md.append( f"- {tag}, all {n} trials → " + ( - " → ".join(step_text(s) for s in prefix) + " → ".join(step_text(step) for step in prefix) or "(same steps as the shared flow)" ) ) return lead = f"- {tag}" if prefix: - lead += ", all trials → " + " → ".join(step_text(s) for s in prefix) + lead += ", all trials → " + " → ".join( + step_text(step) for step in prefix + ) flow_md.append(lead + ", then splits:") - for path, cnt in paths: + for path, count in paths: flow_md.append( - f" - {cnt} of {n} trials → " - + " → ".join(step_text(s) for s in path) + f" - {count} of {n} trials → " + + " → ".join(step_text(step) for step in path) ) md_branch("BEFORE", bprefix, bpaths, nb) md_branch("AFTER", aprefix, apaths, na) flow_md.append("") if dec_md: - md.append( - "
Flow diff — command-derived (deterministic, " - "no model involved)\n" - ) + md.append("
" + content.flow_fold_summary() + "\n") md += flow_md md.append("
\n") else: md += flow_md -for v, label in ( - ("before", f"BEFORE — {BEFORE_LABEL}"), - ("after", f"AFTER — {AFTER_LABEL}"), +for variant_name, label in ( + ("before", f"BEFORE — {metadata.before_label}"), + ("after", f"AFTER — {metadata.after_label}"), ): - d = variants[v] + variant = variants[variant_name] md.append(f"## {label}\n") - md.append(count_line[v] + "\n") - for t in d["trials"]: - md.append(f"### {t['name']} — {t['verdict']}\n") - if t["actions"] != "-": - md.append(t["actions"] + "\n") + md.append(count_line[variant_name] + "\n") + for trial in variant["trials"]: + md.append(f"### {trial['name']} — {trial['verdict']}\n") + if trial["actions"] != "-": + md.append(trial["actions"] + "\n") action_label = ( "self-reported actions" if SELF_REPORTED else "commands the agent ran" ) md.append( f"
{action_label} " - f"({len(t['cmds'])})\n\n```\n" - + "\n\n".join(c[:500] for c in t["cmds"]) + f"({len(trial['cmds'])})\n\n```\n" + + "\n\n".join(command[:500] for command in trial["cmds"]) + "\n```\n
\n" ) md.append( "
final answer to the user\n\n" - + t["final"].strip() + + trial["final"].strip() + "\n\n
\n" ) -md += ["## Result\n", f"**{result}**\n", BOUNDARY + "\n"] -(run / "report.md").write_text("\n".join(md)) +md += [ + f"## {report_content.result_heading}\n", + f"**{result}**\n", + BOUNDARY + "\n", +] +markdown = "\n".join(md) # ---------- HTML ---------- esc = html.escape -def card(t): - cls = t["verdict"].lower() +def card(trial): + cls = trial["verdict"].lower() if SELF_REPORTED: - evidence = "\n\n".join(t["cmds"]) or "(no self-reported actions)" - evidence_label = "self-reported actions" + evidence = "\n\n".join(trial["cmds"]) or "(no self-reported actions)" else: - evidence = "\n\n".join("$ " + c for c in t["cmds"]) or "(no commands)" - evidence_label = "Commands the agent ran" - acts = "" if t["actions"] == "-" else f'

{esc(t["actions"])}

' + evidence = "\n\n".join("$ " + command for command in trial["cmds"]) or "(no commands)" + acts = ( + "" + if trial["actions"] == "-" + else f'

{esc(trial["actions"])}

' + ) return ( f'
' - f'

{t["verdict"]}' - f"{esc(t['name'])}

{acts}" - f"
{evidence_label} ({len(t['cmds'])})" - f"
{esc(evidence)}
" + f'

{trial["verdict"]}' + f"{esc(trial['name'])}

{acts}" + f"
{'self-reported actions' if SELF_REPORTED else 'Commands the agent ran'} " + f"({len(trial['cmds'])})
{esc(evidence)}
" f"
" f"Final answer to the user" - f"
{esc(t['final'].strip())}
" + f"
{esc(trial['final'].strip())}
" ) cols = "" -for v, label, note in ( - ("before", "Before", BEFORE_LABEL), - ("after", "After", AFTER_LABEL), +for name, label, variant in ( + ("before", "Before", report.variants.before), + ("after", "After", report.variants.after), ): - d = variants[v] - cl = count_line[v].replace("**", "") cols += ( f'

{label}

' - f'{esc(note)}
' - f'

{esc(cl)}

' - + "".join(card(t) for t in d["trials"]) + f'{esc(variant.note)}' + f'

{esc(variant.count_text)}

' + + "".join(card(trial) for trial in variants[name]["trials"]) + "
" ) @@ -720,26 +460,13 @@ def dec_label(i, row, fork): + "" ) parts.append(f'
{"".join(grid)}
') - n_div = sum(r["diverges"] for r in dec["chain"]) - if fork: - rest = n_div - 1 - foot = ( - f"One target decision changed (#{fork}); {rest} later " - f"difference{'s' if rest != 1 else ''} " - "diverge downstream of it (the extractor's causal " - "reading, not a measured chain)." - ) - else: - foot = f"{n_div} of {len(dec['chain'])} decisions diverge." - if dec.get("fork_note"): - foot += " " + dec["fork_note"] - if dec.get("dropped"): - foot += ( - f" {dec['dropped']} extractor row(s) were dropped because " - "their counts did not match the trials." - ) + foot = content.decision_footer(report.decisions.rows, report.decisions.fork) + if report.decisions.fork_note: + foot += " " + report.decisions.fork_note + if report.decisions.dropped: + foot += " " + content.dropped_rows(report.decisions.dropped) dec_html = ( - '' + f'' f'

{esc(dec_blurb)}

' f'
{"".join(parts)}
' f'

{esc(foot)}

' @@ -748,17 +475,22 @@ def dec_label(i, row, fork): flow_section = "" if not SELF_REPORTED: flow_section = ( - '' - '

Steps are described from the agents\' actual commands. ' - "A path is a sequence at least one trial literally took — arrows " - "connect steps inside a path, and a split shows where trials went " - "different ways. Full commands are in the trial cards below.

" + flow_html + f'' + + ( + "

Steps are described from the agents' actual commands. " + "A path is a sequence at least one trial literally took — arrows " + "connect steps inside a path, and a split shows where trials went " + "different ways. Full commands are in the trial cards below.

" + ) + + flow_html ) if dec_html: flow_section = ( - '
Flow diff — ' - "command-derived (deterministic, no model involved)" - "" + flow_section + "
" + '
' + + content.flow_fold_summary() + + "" + + flow_section + + "
" ) obs_html = f'

{esc(obs_md)}

' if obs_md else "" @@ -767,7 +499,7 @@ def dec_label(i, row, fork): "" if not EXPECTED else ( - '' + f'' f'

{esc(EXPECTED)}

' ) ) @@ -887,11 +619,11 @@ def dec_label(i, row, fork):

{esc(SUB)}

{obs_html} - +
{esc(scenario)}
{expected_html} - +
{diff_html}
{dec_html} @@ -901,19 +633,24 @@ def dec_label(i, row, fork):
{cols}
- +
{esc(result)}
""" -(run / "report-artifact.html").write_text(body) -(run / "report.html").write_text( +document = ( '' '' "" + body + "" ) +report_json = report.to_json() + +(run / "report-data.json").write_text(report_json) +(run / "report.md").write_text(markdown) +(run / "report-artifact.html").write_text(body) +(run / "report.html").write_text(document) print( f"mode {MODE} · BEFORE pass {b['passed']}/{b['valid']} · " diff --git a/plugin/skills/behavior-diff/scripts/reporting/content.py b/plugin/skills/behavior-diff/scripts/reporting/content.py new file mode 100644 index 0000000..3a00e38 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/content.py @@ -0,0 +1,194 @@ +"""Format-neutral wording for Behavior Diff reports.""" + +from reporting.schema import ContentData + + +def subtitle(self_reported): + if self_reported: + return ( + "Same scenario, same recorded settings, six fresh agent runs. " + "The only difference between the two columns is one proposed " + "rule in the project's CLAUDE.md. Each trial shows the agent's " + "self-reported actions, not captured traces." + ) + return ( + "Same scenario, same recorded settings, six fresh agent runs. " + "The only difference between the two columns is one proposed " + "rule in the project's CLAUDE.md. Each trial is graded from " + "the agent's actual tool calls, never its self-report." + ) + + +def boundary(): + return ( + "This is simulation evidence. Real-use evidence is still pending.\n" + "It does not repair the original incident; it tests the change " + "for future tasks." + ) + + +def result_data(mode, self_reported, before, after, trial_count, has_decisions): + if mode == "review": + text = ( + "No automatic verdict — compare the reported actions and final answers" + if self_reported + else "No automatic verdict — compare the flows and final answers" + ) + if self_reported and has_decisions: + text = ( + "No automatic verdict — compare the reported actions, decision diff, " + "and final answers" + ) + return text, "neutral" + if before["valid"] < before["total"] or after["valid"] < after["total"]: + text, kind = "Could not test", "neutral" + elif before["passed"] == 0 and after["passed"] == after["valid"]: + text, kind = "Changed in this scenario", "good" + elif before["passed"] == 0 and after["passed"] == 0: + text, kind = "The proposed rule did not change behavior", "bad" + elif before["passed"] == before["valid"] and after["passed"] == after["valid"]: + text, kind = "The original problem was not reproduced", "neutral" + elif before["passed"] == before["valid"] and after["passed"] == 0: + text, kind = "The proposed rule made behavior worse", "bad" + else: + text, kind = "Behavior was inconsistent", "neutral" + return text + single_run_suffix(mode, trial_count), kind + + +def single_run_suffix(mode, trial_count): + if mode != "review" and trial_count == 1: + return " — in this single run, weaker evidence" + return "" + + +def count_data(mode, passed, valid, blocked): + if mode == "review": + return "{0} valid trial(s) · no automatic grading (blocked: {1})".format( + valid, blocked + ), False + return "{0} of {1} valid trials met the expectation (blocked: {2})".format( + passed, valid, blocked + ), True + + +def decision_blurb(self_reported, trial_count): + if self_reported: + blurb = ( + "A decision is a point where the agent had a real choice. The " + "decisions come from self-reported actions and final answers. Some " + "decisions leave no reported action behind. Order follows the " + "report: decisions visible in actions come in reported action " + "order, and decisions visible only in the final answer come last. " + "Extractor output can vary from run to run." + ) + else: + blurb = ( + "A decision is a point where the agent had a real choice. These " + "are recovered from what the trials did and said, not from the " + "instruction diff, and some of them leave no command behind. Order " + "is real: decisions visible in commands come in command order, and " + "decisions visible only in the final answer come last. The fork " + "and main divergences are stable across extractions; minor rows " + "can vary run to run." + ) + if trial_count == 1: + blurb += ( + " CAUTION — one trial per side: any divergence here can be " + "run-to-run variation rather than a rule effect; confirm with " + "repeated trials (behavior-diff 3+3) before acting on it." + ) + return blurb + + +def observation(mode, decisions, before_count, after_count): + if mode != "review" or not decisions.rows or not decisions.fork: + return "" + row = decisions.rows[decisions.fork - 1] + title = row.topic or row.decision + before = branch_text(row.before, decisions.before_count or before_count) + after = branch_text(row.after, decisions.after_count or after_count) + return ( + "Observed in this run — {0}: BEFORE {1} · AFTER {2}. " + "Single-run observation, not a verdict." + ).format(title, before, after) + + +def branch_text(choices, total): + return " · ".join( + choice.choice + if choice.count == total + else "{0} ({1}/{2})".format(choice.choice, choice.count, total) + for choice in choices + ) or "—" + + +def headings(target_file): + return { + "scenario": "Scenario", + "expected": "Expected behavior", + "diff": "Diff of {0} — the only difference between the variants".format( + target_file + ), + "decision": "Decision diff — top divergences", + "flow": "Flow diff — where the variants diverge", + "result": "Result", + } + + + + +def flow_fold_summary(): + return "Flow diff — command-derived (deterministic, no model involved)" + + + +def decision_footer(rows, fork): + divergent = sum(row.diverges for row in rows) + if fork: + rest = divergent - 1 + return ( + "One target decision changed (#{0}); {1} later difference{2} diverge " + "downstream of it (the extractor's causal reading, not a measured chain)." + ).format(fork, rest, "s" if rest != 1 else "") + return "{0} of {1} decisions diverge.".format(divergent, len(rows)) + + +def dropped_rows(dropped): + return ( + "{0} extractor row(s) were dropped because their counts did not match the trials." + ).format(dropped) + + + +def build_content( + config, + scenario, + mode, + trace_source, + target_file, + decisions, + before_total, + after_total, +): + self_reported = trace_source == "self-reported" + names = headings(target_file) + return ContentData( + title=config.get("title", "rk-monitor Behavior Check"), + subtitle=config.get("sub", subtitle(self_reported)), + observation=observation(mode, decisions, before_total, after_total), + scenario_heading=names["scenario"], + scenario=scenario, + expected_heading=names["expected"], + expected=config.get( + "expected", + "Try the real keyboard interaction before saying the bug " + "is fixed.\nIf that cannot be tested, say it is " + "unverified.", + ), + diff_heading=names["diff"], + decision_heading=names["decision"], + decision_blurb=decision_blurb(self_reported, before_total), + flow_heading=names["flow"], + result_heading=names["result"], + boundary=boundary(), + ) \ No newline at end of file diff --git a/plugin/skills/behavior-diff/scripts/reporting/load.py b/plugin/skills/behavior-diff/scripts/reporting/load.py new file mode 100644 index 0000000..625a878 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/load.py @@ -0,0 +1,418 @@ +"""Load persisted Behavior Diff evidence into format-neutral report data.""" + +import difflib +import json +import re +from collections import Counter +from pathlib import Path +from typing import Optional + +from reporting import content +from reporting.schema import ( + SCHEMA_VERSION, + CommandFlowData, + DecisionChoiceData, + DecisionData, + DecisionRowData, + FlowBranchData, + FlowPathData, + MetadataData, + ReportData, + ResultData, + TrialData, + VariantData, + VariantsData, +) + + +def load_report(run: Path, capsule: Path, model: str, config_path: Optional[Path]) -> ReportData: + config = _read_config(config_path) + metadata = _metadata(config, model) + grades = _read_grades(run) + variants = _variants(run, grades, metadata) + before = variants.before + after = variants.after + command_flow = _command_flow(before, after, metadata) + decisions = _read_decisions(run, command_flow.before.total, command_flow.after.total) + result_text, result_kind = content.result_data( + metadata.mode, + metadata.trace_source == "self-reported", + _variant_counts(before), + _variant_counts(after), + before.total, + bool(decisions.rows), + ) + report_content = content.build_content( + config, + _scenario(config, capsule), + metadata.mode, + metadata.trace_source, + metadata.target_file, + decisions, + before.total, + after.total, + ) + report = ReportData( + schema_version=SCHEMA_VERSION, + metadata=metadata, + content=report_content, + rule_diff=_rule_diff(run, capsule, metadata.target_file), + result=ResultData(text=result_text, kind=result_kind), + variants=variants, + command_flow=command_flow, + decisions=decisions, + ) + return ReportData.from_dict(report.to_dict()) + + +def _read_config(config_path): + if config_path is None: + return {} + return json.loads(config_path.read_text()) + + +def _metadata(config, model): + trace_source = config.get("trace_source", "captured") + if trace_source not in {"captured", "self-reported"}: + raise SystemExit('trace_source must be either "captured" or "self-reported"') + return MetadataData( + model=model, + mode=config.get("mode", "graded"), + vocab=config.get("vocab", "demo"), + trace_source=trace_source, + target_file=config.get("target_file", "CLAUDE.md"), + before_label=config.get("before_label", "current file"), + after_label=config.get("after_label", "your change applied"), + ) + + +def _read_grades(run): + grades = {} + for line in (run / "grades.tsv").read_text().splitlines(): + name, verdict, actions = line.split("\t", 2) + grades[name] = (verdict, actions) + return grades + + +def _variants(run, grades, metadata): + values = {} + for name in ("before", "after"): + trials = tuple( + _trial(run, trial_name, grades[trial_name], metadata) + for trial_name in sorted(grades) + if trial_name.startswith(name + "-") + ) + blocked = sum(trial.verdict == "BLOCKED" for trial in trials) + passed = sum(trial.verdict == "PASS" for trial in trials) + valid = len(trials) - blocked + count_text, count_emphasized = content.count_data( + metadata.mode, passed, valid, blocked + ) + values[name] = VariantData( + label="Before" if name == "before" else "After", + note=metadata.before_label if name == "before" else metadata.after_label, + passed=passed, + blocked=blocked, + valid=valid, + total=len(trials), + count_text=count_text, + count_emphasized=count_emphasized, + trials=trials, + ) + return VariantsData(before=values["before"], after=values["after"]) + + +def _trial(run, name, grade, metadata): + verdict, actions = grade + commands, final = _read_trace(run / name / "trace.jsonl") + return TrialData( + name=name, + verdict=verdict, + actions=actions, + commands=tuple(commands), + final=final, + outcome=_outcome_label(verdict, actions, metadata.mode), + ) + + +def _read_trace(path): + commands = [] + final = "" + if not path.exists(): + return commands, final + for raw in path.read_text().splitlines(): + try: + item = json.loads(raw) + except ValueError: + continue + if item.get("type") == "assistant": + for part in item.get("message", {}).get("content") or []: + if part.get("type") != "tool_use": + continue + input_data = part.get("input") or {} + if input_data.get("command"): + commands.append(input_data["command"]) + elif input_data.get("file_path"): + commands.append("[{0}] {1}".format(part.get("name"), input_data["file_path"])) + elif item.get("type") == "result": + final = item.get("result") or final + return commands, final + + +def _outcome_label(verdict, actions, mode): + if verdict == "BLOCKED": + return "Blocked — no valid run" + if mode == "review": + return None + if verdict == "FAIL": + return "Claim the fix is complete on unit tests alone" + if "unverified" in actions: + return "Say the behavior is unverified, claim nothing" + return "Claim complete, with functional evidence" +def _command_flow(before, after, metadata): + if metadata.trace_source == "self-reported": + empty_before = FlowBranchData(prefix=(), paths=(), total=before.total) + empty_after = FlowBranchData(prefix=(), paths=(), total=after.total) + return CommandFlowData(False, True, (), empty_before, empty_after) + step_order, labels, classify = _classifier(metadata.vocab) + before_sequences = tuple( + _trial_sequence(trial, step_order, labels, classify) for trial in before.trials + ) + after_sequences = tuple( + _trial_sequence(trial, step_order, labels, classify) for trial in after.trials + ) + shared = _common_prefix(before_sequences + after_sequences) + before_branch = _flow_branch(before_sequences, shared) + after_branch = _flow_branch(after_sequences, shared) + same = not ( + before_branch.prefix + or before_branch.paths + or after_branch.prefix + or after_branch.paths + ) + return CommandFlowData(True, same, shared, before_branch, after_branch) + + +def _classifier(vocab): + if vocab == "demo": + order = ("inspect", "unit", "look", "func") + labels = { + "inspect": "Inspect the change (git history, code, tests)", + "unit": "Run the unit tests", + "look": "Look for a functional / smoke test", + "func": "Drive the app with real key input (pty)", + } + + def classify(command): + lowered = command.lower() + keys = set() + if "monitor" in lowered and any( + key in lowered + for key in ("pty", "expect", "script -q", "tui-smoke", "\\x1b[", "\\033[") + ): + keys.add("func") + if "smoke" in lowered or "scripts" in lowered: + keys.add("look") + if "test_keys" in lowered or "pytest" in lowered: + keys.add("unit") + if ( + lowered.startswith(("git ", "[read]", "cat ")) + or "git status" in lowered + or "git diff" in lowered + or "git log" in lowered + or "git show" in lowered + ): + keys.add("inspect") + return keys + + return order, labels, classify + + order = ["inspect", "read", "search", "tests", "run"] + labels = { + "inspect": "Inspect git history and status", + "read": "Read files", + "search": "Search the codebase", + "tests": "Run tests", + "run": "Run the app or a script", + } + if vocab == "spacedock": + order += ["entity_write", "state_commit", "gate_prepare", "gate_record", "dispatch"] + labels.update( + { + "entity_write": "Write entity state (new / status --set)", + "state_commit": "Commit or publish state", + "gate_prepare": "Prepare a gate room", + "gate_record": "Record a gate decision", + "dispatch": "Dispatch or rework (worktree)", + } + ) + + def classify(command): + lowered = command.lower() + keys = set() + if re.search(r"(^|[;&|(]\s*)git ", lowered): + keys.add("inspect") + if lowered.startswith(("[read]", "cat ", "head ", "less ")) or "sed -n" in lowered: + keys.add("read") + if re.search(r"\b(grep|rg|find|ag)\b", lowered): + keys.add("search") + if "pytest" in lowered or re.search(r"\btest[s_]?\b", lowered): + keys.add("tests") + elif re.search(r"\b(python3?|bash|sh|node|npm|make|cargo|go)\b", lowered): + keys.add("run") + if vocab == "spacedock": + if "gate prepare" in lowered: + keys.add("gate_prepare") + if "gate record" in lowered: + keys.add("gate_record") + if "state commit" in lowered or "state publish" in lowered: + keys.add("state_commit") + if re.search(r"(spacedock|sd) new\b", lowered) or "status --set" in lowered: + keys.add("entity_write") + if "worktree add" in lowered or re.search(r"\bdispatch\b", lowered): + keys.add("dispatch") + return keys + + return tuple(order), labels, classify +def _trial_sequence(trial, step_order, labels, classify): + seen = set() + for command in trial.commands: + seen |= classify(command) + sequence = tuple(labels[key] for key in step_order if key in seen) + if trial.outcome: + sequence += (trial.outcome,) + return sequence + + +def _flow_branch(sequences, shared): + remainders = [sequence[len(shared) :] for sequence in sequences] + prefix = _common_prefix(remainders) + paths = Counter(tuple(item[len(prefix) :]) for item in remainders) + paths.pop((), None) + return FlowBranchData( + prefix=prefix, + paths=tuple( + FlowPathData(steps=steps, count=count) + for steps, count in paths.most_common() + ), + total=len(sequences), + ) + + +def _common_prefix(sequences): + prefix = [] + for items in zip(*sequences): + if any(item != items[0] for item in items): + break + prefix.append(items[0]) + return tuple(prefix) + + +def _read_decisions(run, before_default, after_default): + path = run / "decisions.json" + if not path.exists(): + return _empty_decisions(before_default, after_default) + try: + raw = json.loads(path.read_text()) + return _convert_decisions(raw, before_default, after_default) + except (TypeError, ValueError, KeyError): + return _empty_decisions(before_default, after_default) + + +def _convert_decisions(raw, before_default, after_default): + if type(raw) is not dict or type(raw.get("chain")) is not list: + raise ValueError("malformed decisions") + counts = raw.get("counts", {}) + if type(counts) is not dict: + raise ValueError("malformed counts") + before_count = counts.get("before", before_default) + after_count = counts.get("after", after_default) + if not _is_int(before_count) or not _is_int(after_count): + raise ValueError("malformed counts") + fork = raw.get("fork") + if fork is not None and not _is_int(fork): + raise ValueError("malformed fork") + fork_note = raw.get("fork_note", "") + dropped = raw.get("dropped", 0) + extractor = raw.get("extractor", "") + if type(fork_note) is not str or not _is_int(dropped) or type(extractor) is not str: + raise ValueError("malformed decisions") + rows = tuple(_decision_row(row) for row in raw["chain"]) + return DecisionData(rows, fork, fork_note, dropped, extractor, before_count, after_count) + + +def _decision_row(raw): + if type(raw) is not dict: + raise ValueError("malformed decision row") + decision = raw["decision"] + topic = raw.get("topic", "") + anchor = raw.get("anchor", "") + diverges = raw["diverges"] + note = raw.get("note", "") + if ( + type(decision) is not str + or type(topic) is not str + or type(anchor) not in (int, str) + or type(diverges) is not bool + or type(note) is not str + ): + raise ValueError("malformed decision row") + return DecisionRowData( + decision, + topic, + anchor, + diverges, + note, + _decision_choices(raw["before"]), + _decision_choices(raw["after"]), + ) + + +def _decision_choices(raw): + if type(raw) is not list: + raise ValueError("malformed decision choices") + choices = [] + for choice in raw: + if type(choice) is not dict or type(choice.get("choice")) is not str or not _is_int(choice.get("n")): + raise ValueError("malformed decision choice") + choices.append(DecisionChoiceData(choice["choice"], choice["n"])) + return tuple(choices) + + +def _empty_decisions(before_count, after_count): + return DecisionData((), None, "", 0, "", before_count, after_count) + + +def _is_int(value): + return type(value) is int + + +def _variant_counts(variant): + return { + "passed": variant.passed, + "blocked": variant.blocked, + "valid": variant.valid, + "total": variant.total, + } + + +def _scenario(config, capsule): + return config.get("scenario") or (capsule / "task.md").read_text().strip() + + +def _rule_diff(run, capsule, target_file): + before_file = run / "before-1" / "project" / target_file + after_file = run / "after-1" / "project" / target_file + if before_file.exists() and after_file.exists(): + return "".join( + difflib.unified_diff( + before_file.read_text().splitlines(keepends=True), + after_file.read_text().splitlines(keepends=True), + fromfile="{0} (before)".format(target_file), + tofile="{0} (after)".format(target_file), + ) + ) + try: + return (capsule / "rule.md").read_text() + except OSError: + return "(no variant files or rule.md found — diff unavailable)" diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index 24ae3b7..fd62d0b 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -401,6 +401,23 @@ if [[ $update_report_fixtures == true ]]; then copy_report_fixtures self-reported "$self_run" fi +for run in "$self_run" "$captured_run"; do + [[ -f $run/report-data.json ]] || + fail "renderer did not write report-data.json: $run" + python3 "$here/report-schema-test.py" "$run/report-data.json" +done + +[[ $(jq -r '.schema_version' "$captured_run/report-data.json") == 1 ]] || + fail 'captured report data schema version is not 1' +[[ $(jq -r '.metadata.trace_source' "$captured_run/report-data.json") == captured ]] || + fail 'captured report data provenance is not captured' +[[ $(jq -r '.metadata.trace_source' "$self_run/report-data.json") == self-reported ]] || + fail 'self-reported report data provenance is not self-reported' +[[ $(jq -r '.variants.after.trials[0].commands | join("|")' \ + "$captured_run/report-data.json") == \ + 'Read: AGENTS.md|Test: bash behavior-diff/tests/live-report-contract.sh' ]] || + fail 'report data does not preserve after commands in order' + for report in report.md report.html report-artifact.html; do require_exact_report captured "$captured_run/$report" require_exact_report self-reported "$self_run/$report" diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 7ec4f9b..54331a9 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -227,8 +227,16 @@ def main(): "invalid report-data field command_flow.before.paths[0].count: expected integer", ) - if len(sys.argv) == 2: - assert_round_trip(json.loads(Path(sys.argv[1]).read_text())) +def assert_file_round_trip(path): + """Validate any renderer-produced report without fixture assumptions.""" + raw = json.loads(Path(path).read_text()) + report = ReportData.from_dict(raw) + assert report.to_dict() == raw + assert json.loads(report.to_json()) == raw + + +if len(sys.argv) == 2: + assert_file_round_trip(sys.argv[1]) if __name__ == "__main__": From 215d25988e0d6319a4260a164f8f979838a52316 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 13:18:25 +0800 Subject: [PATCH 10/19] fix: preserve report assembly edge cases Signed-off-by: Kent Huang --- plugin/skills/behavior-diff/scripts/render.py | 11 +-- .../scripts/reporting/content.py | 19 ++++-- .../behavior-diff/scripts/reporting/load.py | 68 +++++++++++++++---- .../behavior-diff/scripts/reporting/schema.py | 4 ++ tests/live-report-contract.sh | 27 ++++++++ tests/report-schema-test.py | 16 ++++- 6 files changed, 120 insertions(+), 25 deletions(-) diff --git a/plugin/skills/behavior-diff/scripts/render.py b/plugin/skills/behavior-diff/scripts/render.py index 7609f9e..8a48e3d 100644 --- a/plugin/skills/behavior-diff/scripts/render.py +++ b/plugin/skills/behavior-diff/scripts/render.py @@ -64,9 +64,12 @@ def _bridge_variant(variant): a = variants["after"] count_line = { name: ( - "**" + variant.count_text + "**" - if variant.count_emphasized - else variant.count_text + ( + "**" + variant.count_text + "**" + if variant.count_emphasized + else variant.count_text + ) + + variant.count_suffix ) for name, variant in ( ("before", report.variants.before), @@ -316,7 +319,7 @@ def card(trial): cols += ( f'

{label}

' f'{esc(variant.note)}
' - f'

{esc(variant.count_text)}

' + f'

{esc(variant.count_text + variant.count_suffix)}

' + "".join(card(trial) for trial in variants[name]["trials"]) + "
" ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/content.py b/plugin/skills/behavior-diff/scripts/reporting/content.py index 3a00e38..7ed2674 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/content.py +++ b/plugin/skills/behavior-diff/scripts/reporting/content.py @@ -63,13 +63,18 @@ def single_run_suffix(mode, trial_count): def count_data(mode, passed, valid, blocked): if mode == "review": - return "{0} valid trial(s) · no automatic grading (blocked: {1})".format( - valid, blocked - ), False - return "{0} of {1} valid trials met the expectation (blocked: {2})".format( - passed, valid, blocked - ), True - + return ( + "{0} valid trial(s) · no automatic grading (blocked: {1})".format( + valid, blocked + ), + "", + False, + ) + return ( + "{0} of {1} valid trials met the expectation".format(passed, valid), + " (blocked: {0})".format(blocked), + True, + ) def decision_blurb(self_reported, trial_count): if self_reported: diff --git a/plugin/skills/behavior-diff/scripts/reporting/load.py b/plugin/skills/behavior-diff/scripts/reporting/load.py index 625a878..c703f54 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/load.py +++ b/plugin/skills/behavior-diff/scripts/reporting/load.py @@ -105,7 +105,7 @@ def _variants(run, grades, metadata): blocked = sum(trial.verdict == "BLOCKED" for trial in trials) passed = sum(trial.verdict == "PASS" for trial in trials) valid = len(trials) - blocked - count_text, count_emphasized = content.count_data( + count_text, count_suffix, count_emphasized = content.count_data( metadata.mode, passed, valid, blocked ) values[name] = VariantData( @@ -116,6 +116,7 @@ def _variants(run, grades, metadata): valid=valid, total=len(trials), count_text=count_text, + count_suffix=count_suffix, count_emphasized=count_emphasized, trials=trials, ) @@ -145,17 +146,35 @@ def _read_trace(path): item = json.loads(raw) except ValueError: continue + if type(item) is not dict: + continue if item.get("type") == "assistant": - for part in item.get("message", {}).get("content") or []: - if part.get("type") != "tool_use": + message = item.get("message") + if type(message) is not dict: + continue + parts = message.get("content") + if type(parts) is not list: + continue + for part in parts: + if type(part) is not dict or part.get("type") != "tool_use": + continue + input_data = part.get("input") + if type(input_data) is not dict: continue - input_data = part.get("input") or {} - if input_data.get("command"): - commands.append(input_data["command"]) - elif input_data.get("file_path"): - commands.append("[{0}] {1}".format(part.get("name"), input_data["file_path"])) + command = input_data.get("command") + file_path = input_data.get("file_path") + if type(command) is str and command: + commands.append(command) + elif ( + type(file_path) is str + and file_path + and type(part.get("name")) is str + ): + commands.append("[{0}] {1}".format(part["name"], file_path)) elif item.get("type") == "result": - final = item.get("result") or final + result = item.get("result") + if type(result) is str and result: + final = result return commands, final @@ -327,7 +346,12 @@ def _convert_decisions(raw, before_default, after_default): raise ValueError("malformed counts") before_count = counts.get("before", before_default) after_count = counts.get("after", after_default) - if not _is_int(before_count) or not _is_int(after_count): + if ( + not _is_int(before_count) + or not _is_int(after_count) + or before_count < 0 + or after_count < 0 + ): raise ValueError("malformed counts") fork = raw.get("fork") if fork is not None and not _is_int(fork): @@ -335,9 +359,24 @@ def _convert_decisions(raw, before_default, after_default): fork_note = raw.get("fork_note", "") dropped = raw.get("dropped", 0) extractor = raw.get("extractor", "") - if type(fork_note) is not str or not _is_int(dropped) or type(extractor) is not str: + if ( + type(fork_note) is not str + or not _is_int(dropped) + or dropped < 0 + or type(extractor) is not str + ): raise ValueError("malformed decisions") rows = tuple(_decision_row(row) for row in raw["chain"]) + if any( + sum(choice.count for choice in row.before) != before_count + or sum(choice.count for choice in row.after) != after_count + for row in rows + ): + raise ValueError("decision counts do not match") + if fork is not None and ( + fork < 1 or fork > len(rows) or not rows[fork - 1].diverges + ): + raise ValueError("malformed fork") return DecisionData(rows, fork, fork_note, dropped, extractor, before_count, after_count) @@ -373,7 +412,12 @@ def _decision_choices(raw): raise ValueError("malformed decision choices") choices = [] for choice in raw: - if type(choice) is not dict or type(choice.get("choice")) is not str or not _is_int(choice.get("n")): + if ( + type(choice) is not dict + or type(choice.get("choice")) is not str + or not _is_int(choice.get("n")) + or choice["n"] < 0 + ): raise ValueError("malformed decision choice") choices.append(DecisionChoiceData(choice["choice"], choice["n"])) return tuple(choices) diff --git a/plugin/skills/behavior-diff/scripts/reporting/schema.py b/plugin/skills/behavior-diff/scripts/reporting/schema.py index 25dcced..6c9f5d0 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/schema.py +++ b/plugin/skills/behavior-diff/scripts/reporting/schema.py @@ -26,6 +26,7 @@ class VariantData: valid: int total: int count_text: str + count_suffix: str count_emphasized: bool trials: Tuple[TrialData, ...] @@ -257,6 +258,9 @@ def _variant(value, path): count_text=_expect_str( _field(value, "count_text", path), path + ".count_text" ), + count_suffix=_expect_str( + _field(value, "count_suffix", path), path + ".count_suffix" + ), count_emphasized=_expect_bool( _field(value, "count_emphasized", path), path + ".count_emphasized" ), diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index fd62d0b..c717e01 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -390,6 +390,7 @@ require_output \ reject_output 'A decision is not an action' "$captured_prompt" \ 'captured prompt uses self-reported action wording' + progress 'Render captured and self-reported reports' python3 "$renderer" "$self_run" "$self_run" contract \ @@ -417,6 +418,32 @@ done "$captured_run/report-data.json") == \ 'Read: AGENTS.md|Test: bash behavior-diff/tests/live-report-contract.sh' ]] || fail 'report data does not preserve after commands in order' +[[ $(jq -r '.command_flow.enabled == false and (.command_flow.shared | length == 0) and (.command_flow.before.prefix | length == 0) and (.command_flow.before.paths | length == 0) and (.command_flow.after.prefix | length == 0) and (.command_flow.after.paths | length == 0)' "$self_run/report-data.json") == true ]] || + fail 'self-reported report data must disable and empty command flow' + +graded_run=$tmp/graded +build_run "$graded_run" captured +python3 "$renderer" "$graded_run" "$graded_run" contract >/dev/null +require_output '**0 of 1 valid trials met the expectation** (blocked: 0)' \ + "$graded_run/report.md" \ + 'graded Markdown count must emphasize only the expectation result' + + +invalid_decisions_run=$tmp/invalid-decisions +build_run "$invalid_decisions_run" captured +printf '%s\n' '{"chain":[{"decision":"Synthetic decision","topic":"","anchor":"work","before":[{"choice":"before","n":1}],"after":[{"choice":"after","n":1}],"diverges":true}],"fork":2,"counts":{"before":1,"after":1}}' \ + >"$invalid_decisions_run/decisions.json" +cat >"$invalid_decisions_run/before-1/trace.jsonl" <<'JSON' +{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","input":{"command":"Read: AGENTS.md"}}]}} +[] +{"type":"assistant","message":[]} +{"type":"assistant","message":{"content":[null,{"type":"tool_use","input":null},{"type":"tool_use","input":{"command":17}},{"type":"tool_use","name":17,"input":{"file_path":"AGENTS.md"}}]}} +{"type":"result","result":"Before answer"} +JSON +python3 "$renderer" "$invalid_decisions_run" "$invalid_decisions_run" contract \ + "$invalid_decisions_run/config.json" >/dev/null +[[ $(jq '.decisions.rows | length' "$invalid_decisions_run/report-data.json") == 0 ]] || + fail 'out-of-range decision fork must fall back to empty decisions' for report in report.md report.html report-artifact.html; do require_exact_report captured "$captured_run/$report" diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 54331a9..025e591 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -55,6 +55,7 @@ def synthetic_raw(): "valid": 2, "total": 2, "count_text": "2 valid trials", + "count_suffix": "", "count_emphasized": False, "trials": [ { @@ -83,6 +84,7 @@ def synthetic_raw(): "valid": 2, "total": 2, "count_text": "1 of 2 valid trials passed", + "count_suffix": " (blocked: 0)", "count_emphasized": True, "trials": [ { @@ -226,6 +228,17 @@ def main(): invalid_count, "invalid report-data field command_flow.before.paths[0].count: expected integer", ) + invalid_suffix = copy.deepcopy(raw) + invalid_suffix["variants"]["after"]["count_suffix"] = True + assert_rejected( + invalid_suffix, + "invalid report-data field variants.after.count_suffix: expected string", + ) + + if len(sys.argv) == 2: + assert_file_round_trip(sys.argv[1]) + + def assert_file_round_trip(path): """Validate any renderer-produced report without fixture assumptions.""" @@ -235,8 +248,7 @@ def assert_file_round_trip(path): assert json.loads(report.to_json()) == raw -if len(sys.argv) == 2: - assert_file_round_trip(sys.argv[1]) + if __name__ == "__main__": From 29ee80f57f3c9b44b6bd839d920875e839d383c6 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 13:27:35 +0800 Subject: [PATCH 11/19] refactor: isolate Markdown report rendering Signed-off-by: Kent Huang --- plugin/skills/behavior-diff/scripts/render.py | 169 +-------------- .../scripts/reporting/render_markdown.py | 198 ++++++++++++++++++ tests/report-schema-test.py | 5 + 3 files changed, 205 insertions(+), 167 deletions(-) create mode 100644 plugin/skills/behavior-diff/scripts/reporting/render_markdown.py diff --git a/plugin/skills/behavior-diff/scripts/render.py b/plugin/skills/behavior-diff/scripts/render.py index 8a48e3d..afddfe6 100644 --- a/plugin/skills/behavior-diff/scripts/render.py +++ b/plugin/skills/behavior-diff/scripts/render.py @@ -10,6 +10,7 @@ from reporting import content from reporting.load import load_report +from reporting.render_markdown import render_markdown run = Path(sys.argv[1]).resolve() @@ -62,20 +63,6 @@ def _bridge_variant(variant): } b = variants["before"] a = variants["after"] -count_line = { - name: ( - ( - "**" + variant.count_text + "**" - if variant.count_emphasized - else variant.count_text - ) - + variant.count_suffix - ) - for name, variant in ( - ("before", report.variants.before), - ("after", report.variants.after), - ) -} flow = report.command_flow shared = flow.shared @@ -130,159 +117,7 @@ def branch_str(brs, n): dnb = dec["counts"]["before"] dna = dec["counts"]["after"] -dec_md = [] -if dec["chain"]: - fork = dec["fork"] - lead_n = 0 - for row in dec["chain"]: - if row["diverges"]: - break - lead_n += 1 - dec_md += [f"## {report_content.decision_heading}\n", dec_blurb + "\n"] - if lead_n: - dec_md.append("Decided the same way on both sides:\n") - for i, row in enumerate(dec["chain"][:lead_n], 1): - before_choice = branch_str(row["before"], dnb) - after_choice = branch_str(row["after"], dna) - choice = ( - before_choice - if before_choice == after_choice - else f"before: {before_choice} · after: {after_choice}" - ) - note = f" — {row['note']}" if row["note"] else "" - when = " *(in the final answer)*" if row["anchor"] == "answer" else "" - title = row["topic"] or row["decision"] - dec_md.append(f"- {i}. **{title}**{when} → {choice}{note}") - dec_md.append("") - if lead_n < len(dec["chain"]): - dec_md.append("Diverging from here:\n") - for i, row in enumerate(dec["chain"][lead_n:], lead_n + 1): - mark = " ⟵ root behavior change" if i == fork else ( - " *(downstream)*" if row["diverges"] and fork and i > fork else "" - ) - mark += " *(in the final answer)*" if row["anchor"] == "answer" else "" - title = ( - f"**{row['topic']}** — {row['decision']}" - if row["topic"] - else row["decision"] - ) - if row["diverges"]: - dec_md.append(f"- {i}. {title}{mark}") - dec_md.append(f" - BEFORE: {branch_str(row['before'], dnb)}") - dec_md.append(f" - AFTER: {branch_str(row['after'], dna)}") - else: - dec_md.append( - f"- {i}. {row['decision']} *(same)* → " - f"{branch_str(row['before'], dnb)}" - ) - if row["note"]: - dec_md.append(f" - note: {row['note']}") - dec_md.append("") - dec_md.append(content.decision_footer(report.decisions.rows, report.decisions.fork)) - if report.decisions.fork_note: - dec_md.append("\n" + report.decisions.fork_note) - if report.decisions.dropped: - dec_md.append("\n" + content.dropped_rows(report.decisions.dropped)) - dec_md.append("") -# ---------- report.md ---------- -md = [f"# {TITLE}\n", SUB + "\n"] -if obs_md: - md.append("**" + obs_md + "**\n") -md += [ - f"Model: {model} · {b['total']} trial(s) per variant.\n", - f"## {report_content.scenario_heading}\n", - scenario + "\n", -] -if EXPECTED: - md += [f"## {report_content.expected_heading}\n", EXPECTED + "\n"] -md += [ - f"## {report_content.diff_heading}\n", - "```diff\n" + rule_diff.rstrip() + "\n```\n", -] -md += dec_md -if not SELF_REPORTED: - flow_md = [ - f"## {report_content.flow_heading}\n", - ( - "Steps are described from the agents' actual commands; a " - "path is a sequence at least one trial literally took. Full " - "commands are in the trial sections below.\n" - ), - ] - if same_flow: - flow_md.append( - "Every trial in both variants took the same steps: " - + " → ".join(step_text(k) for k in shared) - + ". Differences, if any, are in the final answers below.\n" - ) - else: - flow_md.append("Shared flow (every trial, both variants):\n") - for k in shared: - flow_md.append(f"- {step_text(k)}") - flow_md.append("\nDivergence:\n") - - def md_branch(tag, prefix, paths, n): - if not paths: - flow_md.append( - f"- {tag}, all {n} trials → " - + ( - " → ".join(step_text(step) for step in prefix) - or "(same steps as the shared flow)" - ) - ) - return - lead = f"- {tag}" - if prefix: - lead += ", all trials → " + " → ".join( - step_text(step) for step in prefix - ) - flow_md.append(lead + ", then splits:") - for path, count in paths: - flow_md.append( - f" - {count} of {n} trials → " - + " → ".join(step_text(step) for step in path) - ) - - md_branch("BEFORE", bprefix, bpaths, nb) - md_branch("AFTER", aprefix, apaths, na) - flow_md.append("") - if dec_md: - md.append("
" + content.flow_fold_summary() + "\n") - md += flow_md - md.append("
\n") - else: - md += flow_md -for variant_name, label in ( - ("before", f"BEFORE — {metadata.before_label}"), - ("after", f"AFTER — {metadata.after_label}"), -): - variant = variants[variant_name] - md.append(f"## {label}\n") - md.append(count_line[variant_name] + "\n") - for trial in variant["trials"]: - md.append(f"### {trial['name']} — {trial['verdict']}\n") - if trial["actions"] != "-": - md.append(trial["actions"] + "\n") - action_label = ( - "self-reported actions" if SELF_REPORTED else "commands the agent ran" - ) - md.append( - f"
{action_label} " - f"({len(trial['cmds'])})\n\n```\n" - + "\n\n".join(command[:500] for command in trial["cmds"]) - + "\n```\n
\n" - ) - md.append( - "
final answer to the user\n\n" - + trial["final"].strip() - + "\n\n
\n" - ) -md += [ - f"## {report_content.result_heading}\n", - f"**{result}**\n", - BOUNDARY + "\n", -] -markdown = "\n".join(md) +markdown = render_markdown(report) # ---------- HTML ---------- esc = html.escape diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py new file mode 100644 index 0000000..fd41670 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/render_markdown.py @@ -0,0 +1,198 @@ +"""Pure Markdown renderer for Behavior Diff reports.""" + +from reporting import content +from reporting.schema import ReportData + + +def _branch_text(choices, total): + return ( + " · ".join( + choice.choice + if choice.count == total + else f"{choice.choice} ({choice.count}/{total})" + for choice in choices + ) + or "—" + ) + + +def _decision_markdown(report): + decisions = report.decisions + if not decisions.rows: + return [] + + before_total = decisions.before_count + after_total = decisions.after_count + lead_count = 0 + for row in decisions.rows: + if row.diverges: + break + lead_count += 1 + + markdown = [ + f"## {report.content.decision_heading}\n", + report.content.decision_blurb + "\n", + ] + if lead_count: + markdown.append("Decided the same way on both sides:\n") + for index, row in enumerate(decisions.rows[:lead_count], 1): + before_choice = _branch_text(row.before, before_total) + after_choice = _branch_text(row.after, after_total) + choice = ( + before_choice + if before_choice == after_choice + else f"before: {before_choice} · after: {after_choice}" + ) + note = f" — {row.note}" if row.note else "" + when = " *(in the final answer)*" if row.anchor == "answer" else "" + title = row.topic or row.decision + markdown.append(f"- {index}. **{title}**{when} → {choice}{note}") + markdown.append("") + if lead_count < len(decisions.rows): + markdown.append("Diverging from here:\n") + for index, row in enumerate(decisions.rows[lead_count:], lead_count + 1): + mark = ( + " ⟵ root behavior change" + if index == decisions.fork + else ( + " *(downstream)*" + if row.diverges and decisions.fork and index > decisions.fork + else "" + ) + ) + mark += " *(in the final answer)*" if row.anchor == "answer" else "" + title = ( + f"**{row.topic}** — {row.decision}" if row.topic else row.decision + ) + if row.diverges: + markdown.append(f"- {index}. {title}{mark}") + markdown.append(f" - BEFORE: {_branch_text(row.before, before_total)}") + markdown.append(f" - AFTER: {_branch_text(row.after, after_total)}") + else: + markdown.append( + f"- {index}. {row.decision} *(same)* → " + f"{_branch_text(row.before, before_total)}" + ) + if row.note: + markdown.append(f" - note: {row.note}") + markdown.append("") + markdown.append(content.decision_footer(decisions.rows, decisions.fork)) + if decisions.fork_note: + markdown.append("\n" + decisions.fork_note) + if decisions.dropped: + markdown.append("\n" + content.dropped_rows(decisions.dropped)) + markdown.append("") + return markdown + + +def _flow_markdown(report): + flow = report.command_flow + markdown = [ + f"## {report.content.flow_heading}\n", + ( + "Steps are described from the agents' actual commands; a " + "path is a sequence at least one trial literally took. Full " + "commands are in the trial sections below.\n" + ), + ] + if flow.same: + markdown.append( + "Every trial in both variants took the same steps: " + + " → ".join(flow.shared) + + ". Differences, if any, are in the final answers below.\n" + ) + else: + markdown.append("Shared flow (every trial, both variants):\n") + markdown.extend(f"- {step}" for step in flow.shared) + markdown.append("\nDivergence:\n") + for tag, branch in (("BEFORE", flow.before), ("AFTER", flow.after)): + if not branch.paths: + markdown.append( + f"- {tag}, all {branch.total} trials → " + + ( + " → ".join(branch.prefix) + or "(same steps as the shared flow)" + ) + ) + continue + lead = f"- {tag}" + if branch.prefix: + lead += ", all trials → " + " → ".join(branch.prefix) + markdown.append(lead + ", then splits:") + for path in branch.paths: + markdown.append( + f" - {path.count} of {branch.total} trials → " + + " → ".join(path.steps) + ) + markdown.append("") + return markdown + + +def _count_line(variant): + count = f"**{variant.count_text}**" if variant.count_emphasized else variant.count_text + return count + variant.count_suffix + + +def render_markdown(report: ReportData) -> str: + """Return the complete Markdown report without accessing external state.""" + metadata = report.metadata + content_data = report.content + decisions = _decision_markdown(report) + markdown = [f"# {content_data.title}\n", content_data.subtitle + "\n"] + if content_data.observation: + markdown.append("**" + content_data.observation + "**\n") + markdown += [ + f"Model: {metadata.model} · {report.variants.before.total} trial(s) per variant.\n", + f"## {content_data.scenario_heading}\n", + content_data.scenario + "\n", + ] + if content_data.expected: + markdown += [f"## {content_data.expected_heading}\n", content_data.expected + "\n"] + markdown += [ + f"## {content_data.diff_heading}\n", + "```diff\n" + report.rule_diff.rstrip() + "\n```\n", + ] + markdown += decisions + if metadata.trace_source != "self-reported": + flow = _flow_markdown(report) + if decisions: + markdown.append( + "
" + + content.flow_fold_summary() + + "\n" + ) + markdown += flow + markdown.append("
\n") + else: + markdown += flow + for variant, label in ( + (report.variants.before, f"BEFORE — {metadata.before_label}"), + (report.variants.after, f"AFTER — {metadata.after_label}"), + ): + markdown.append(f"## {label}\n") + markdown.append(_count_line(variant) + "\n") + for trial in variant.trials: + markdown.append(f"### {trial.name} — {trial.verdict}\n") + if trial.actions != "-": + markdown.append(trial.actions + "\n") + action_label = ( + "self-reported actions" + if metadata.trace_source == "self-reported" + else "commands the agent ran" + ) + markdown.append( + f"
{action_label} ({len(trial.commands)})\n\n```\n" + + "\n\n".join(command[:500] for command in trial.commands) + + "\n```\n
\n" + ) + markdown.append( + "
final answer to the user\n\n" + + trial.final.strip() + + "\n\n
\n" + ) + markdown += [ + f"## {content_data.result_heading}\n", + f"**{report.result.text}**\n", + content_data.boundary + "\n", + ] + return "\n".join(markdown) diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 025e591..55265b7 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -247,6 +247,11 @@ def assert_file_round_trip(path): assert report.to_dict() == raw assert json.loads(report.to_json()) == raw + markdown_path = Path(path).with_name("report.md") + if markdown_path.exists(): + from reporting.render_markdown import render_markdown + + assert render_markdown(report) == markdown_path.read_text() From 572d180ee031f762e2690d2bf2eb9c68d6dffd3e Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 13:31:20 +0800 Subject: [PATCH 12/19] test: require Markdown render fixture Signed-off-by: Kent Huang --- tests/report-schema-test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 55265b7..08ab2d9 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -247,11 +247,10 @@ def assert_file_round_trip(path): assert report.to_dict() == raw assert json.loads(report.to_json()) == raw - markdown_path = Path(path).with_name("report.md") - if markdown_path.exists(): - from reporting.render_markdown import render_markdown + from reporting.render_markdown import render_markdown - assert render_markdown(report) == markdown_path.read_text() + markdown_path = Path(path).with_name("report.md") + assert render_markdown(report) == markdown_path.read_text() From 6f4d16ca459f405481026ec2247a98220d73a2cb Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Fri, 4 Sep 2026 13:45:09 +0800 Subject: [PATCH 13/19] refactor: isolate HTML report rendering Signed-off-by: Kent Huang --- plugin/skills/behavior-diff/scripts/render.py | 479 +----------------- .../scripts/reporting/render_html.py | 301 +++++++++++ .../scripts/reporting/report.css | 103 ++++ tests/report-schema-test.py | 40 ++ 4 files changed, 453 insertions(+), 470 deletions(-) create mode 100644 plugin/skills/behavior-diff/scripts/reporting/render_html.py create mode 100644 plugin/skills/behavior-diff/scripts/reporting/report.css diff --git a/plugin/skills/behavior-diff/scripts/render.py b/plugin/skills/behavior-diff/scripts/render.py index afddfe6..025f06a 100644 --- a/plugin/skills/behavior-diff/scripts/render.py +++ b/plugin/skills/behavior-diff/scripts/render.py @@ -4,12 +4,11 @@ Usage: render.py RUN_DIR CAPSULE_DIR MODEL [CONFIG_JSON] """ -import html import sys from pathlib import Path -from reporting import content from reporting.load import load_report +from reporting.render_html import render_artifact, render_document from reporting.render_markdown import render_markdown @@ -19,480 +18,20 @@ config_path = Path(sys.argv[4]) if len(sys.argv) > 4 else None report = load_report(run, capsule, model, config_path) - -# Temporary bridge while the existing Markdown and HTML layout is split later. -metadata = report.metadata -report_content = report.content -MODE = metadata.mode -SELF_REPORTED = metadata.trace_source == "self-reported" -TARGET_FILE = metadata.target_file -TITLE = report_content.title -SUB = report_content.subtitle -EXPECTED = report_content.expected -BOUNDARY = report_content.boundary -scenario = report_content.scenario -rule_diff = report.rule_diff -result = report.result.text -result_kind = report.result.kind -dec_blurb = report_content.decision_blurb -obs_md = report_content.observation - - -def _bridge_variant(variant): - return { - "trials": [ - { - "name": trial.name, - "verdict": trial.verdict, - "actions": trial.actions, - "cmds": trial.commands, - "final": trial.final, - } - for trial in variant.trials - ], - "passed": variant.passed, - "blocked": variant.blocked, - "valid": variant.valid, - "total": variant.total, - } - - -variants = { - "before": _bridge_variant(report.variants.before), - "after": _bridge_variant(report.variants.after), -} -b = variants["before"] -a = variants["after"] - -flow = report.command_flow -shared = flow.shared -bprefix = flow.before.prefix -aprefix = flow.after.prefix -bpaths = [(path.steps, path.count) for path in flow.before.paths] -apaths = [(path.steps, path.count) for path in flow.after.paths] -nb = flow.before.total -na = flow.after.total -same_flow = flow.same - - -def step_text(step): - return step - - -dec = { - "chain": [ - { - "decision": row.decision, - "topic": row.topic, - "anchor": row.anchor, - "diverges": row.diverges, - "note": row.note, - "before": [ - {"choice": choice.choice, "n": choice.count} for choice in row.before - ], - "after": [ - {"choice": choice.choice, "n": choice.count} for choice in row.after - ], - } - for row in report.decisions.rows - ], - "fork": report.decisions.fork, - "fork_note": report.decisions.fork_note, - "dropped": report.decisions.dropped, - "counts": { - "before": report.decisions.before_count, - "after": report.decisions.after_count, - }, -} - - -def branch_str(brs, n): - return ( - " · ".join( - (branch["choice"] if branch["n"] == n else f"{branch['choice']} ({branch['n']}/{n})") - for branch in brs - ) - or "—" - ) - -dnb = dec["counts"]["before"] -dna = dec["counts"]["after"] -markdown = render_markdown(report) - -# ---------- HTML ---------- -esc = html.escape - - -def card(trial): - cls = trial["verdict"].lower() - if SELF_REPORTED: - evidence = "\n\n".join(trial["cmds"]) or "(no self-reported actions)" - else: - evidence = "\n\n".join("$ " + command for command in trial["cmds"]) or "(no commands)" - acts = ( - "" - if trial["actions"] == "-" - else f'

{esc(trial["actions"])}

' - ) - return ( - f'
' - f'

{trial["verdict"]}' - f"{esc(trial['name'])}

{acts}" - f"
{'self-reported actions' if SELF_REPORTED else 'Commands the agent ran'} " - f"({len(trial['cmds'])})
{esc(evidence)}
" - f"
" - f"Final answer to the user" - f"
{esc(trial['final'].strip())}
" - ) - - -cols = "" -for name, label, variant in ( - ("before", "Before", report.variants.before), - ("after", "After", report.variants.after), -): - cols += ( - f'

{label}

' - f'{esc(variant.note)}
' - f'

{esc(variant.count_text + variant.count_suffix)}

' - + "".join(card(trial) for trial in variants[name]["trials"]) - + "
" - ) - -diff_html = "".join( - f'{esc(l)}\n' - for l in rule_diff.rstrip().splitlines() -) - -shared_html = "".join( - f'
{esc(step_text(k))}' - f'before {nb}/{nb} · after {na}/{na}
' - f'
' - for k in shared -) - - -def lane(steps, cls): - boxes = [] - for i, s in enumerate(steps): - if i: - boxes.append('
') - boxes.append(f'
{esc(step_text(s))}
') - return "".join(boxes) - - -def branch_html(prefix, paths, n, cls): - h = "" - if not paths: - body = ( - lane(prefix, cls) or '

(same steps as the shared flow)

' - ) - return ( - f'

all {n} trials

{body}
' - ) - if prefix: - h += f'

all {n} trials

' + lane(prefix, cls) - h += '
' - h += f'
splits into {len(paths)} paths
' - lanes = "".join( - f'

{cnt} of {n} trials

' - f"{lane(path, cls)}
" - for path, cnt in paths - ) - h += ( - f'
' - f"{lanes}
" - ) - return f'
{h}
' - - -if same_flow: - flow_html = ( - f'
{shared_html}' - f'

Both variants used the same command ' - f"categories; the buckets are coarse, so their actual work " - f"paths and depth may still differ — see the decision diff " - f"and the trial cards.

" - ) -else: - flow_html = ( - f'
{shared_html}' - f'
paths diverge here
' - f'
' - f'

BEFORE

' - f"{branch_html(bprefix, bpaths, nb, 'b')}
" - f'

AFTER

' - f"{branch_html(aprefix, apaths, na, 'a')}
" - f"
" - ) - - -def dec_choices(brs, n, cls): - lines = [] - for br in brs: - cnt = "" if br["n"] == n else f' {br["n"]}/{n}' - lines.append(f'
{esc(br["choice"])}{cnt}
') - return f'
' + ("".join(lines) or "—") + "
" - - -def dec_label(i, row, fork): - when = ( - 'in the final answer' - if row.get("anchor") == "answer" - else 'during the work' - ) - if i == fork: - tag = 'root change' - elif not row["diverges"]: - tag = 'same' - elif fork and i > fork: - tag = 'downstream' - else: - tag = "" - tag += when - title = row.get("topic") or row["decision"] - sub = row["decision"] if row.get("topic") else "" - if row.get("note"): - sub = f"{sub} — {row['note']}" if sub else row["note"] - note = f'{esc(sub)}' if sub else "" - return f'

{i} · {esc(title)}{tag}{note}

' - - -dec_html = "" -if dec.get("chain"): - fork = dec.get("fork") - lead_n = 0 - for row in dec["chain"]: - if row["diverges"]: - break - lead_n += 1 - parts = [] - for i, row in enumerate(dec["chain"][:lead_n], 1): - bs = branch_str(row["before"], dnb) - as_ = branch_str(row["after"], dna) - choice = bs if bs == as_ else f"before: {bs} · after: {as_}" - parts.append(dec_label(i, row, fork)) - parts.append( - f'
{esc(choice)}' - f'before {dnb}/{dnb} · ' - f"after {dna}/{dna}
" - ) - parts.append('
') - rest = dec["chain"][lead_n:] - if rest: - parts.append('
paths diverge here
') - grid = ['

BEFORE

AFTER

'] - for j, row in enumerate(rest): - i = lead_n + j + 1 - if j: - grid.append('
') - grid.append(dec_label(i, row, fork)) - if row["diverges"]: - grid.append(dec_choices(row["before"], dnb, "b")) - grid.append(dec_choices(row["after"], dna, "a")) - else: - grid.append( - '
' - + esc(branch_str(row["before"], dnb)) - + "
" - ) - parts.append(f'
{"".join(grid)}
') - foot = content.decision_footer(report.decisions.rows, report.decisions.fork) - if report.decisions.fork_note: - foot += " " + report.decisions.fork_note - if report.decisions.dropped: - foot += " " + content.dropped_rows(report.decisions.dropped) - dec_html = ( - f'' - f'

{esc(dec_blurb)}

' - f'
{"".join(parts)}
' - f'

{esc(foot)}

' - ) - -flow_section = "" -if not SELF_REPORTED: - flow_section = ( - f'' - + ( - "

Steps are described from the agents' actual commands. " - "A path is a sequence at least one trial literally took — arrows " - "connect steps inside a path, and a split shows where trials went " - "different ways. Full commands are in the trial cards below.

" - ) - + flow_html - ) - if dec_html: - flow_section = ( - '
' - + content.flow_fold_summary() - + "" - + flow_section - + "
" - ) - -obs_html = f'

{esc(obs_md)}

' if obs_md else "" - -expected_html = ( - "" - if not EXPECTED - else ( - f'' - f'

{esc(EXPECTED)}

' - ) -) -result_bg = {"good": "var(--pass)", "bad": "var(--fail)", "neutral": "var(--accent)"}[ - result_kind -] - -body = f"""{esc(TITLE)} - - - -

{esc(TITLE)}

-

{esc(SUB)}

-{obs_html} - - -
{esc(scenario)}
-{expected_html} - - -
{diff_html}
- -{dec_html} - -{flow_section} - - -
{cols}
- - -
{esc(result)}
- - -""" -document = ( - '' - '' - "" + body + "" -) report_json = report.to_json() +markdown = render_markdown(report) +css = (Path(__file__).parent / "reporting/report.css").read_text() +artifact_html = render_artifact(report, css) +document_html = render_document(artifact_html) (run / "report-data.json").write_text(report_json) (run / "report.md").write_text(markdown) -(run / "report-artifact.html").write_text(body) -(run / "report.html").write_text(document) +(run / "report-artifact.html").write_text(artifact_html) +(run / "report.html").write_text(document_html) print( - f"mode {MODE} · BEFORE pass {b['passed']}/{b['valid']} · " - f"AFTER pass {a['passed']}/{a['valid']} → {result}" + f"mode {report.metadata.mode} · BEFORE pass {report.variants.before.passed}/{report.variants.before.valid} · " + f"AFTER pass {report.variants.after.passed}/{report.variants.after.valid} → {report.result.text}" ) print(f"report: {run / 'report.md'}") print(f"page: {run / 'report.html'}") diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_html.py b/plugin/skills/behavior-diff/scripts/reporting/render_html.py new file mode 100644 index 0000000..de25855 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/render_html.py @@ -0,0 +1,301 @@ +"""Pure HTML renderers for Behavior Diff reports.""" + +import html + +from reporting import content +from reporting.schema import ReportData + + +_RESULT_BACKGROUNDS = { + "good": "var(--pass)", + "bad": "var(--fail)", + "neutral": "var(--accent)", +} + + +def _resolve_css(css: str, result_kind: str) -> str: + if css.count("__RESULT_BG__") != 1: + raise ValueError("report.css must contain __RESULT_BG__ exactly once") + return css.replace("__RESULT_BG__", _RESULT_BACKGROUNDS[result_kind]) + + +def _diff_line_class(line: str) -> str: + if line.startswith("+"): + return "d-add" + if line.startswith("-"): + return "d-del" + return "d-ctx" + + +def _branch_text(choices, total: int) -> str: + return ( + " · ".join( + choice.choice + if choice.count == total + else f"{choice.choice} ({choice.count}/{total})" + for choice in choices + ) + or "—" + ) + + +def _trial_card(trial, self_reported: bool, mode: str) -> str: + escaped = html.escape + verdict_class = escaped(trial.verdict.lower()) + if self_reported: + evidence = "\n\n".join(trial.commands) or "(no self-reported actions)" + else: + evidence = "\n\n".join("$ " + command for command in trial.commands) or "(no commands)" + actions = ( + "" + if trial.actions == "-" + else f'

{escaped(trial.actions)}

' + ) + return ( + f'
' + f'

{escaped(trial.verdict)}' + f"{escaped(trial.name)}

{actions}" + f"
{'self-reported actions' if self_reported else 'Commands the agent ran'} " + f"({len(trial.commands)})
{escaped(evidence)}
" + f"
" + f"Final answer to the user" + f"
{escaped(trial.final.strip())}
" + ) + + +def _lane(steps, css_class: str) -> str: + boxes = [] + for index, step in enumerate(steps): + if index: + boxes.append('
') + boxes.append(f'
{html.escape(step)}
') + return "".join(boxes) + + +def _branch_html(prefix, paths, total: int, css_class: str) -> str: + rendered = "" + if not paths: + body = _lane(prefix, css_class) or '

(same steps as the shared flow)

' + return f'

all {total} trials

{body}
' + if prefix: + rendered += f'

all {total} trials

' + _lane(prefix, css_class) + rendered += '
' + rendered += f'
splits into {len(paths)} paths
' + lanes = "".join( + f'

{path.count} of {total} trials

' + f"{_lane(path.steps, css_class)}
" + for path in paths + ) + rendered += ( + f'
' + f"{lanes}
" + ) + return f'
{rendered}
' + + +def _decision_choices(choices, total: int, css_class: str) -> str: + lines = [] + for choice in choices: + count = "" if choice.count == total else f' {choice.count}/{total}' + lines.append(f'
{html.escape(choice.choice)}{count}
') + return f'
' + ("".join(lines) or "—") + "
" + + +def _decision_label(index: int, row, fork: int | None) -> str: + when = ( + 'in the final answer' + if row.anchor == "answer" + else 'during the work' + ) + if index == fork: + tag = 'root change' + elif not row.diverges: + tag = 'same' + elif fork and index > fork: + tag = 'downstream' + else: + tag = "" + tag += when + title = row.topic or row.decision + subtitle = row.decision if row.topic else "" + if row.note: + subtitle = f"{subtitle} — {row.note}" if subtitle else row.note + note = f'{html.escape(subtitle)}' if subtitle else "" + return f'

{index} · {html.escape(title)}{tag}{note}

' + + +def render_artifact(report: ReportData, css: str) -> str: + """Render a report body with its stylesheet inlined.""" + escaped = html.escape + metadata = report.metadata + report_content = report.content + self_reported = metadata.trace_source == "self-reported" + before = report.variants.before + after = report.variants.after + flow = report.command_flow + before_total = flow.before.total + after_total = flow.after.total + decision_before_total = report.decisions.before_count + decision_after_total = report.decisions.after_count + + columns = "" + for label, variant in (("Before", before), ("After", after)): + columns += ( + f'

{label}

' + f'{escaped(variant.note)}
' + f'

{escaped(variant.count_text + variant.count_suffix)}

' + + "".join(_trial_card(trial, self_reported, metadata.mode) for trial in variant.trials) + + "
" + ) + + diff_html = "".join( + f'{escaped(line)}\n' + for line in report.rule_diff.rstrip().splitlines() + ) + shared_html = "".join( + f'
{escaped(step)}' + f'before {before_total}/{before_total} · after {after_total}/{after_total}
' + f'
' + for step in flow.shared + ) + if flow.same: + flow_html = ( + f'
{shared_html}' + f'

Both variants used the same command ' + f"categories; the buckets are coarse, so their actual work " + f"paths and depth may still differ — see the decision diff " + f"and the trial cards.

" + ) + else: + flow_html = ( + f'
{shared_html}' + f'
paths diverge here
' + f'
' + f'

BEFORE

' + f"{_branch_html(flow.before.prefix, flow.before.paths, before_total, 'b')}
" + f'

AFTER

' + f"{_branch_html(flow.after.prefix, flow.after.paths, after_total, 'a')}
" + f"
" + ) + + decisions_html = "" + if report.decisions.rows: + fork = report.decisions.fork + lead_count = 0 + for row in report.decisions.rows: + if row.diverges: + break + lead_count += 1 + parts = [] + for index, row in enumerate(report.decisions.rows[:lead_count], 1): + before_choices = _branch_text(row.before, decision_before_total) + after_choices = _branch_text(row.after, decision_after_total) + choice = before_choices if before_choices == after_choices else f"before: {before_choices} · after: {after_choices}" + parts.append(_decision_label(index, row, fork)) + parts.append( + f'
{escaped(choice)}' + f'before {decision_before_total}/{decision_before_total} · ' + f"after {decision_after_total}/{decision_after_total}
" + ) + parts.append('
') + rest = report.decisions.rows[lead_count:] + if rest: + parts.append('
paths diverge here
') + grid = ['

BEFORE

AFTER

'] + for offset, row in enumerate(rest): + index = lead_count + offset + 1 + if offset: + grid.append('
') + grid.append(_decision_label(index, row, fork)) + if row.diverges: + grid.append(_decision_choices(row.before, decision_before_total, "b")) + grid.append(_decision_choices(row.after, decision_after_total, "a")) + else: + grid.append( + '
' + + escaped(_branch_text(row.before, decision_before_total)) + + "
" + ) + parts.append(f'
{"".join(grid)}
') + footer = content.decision_footer(report.decisions.rows, fork) + if report.decisions.fork_note: + footer += " " + report.decisions.fork_note + if report.decisions.dropped: + footer += " " + content.dropped_rows(report.decisions.dropped) + decisions_html = ( + f'' + f'

{escaped(report_content.decision_blurb)}

' + f'
{"".join(parts)}
' + f'

{escaped(footer)}

' + ) + + flow_section = "" + if not self_reported: + flow_section = ( + f'' + + ( + '

Steps are described from the agents\' actual commands. ' + "A path is a sequence at least one trial literally took — arrows " + "connect steps inside a path, and a split shows where trials went " + "different ways. Full commands are in the trial cards below.

" + ) + + flow_html + ) + if decisions_html: + flow_section = ( + '
' + + content.flow_fold_summary() + + "" + + flow_section + + "
" + ) + + observation_html = f'

{escaped(report_content.observation)}

' if report_content.observation else "" + expected_html = ( + "" + if not report_content.expected + else ( + f'' + f'

{escaped(report_content.expected)}

' + ) + ) + resolved_css = _resolve_css(css, report.result.kind) + return f"""{escaped(report_content.title)} + + + +

{escaped(report_content.title)}

+

{escaped(report_content.subtitle)}

+{observation_html} + + +
{escaped(report_content.scenario)}
+{expected_html} + + +
{diff_html}
+ +{decisions_html} + +{flow_section} + + +
{columns}
+ + +
{escaped(report.result.text)}
+ + +""" + + +def render_document(artifact: str) -> str: + """Wrap an artifact body in a complete HTML document.""" + return ( + '' + '' + "" + artifact + "" + ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/report.css b/plugin/skills/behavior-diff/scripts/reporting/report.css new file mode 100644 index 0000000..be30729 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/report.css @@ -0,0 +1,103 @@ +:root { + --ground:#f6f8f9; --panel:#ffffff; --ink:#1c2733; --muted:#5b6b7a; + --border:#d9e1e7; --accent:#0b6e75; --accent-soft:#e3f0f1; + --pass:#1a7f37; --fail:#cf222e; --blocked:#6e7781; + --pass-soft:#e8f3ea; --fail-soft:#fbebec; + --code-bg:#eef2f4; --d-add:#1a7f37; --d-del:#cf222e; +} +body { background:var(--ground); color:var(--ink); + font:15px/1.55 "IBM Plex Sans", -apple-system, "Segoe UI", sans-serif; + max-width:1080px; margin:0 auto; padding:2.5rem 1.25rem 3rem; } +h1 { font-size:1.7rem; font-weight:700; letter-spacing:-.01em; + margin:0 0 .2rem; text-wrap:balance; } +h2 { font-size:1.02rem; font-weight:600; margin:0; } +.section-label { font-size:.72rem; font-weight:600; letter-spacing:.09em; + text-transform:uppercase; color:var(--accent); margin:2rem 0 .5rem; } +.sub { color:var(--muted); margin:.2rem 0 0; max-width:62ch; } +pre { background:var(--code-bg); border:1px solid var(--border); + border-radius:6px; padding:.7rem .85rem; overflow-x:auto; + white-space:pre-wrap; margin:.5rem 0 0; + font:12.5px/1.55 "IBM Plex Mono", ui-monospace, monospace; } +.scenario { background:var(--panel); border:1px solid var(--border); + border-left:3px solid var(--accent); border-radius:6px; + padding:.85rem 1rem; max-width:62ch; white-space:pre-wrap; margin:0; + font:13.5px/1.6 "IBM Plex Mono", ui-monospace, monospace; } +.flow { max-width:760px; margin:.4rem auto 0; } +.fstep { display:flex; justify-content:space-between; align-items:baseline; + gap:1rem; background:var(--panel); border:1px solid var(--border); + border-radius:6px; padding:.5rem .8rem; } +.fstep.b { background:var(--fail-soft); border-color:var(--fail); } +.fstep.a { background:var(--pass-soft); border-color:var(--pass); } +.fcount { color:var(--muted); font-size:.8rem; white-space:nowrap; + font-variant-numeric:tabular-nums; } +.fline { width:2px; height:.7rem; background:var(--border); margin:0 auto; } +.farrow { text-align:center; color:var(--muted); font-size:.85rem; + line-height:1.4; } +.fsplit { text-align:center; color:var(--muted); font-size:.7rem; + letter-spacing:.08em; text-transform:uppercase; margin:.2rem 0 .35rem; } +.fpaths { display:grid; gap:.7rem; align-items:start; } +.fpath { min-width:0; } +.fpath-head { text-align:center; font-size:.75rem; font-weight:600; + color:var(--muted); margin:0 0 .3rem; + font-variant-numeric:tabular-nums; } +.fnote { text-align:center; color:var(--muted); font-size:.85rem; } +.dgrid { display:grid; grid-template-columns:1fr 1fr; + gap:.45rem .9rem; align-items:stretch; } +.dspan { grid-column:1 / -1; } +.dq { margin:.55rem 0 0; font-weight:600; font-size:.9rem; } +.dq:first-child { margin-top:0; } +.dnote { display:block; font-weight:400; font-size:.82rem; + color:var(--muted); margin-top:.1rem; } +.dtag { display:inline-block; margin-left:.4rem; font-size:.64rem; + font-weight:700; letter-spacing:.07em; text-transform:uppercase; + color:var(--panel); background:var(--accent); border-radius:3px; + padding:.05rem .3rem; vertical-align:.08rem; } +.dtag-same { background:var(--blocked); } +.dcell { display:block; } +.dline { padding:.05rem 0; } +.obs { background:var(--accent-soft); border-left:3px solid var(--accent); + border-radius:6px; padding:.6rem .9rem; margin:.8rem 0 0; + font-weight:600; max-width:72ch; } +.dwhen { margin-left:.45rem; font-size:.72rem; font-weight:400; + color:var(--muted); letter-spacing:.02em; } +.flowfold { margin-top:2rem; } +.flowfold > summary { font-size:.9rem; font-weight:600; } +.flowfold .section-label { margin-top:.8rem; } +.dfoot { margin:.6rem 0 0; } +.fork-label { text-align:center; color:var(--muted); font-size:.78rem; + letter-spacing:.08em; text-transform:uppercase; margin:.2rem 0 .6rem; } +.fork { display:grid; grid-template-columns:1fr 1fr; gap:1rem; } +.fork-side { text-align:center; font-size:.75rem; font-weight:700; + letter-spacing:.08em; color:var(--muted); margin:0 0 .4rem; } +.fbranch { min-width:0; } +.cols { display:grid; grid-template-columns:1fr 1fr; gap:1.1rem; + margin-top:.5rem; } +@media (max-width: 780px) { + .cols, .fork, .dgrid { grid-template-columns:1fr; } } +.col { background:var(--panel); border:1px solid var(--border); + border-radius:8px; padding:1rem 1.1rem 1.1rem; min-width:0; } +.col-head { display:flex; align-items:baseline; gap:.6rem; } +.col-note { color:var(--muted); font-size:.85rem; } +.count { margin:.35rem 0 .8rem; color:var(--muted); + font-variant-numeric:tabular-nums; } +.trial { border-top:1px solid var(--border); padding:.65rem 0 .35rem; } +.trial-head { display:flex; align-items:center; gap:.55rem; margin:0; } +.acts { margin:.25rem 0 .3rem; color:var(--muted); font-size:.92rem; } +.badge { border-radius:4px; padding:1.5px 8px; font-size:11.5px; + font-weight:700; letter-spacing:.04em; color:#fff; } +.badge.pass { background:var(--pass); } +.badge.fail { background:var(--fail); } +.badge.blocked { background:var(--blocked); } +.badge.review { background:var(--accent); } +summary { cursor:pointer; color:var(--accent); font-size:.86rem; + margin:.15rem 0; } +summary:focus-visible { outline:2px solid var(--accent); + outline-offset:2px; } +details { margin-bottom:.25rem; } +.d-add { color:var(--d-add); } .d-del { color:var(--d-del); } +.d-ctx { color:var(--muted); } +.result { background:__RESULT_BG__; color:#fff; border-radius:8px; + padding:.85rem 1.1rem; font-size:1.15rem; font-weight:700; + margin-top:.5rem; } +.footer { color:var(--muted); font-size:.85rem; margin-top:1.6rem; + max-width:70ch; } diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 08ab2d9..72b39ab 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -235,6 +235,39 @@ def main(): "invalid report-data field variants.after.count_suffix: expected string", ) + from reporting.render_html import _resolve_css, render_artifact, render_document + + assert _resolve_css(".result { background:__RESULT_BG__; }", "good") == ( + ".result { background:var(--pass); }" + ) + assert _resolve_css(".result { background:__RESULT_BG__; }", "bad") == ( + ".result { background:var(--fail); }" + ) + assert _resolve_css(".result { background:__RESULT_BG__; }", "neutral") == ( + ".result { background:var(--accent); }" + ) + for css in (".result {}", "__RESULT_BG__ __RESULT_BG__"): + try: + _resolve_css(css, "good") + except ValueError as error: + assert str(error) == "report.css must contain __RESULT_BG__ exactly once" + else: + raise AssertionError("invalid CSS token count was accepted") + + artifact = render_artifact(report, ".result { background:__RESULT_BG__; }") + assert artifact == render_artifact(report, ".result { background:__RESULT_BG__; }") + document = render_document(artifact) + assert document == render_document(artifact) + escaping_raw = copy.deepcopy(raw) + escaping_raw["variants"]["before"]["trials"][0]["verdict"] = 'REVIEW">