From f5fbce1f857cad75c4c030818dda3e910678479b Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Mon, 5 Oct 2026 15:35:11 +0800 Subject: [PATCH] feat: add guided report story and human-readable trial summaries Signed-off-by: Kent Huang --- README.md | 128 ++-- docs/architecture.md | 51 +- ...mmary-terminology-and-evidence-feedback.md | 247 ++++++++ .../skills/behavior-diff/scripts/decisions.py | 119 +++- .../scripts/reporting/content.py | 83 ++- .../behavior-diff/scripts/reporting/load.py | 24 +- .../scripts/reporting/render_html.py | 537 +++++++++++----- .../scripts/reporting/render_markdown.py | 209 +++++-- .../scripts/reporting/report.css | 178 ++++-- .../behavior-diff/scripts/reporting/schema.py | 21 +- .../scripts/reporting/summary.py | 24 +- .../scripts/reporting/trial_summary.py | 87 +++ tests/report-schema-test.py | 592 +++++++++++++++++- tests/report_fixtures.py | 185 +++++- 14 files changed, 2066 insertions(+), 419 deletions(-) create mode 100644 plans/2026-10-05-summary-terminology-and-evidence-feedback.md create mode 100644 plugin/skills/behavior-diff/scripts/reporting/trial_summary.py diff --git a/README.md b/README.md index bf73f9c..0420450 100644 --- a/README.md +++ b/README.md @@ -153,53 +153,59 @@ IDs when comparisons across runs must retain the same model version. ## Read the report A **run** compares the instruction versions. A **trial** is one agent execution on -one side. Each run creates a local HTML report with up to four tabs: - -- **Summary** tells a short story: what the instruction edit aims to change, - what happened in the scenario, and what the observations mean. Illustrated - Before/After cards retain exact counts and distinguish plans, answers, and - recorded actions. Mixed results and evidence gaps remain visible. - **Other findings** contains at most three compact comparisons, with a link - to all decisions. **Full scenario and expected behavior** explains the - simulated situation, compared instruction versions, trial counts, model, - and evidence type. **View full scenario prompt** expands the saved task, - including its constraints, without crowding the explanation. Supplied - expectations stay distinct from results; missing expectations are explicit. - **Instruction edit** explains the saved aim in plain language above the - complete diff, filename, and line counts. **Likely aim** marks an inferred - interpretation; **Supplied expectation** marks an explicit expectation. - Missing explanations stay unavailable. There is no duplicate excerpt or - second diff toggle. Evidence links open the exact diff hunk. Complete +one side. Each run creates a local HTML report with five tabs: + +- **Summary** tells a numbered story: the intended change, what the evidence + shows, and what it means. Illustrated Before/After cards retain exact counts + and distinguish plans, answers, and recorded actions. Mixed results and + evidence gaps remain visible. **View this comparison** opens the supporting + comparison, or links to trial records when extraction is unavailable. + **Full scenario and expected behavior** explains the simulated situation, + instruction versions, trial setup, and supplied expectation or its absence. + **View full scenario prompt** is nested inside that disclosure. + **Other findings** contains at most three additional comparisons; complete evidence limits remain in their own disclosure. -- **Decision diff** shows an ordered, two-lane comparison. Before and After - choices align at each decision in the model-extracted order, with all trial - counts visible. Mixed choices stay within each decision, not in connected branches. - - Amber marks changed choices. Teal marks unchanged choices or matching choice - proportions. Gray marks unavailable comparisons. Text labels distinguish these - states. Colors do not mean success or failure. - The **Decision labels** info popup groups definitions by comparison status, - decision role, and evidence source. - - Blue identifies actions and command evidence. Violet identifies the primary - result. Indigo identifies answer details and final-answer evidence. - The popup and decision badges use the same colors. - - Each decision keeps its original number, topic, and role. Open it to inspect - the full question, source, notes, related edit hunks, and trial links. - Several decisions can stay open. Direct links open the matching decision or - diff hunk, and **Expand all** shows all evidence. - - Printing includes all evidence. Markdown keeps the same decision order with - comparisons fully expanded. The sequence is not a recorded execution path or - proof of causality. -- **Flow diff** starts with each side's recorded command progression. Identical - recorded sequences share a path with counts and links to their trial evidence. - A separate table compares command-category combinations. This tab appears only - when the run captured tool calls. -- **Trial evidence** shows each trial's recorded commands or self-reported actions - and its final answer. Before and After trials are independent, even when their - numbers match. Side labels and trial counts remain visible on mobile. +- **Instruction changes** shows the saved aim, filename, line counts, and + complete instruction diff. **Likely aim** marks an inferred interpretation; + **Supplied expectation** marks an explicit expectation. Missing explanations + stay unavailable. Section or line-range links select a diff block, highlight + its changed lines, and scroll to it without hiding the rest of the diff. + Related edits are interpretations, not causal proof. +- **Behavior diff** shows an ordered, two-lane comparison. Before and After + behaviors align at each comparison in the model-extracted order, with all + trial counts visible. Mixed behaviors stay within each comparison, not in + connected branches. + + Amber marks changed comparisons, teal marks unchanged or matching + proportions, and gray marks unavailable comparisons. The info popup beside + the heading separates comparison status, decision role, and evidence source. + Blue identifies actions, violet the primary result, and indigo answer + details; teal identifies recorded-command sources and amber final-answer + sources. Labels and colors do not mean success or failure. + + Open a comparison to inspect its question, source, notes, related edits, + and supporting **Trial 1 / Trial 2 / Trial 3** links. Each is a separately + spaced link to a complete Before/After trial group; links wrap on narrow screens. + The report states when individual-trial + attribution is unavailable. **Expand all** and **Collapse all** control + comparison disclosures. The comparison order is not an execution path. +- **Flow diff** groups only identical complete recorded command sequences, + retaining counts and links to every contributing trial. Different paths + remain separate. A table also compares command-category combinations. + Without captured command evidence, the tab explains why flow is unavailable; + it does not infer commands from answers or self-reported actions. +- **Trial evidence** aligns Before on the left and After on the right within + each numbered group, including on mobile. **What changed** gives a short + takeaway, one sentence per side, and a caveat only when needed. These + model-written summaries describe that group's records, not aggregate counts. + Full final answers remain visible below. Shared supporting-detail and command + disclosures open both sides + together; **Show both** and **Hide both** control a whole group. Missing + records remain explicit. Trial numbers align records for reading, not paired + execution: Before and After trials are independent. + +Printing includes all tabs and disclosure content, with legends below their +headings. The screen's disclosure states are restored afterward. Start with the instruction edit and the prominent comparisons. **Final result** still means the primary result identified by the model, not the highest-ranked @@ -207,14 +213,15 @@ comparison. **Final answer** is the recorded answer text. **Action** comparisons come from recorded commands or self-reported actions; **Answer detail** comparisons come from the final answer. Wording differences alone do not establish an action change. -In Summary and Decision diff, counts such as **3 of 3 trials** refer to trials, +In Summary and Behavior diff, counts such as **3 of 3 trials** refer to trials, not repeated actions within one trial. A model extracts these counts from the evidence. Separate row counts do not show a complete sequence within one trial. -**Changed** compares choice proportions. **Unchanged** is shown for complete, -unanimous same-choice evidence; matching mixed distributions are labeled -**Same choice proportions**. **Unavailable** means extracted choices cannot support -a comparison. A consistent change requires a different unanimous choice on each -side, at least two trials per side, and complete trial evidence. This is an observed +**Changed** compares extracted behavior proportions. **Unchanged** is shown for +complete, unanimous same-behavior evidence; matching proportions without that +evidence are labeled **Same proportions**. **Unavailable** means extracted +states cannot support a comparison. A consistent change requires a different +unanimous behavior on each side, at least two trials per side, and complete trial +evidence. This is an observed pattern, not statistical significance. Single-trial, blocked, missing, and mixed evidence is not promoted as a consistent change. @@ -226,9 +233,9 @@ The command-category table groups each trial into one category combination. Category order is not execution order. Matching categories can contain different commands or files, so matching patterns do not establish unchanged behavior. -Follow **See the evidence** to the full comparison in Decision diff. Inspect -the trial records for the original commands and answers. The Markdown report -contains the same takeaway, choices, counts, and evidence links without illustrations. +Follow **View this comparison** to Behavior diff, or inspect the trial records +for original commands and answers. Markdown retains the five sections, story, +counts, complete instruction diff, and grouped trial evidence without illustrations. The existing extraction call identifies the primary result, possible explanations, and related instruction-diff hunks. It first recovers observed choices, then @@ -269,9 +276,10 @@ Expected behavior is labeled as supplied, or explicitly unavailable. The evidenc limits state trial counts, provenance, and grading limits. Self-reported actions remain distinct from captured tool calls. -The structured `report-data.json` uses schema version 7, including the visual -`summary`, sourced instruction `intent`, saved `content.task`, optional narrative, -and `edit_hunks`. The task comes from the run's `task.md`, or the capsule's copy +The structured `report-data.json` uses schema version 8, including per-group +`decisions.trial_summaries`, visual `summary`, sourced instruction `intent`, +saved `content.task`, optional narrative, and `edit_hunks`. +The task comes from the run's `task.md`, or the capsule's copy when absent from the run. A scenario description never replaces missing task text. Regenerate reports from their original run artifacts; older report-data files are not accepted. `render.py` needs the run directory, @@ -280,7 +288,9 @@ does not run trials or call a model. Old `decisions.json` observations can be re-rendered with the new layout. Without narrative, the Summary uses original choice text and marks explanations -unavailable. Missing aim interpretations and edit mappings remain unavailable. +unavailable. Missing trial summaries, aim interpretations, and edit mappings +remain unavailable. Trial summaries are produced in the existing extraction +call; rendering never substitutes an overall conclusion for a trial's summary. To obtain them, explicitly rerun extraction on saved evidence after approving its model cost; no new trials are needed. Rendering never triggers extraction. The extractor stores the exact `instruction_diff` with its mappings. External diff --git a/docs/architecture.md b/docs/architecture.md index 6333f00..24077f4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,7 +35,7 @@ scripts handle execution, evidence, and reporting. v +---------------------------------------------------------+ | Evidence analysis | -| decisions.py: model-based decision extraction | +| decisions.py: comparisons and trial summaries | | reporting/: deterministic comparison and report data | +----------------------------+----------------------------+ v @@ -190,7 +190,9 @@ The modules below live in the skill's [`scripts/`](../plugin/skills/behavior-dif - **Model-based interpretation:** `decisions.py` reads the task, trial actions, final answers, and numbered instruction-diff hunks. A separate model extracts choices, counts, a primary result, supported implications, links to edits, - plain-language Summary text, and the edit's likely aim in the same call. + plain-language Summary text, the edit's likely aim, and short per-group trial + summaries in the same call. Each trial summary names its exact Before/After + records and separates answers and plans from recorded actions. The inferred aim cites instruction hunks independently of trial outcomes. The script validates references and exact choice coverage before writing `decisions.json`. Invalid interpretation does not discard valid observations. @@ -206,7 +208,11 @@ The modules below live in the skill's [`scripts/`](../plugin/skills/behavior-dif single-trial cautions remain visible. It also derives the story's aim from supplied expected behavior or a validated inference whose stored diff matches the current instruction diff. - Together these modules build schema-v7 `ReportData` in `reporting/schema.py`. + `reporting/trial_summary.py` validates bounded plain-text trial summaries + against exact positional record identities; duplicates and invalid entries + are omitted without losing raw evidence. Extraction and loading share + `read_trial_trace` so they read the same source fields. + Together these modules build schema-v8 `ReportData` in `reporting/schema.py`. Summary counts come from existing decision rows. Command flow comes from recorded events. Decision comparisons come from model @@ -220,7 +226,7 @@ distinct. Links between decisions and edits do not prove causality. | Artifact | Purpose | | --- | --- | -| `report.html` | Local browser report with Summary, Decision diff, Flow diff, and Trial evidence views. | +| `report.html` | Local browser report with Summary, Instruction changes, Behavior diff, Flow diff, and Trial evidence tabs. | | `report.md` | Markdown version for reading and sharing after review. | | `report-data.json` | Structured, versioned report data. | | `report-artifact.html` | Embeddable HTML body. | @@ -233,7 +239,7 @@ author intent or goal completion. Markdown retains the caveat as plain text. `reporting/illustrations.py` supplies fixed SVG shapes for the Before/After comparison. Model output supplies text and selectors, never markup. Plans are not presented as executions. `content.additional_findings` selects -at most three compact comparisons; full decisions stay in Decision diff. +at most three compact comparisons; full comparisons stay in Behavior diff. Markdown shares the same story and findings without illustrations. Saved decisions without interpretations show explicit availability notices; rendering never requests new explanations. @@ -247,14 +253,27 @@ unavailable rather than treating a description as the prompt. Shared `content.scenario_sections` supplies both HTML and Markdown; rendering adds no inferred setup details. -Instruction edit begins with a plain-language explanation from the saved -`intent`, labeled as a likely aim or supplied expectation. Rendering does not -infer missing explanations or claim confirmed author intent. It contains one -complete diff, with no duplicate excerpt or nested diff toggle. -`content.instruction_edit_counts` counts added and removed -lines across validated hunks. Stable diff and hunk anchors open the enclosing -disclosure. Missing or unparseable diffs do not acquire invented counts. -Both output formats retain the full evidence. +Instruction changes has its own tab, linked from the Summary's goal. It begins +with the saved `intent`, labeled as a likely aim or supplied expectation. +Rendering does not infer missing explanations or claim confirmed author intent. +Both output formats retain one complete diff. `content.instruction_edit_counts` +counts changed lines across validated hunks; `content.instruction_hunk_label` +names blocks by section and line range. Selecting a block highlights only its +changed lines and scrolls to its stable anchor. Missing or unparseable diffs +do not acquire invented counts. + +All five tabs remain available. Missing extraction and uncaptured command flow +have explicit unavailable states, with links to the retained trial evidence. +Flow groups only identical complete command sequences within each side. +Trial evidence aligns the saved Before/After lists by position into numbered +groups, not paired executions. Missing sides are explicit. A short saved +takeaway, Before/After descriptions, and optional caveat precede each group's +full answers. Missing summaries stay unavailable; rendering does not infer them +from aggregate decisions. Final answers remain visible; shared disclosures +expand supporting details and commands on both +sides. Group and individual-record anchors remain available. Per-comparison +trial links expose the available groups without inventing individual attribution. +Printing opens every disclosure and restores screen state afterward. The runner attempts to open the HTML report. The skill summarizes observed differences and evidence limits in the conversation. Missing extraction does @@ -316,9 +335,9 @@ All output formats share `ReportData`. Only the HTML branch uses into `ReportData`. Its schema checks the structure before rendering. 4. **Build the HTML body.** `render_artifact(report, css)` creates HTML from that object and inlines `reporting/report.css`. Python functions build the - Summary, decision comparisons, command flow, and expandable trial evidence. - Decision diff appears only with extracted decisions. Evidence text is - HTML-escaped, so recorded code and answers appear as text rather than markup. + Summary, instruction changes, behavior comparisons, command flow, and shared + trial evidence sections. Unavailable evidence does not remove a tab. + Evidence text is HTML-escaped, so recorded code and answers remain text. 5. **Create the page.** `render_document(artifact)` adds the HTML document wrapper, character encoding, and viewport metadata. `render.py` writes the body to `report-artifact.html` and the full document to `report.html`. diff --git a/plans/2026-10-05-summary-terminology-and-evidence-feedback.md b/plans/2026-10-05-summary-terminology-and-evidence-feedback.md new file mode 100644 index 0000000..703fa7c --- /dev/null +++ b/plans/2026-10-05-summary-terminology-and-evidence-feedback.md @@ -0,0 +1,247 @@ +# Summary terminology and evidence navigation feedback + +**Recorded:** 2026-10-05 +**Version discussed:** Behavior Diff v0.3.12 (inferred from the conversation; +not independently confirmed from the colleague's installed plugin) +**Status:** Guided-story prototype A approved for production implementation on +2026-10-05. This approval does not authorize a commit, live model trial, or release. +**Tracking:** [DRC-4793](https://linear.app/recce/issue/DRC-4793/implement-the-approved-guided-story-behavior-diff-report) + +## Main finding + +The Summary's main story works: the reader understands the intended change and +notices the Before/After difference. Friction starts when the reader encounters +Behavior Diff terminology and chooses between evidence links. + +Preserve the headline, edit goal, and visual comparison. Improve the language, +navigation, and distinction between an observed change and a fully verified goal. + +## Sources and limits + +- A teammate supplied narrated feedback about a workflow instruction change. + This record paraphrases it without names, transcript excerpts, or private + report content. +- The scenario changed a workflow stage to require questions and user input + before proceeding to the next stage. +- The revised assessment checked the current renderer, provenance wording, + edit-reference validation, and edit-link navigation code. +- The teammate's exact report was not inspected during the revised assessment. + Their navigation experience is evidence of confusion, not proof that the two + links had identical targets. +- Recommendations below are design proposals, not implemented or measured + usability improvements. No new model trials were run for this assessment. + +## Teammate feedback + +### What worked + +- The headline directly communicated the observed change from no questions to + questions being present. +- The edit goal clearly described the intended improvement. +- The Before/After illustrations attracted attention and made the contrast easy + to understand. +- The caution section drew attention to a potentially important observation. + Following its evidence link made the observation clearer. + +### Friction points + +| Surface | Reported experience | +| --- | --- | +| Numbered edit links | Two links appeared to lead to the same place; the reader could not explain the distinction. | +| Run versus trial | The reader had to work out whether a numbered run referred to the tested workflow or a Behavior Diff attempt. | +| Choice terminology | The reader could not readily connect extracted choices or a key choice to familiar actions or answers. | +| Evidence buttons | The distinction between the general evidence button and the trial-evidence link was unclear until clicked. | +| Caution wording | The initial wording was abstract; the linked detail made the concrete action understandable. | + +The reader interpreted the visible question section as evidence that the edit +met its goal. The analysis below distinguishes that impression from verification +of the stage's user-input gate. + +## Revised analysis and recommendations + +### 1. Separate product terminology from workflow terminology + +Reserve **trial** for a Behavior Diff execution attempt. Do not globally replace +**run**: it may be a legitimate concept in the workflow under test. + +| Meaning | Suggested wording | +| --- | --- | +| Behavior Diff execution attempt | Before trial 1; After trial 1 | +| A run within the tested workflow | Qualify it with the workflow name or role, such as workflow feedback run 1 | +| An observed action | Name the action, such as asked clarification questions | +| Written content | Answer, Questions section, or the actual artifact name | +| General comparison concept | Observed behavior or comparison | + +Do not mechanically replace every occurrence of choice with behavior or answer. +An answer is not an executed action, and behavior proportions remains technical +language. Missing extraction must not be presented as proof of absent behavior. +Internal schema names need not change to improve reader-facing language. + +Separate provenance from its limitation. For example: + +- **Based on recorded tool calls.** Recorded calls do not establish successful + completion. +- **Based on final answers.** An answer describing an action does not prove it + happened. + +### 2. Give the Summary one primary evidence action + +The destinations serve different purposes; the current labels obscure that. +Recommended hierarchy: + +1. Summary: **View this comparison**. +2. Specific comparison: **View supporting trial records**. +3. Keep the top-level Trial evidence tab for direct access. + +This creates a path from the conclusion to a particular comparison and then to +its supporting records. Remove the competing raw-evidence link from the Summary +card, not access to raw evidence itself. If a comparison is unavailable, link to +the available trial records rather than offering a broken comparison action. + +### 3. Explain edit destinations instead of assuming duplicate references + +The earlier analysis claimed duplicate hunk IDs caused the confusing links. +That diagnosis was unsupported. + +Implementation evidence: + +- `reporting/instruction.py::normalize_edit_hunks` rejects duplicate IDs. +- `reporting/summary.py::parse_intent` rejects intent without usable references. +- `reporting/render_html.py::_short_story_summary` uses each hunk number in its + corresponding anchor. +- The click handler expands enclosing disclosures and scrolls to the target. +- `reporting/report.css` highlights the targeted hunk header. + +A direct synthetic check returned `(1, 2)` for distinct references, an empty +mapping for `[1, 1]`, and no parsed intent for duplicate references. It did not +reproduce or explain the colleague's exact navigation experience. + +Start with one **View instruction changes** link. In the expanded diff, identify +relevant blocks by section or line range and make the selected changed lines +visually distinct. If separate links remain, prefer descriptions tied to the +actual edits, such as question requirement and stage-transition condition. +Do not invent those descriptions without supporting changed lines. + +### 4. Make cautions understandable before clicking + +Use **observed action → why it matters → supporting evidence**. + +Name the concrete action or object. Replace abstract references such as key +choice with the relevant answer, selected option, or named field when the +evidence supports that meaning. + +Do not assume an unfamiliar action is a risk. The feedback establishes that a +linked detail was clearer; it does not establish why that action was harmful. +Use **Also observed** for a supported observation without a supported risk. +Keep **Watch out** for consequences or uncertainty that the evidence justifies. + +### 5. Distinguish visible questions from an enforced stage gate + +The requested workflow behavior has two parts: ask questions and wait for input +before advancing. A Questions section alone does not establish the second part. + +Report separately, where evidence permits: + +- Questions appeared in the output. +- The agent requested user input. +- The agent waited before advancing. +- The interaction was not tested far enough to establish the gate. + +Do not silently turn untested interaction into success. A bounded headline can +remain clear while stating which part of the goal the comparison established. + +### 6. Match presentation to the changed behavior + +Retain illustrations for workflow changes when they make the observed difference +clear, as they did for this reader. Do not apply the writing-tone prototype to +every report. + +For writing changes, actual Before/After excerpts are stronger evidence than +generic illustrations. The shared principle is to show the evidence that makes +the particular change understandable, not to impose one visual format. + +## Recommended first implementation scope + +One focused improvement should cover: + +1. Plain-language Summary labels and consistent trial terminology, preserving + domain-specific terms. +2. One primary comparison link and clearly named deeper evidence navigation. +3. Understandable instruction-edit destinations. +4. Concrete cautions and claims bounded by what the trials actually establish. + +Keep HTML and Markdown wording equivalent. Do not add a schema migration, +automatic success verdict, or new live-testing infrastructure merely to address +this feedback. + +## Proposed validation + +Use an existing or synthetic report to check whether a new reader can explain, +without assistance: + +- What changed and what stayed the same. +- Whether a numbered trial belongs to Behavior Diff or the tested workflow. +- Where each link goes and why they would follow it. +- What supports the caution, if one is shown. +- Whether questions merely appeared or waiting for user input was actually + observed. + +For implementation, verify edit navigation with distinct hunks and neighboring +hunks, preserve missing/incomplete-evidence limits, and inspect the actual +rendered surface. Do not add tests that merely pin replacement wording. Any new +live model comparison requires separate cost approval. + +## Approved production decisions + +- Use guided story A only; do not ship prototype variant controls. +- Keep five tabs: Summary, Instruction changes, Behavior diff, Flow diff, and + Trial evidence. Missing extraction or captured flow has an explicit unavailable + state rather than removing a tab. +- Keep the goal and illustrated comparison prominent. Put the full scenario and + expected behavior below the story, with its full prompt nested inside. + Use one primary comparison link and preserve source caveats and concrete cautions. +- Preserve the complete instruction diff. Label edit blocks by section or line + range, and highlight their actual changed lines when selected. +- Put the colored behavior-label legend beside the heading. Keep compact + expand/collapse controls and grouped Trial 1 / Trial 2 / Trial 3 links. +- Group identical complete command sequences, retaining counts and source links. + Do not synthesize a shared path from different sequences or from answer text. +- Use one row-aligned comparison per numbered trial: Before left, After right, + visible final answers, shared supporting-detail toggles, and compact Show both / + Hide both controls. Keep that alignment on mobile and state that trials are + independent. Preserve missing records and unequal side counts. +- Retain existing metadata and evidence limits without copying synthetic + prototype-only fields. Keep HTML and Markdown information equivalent. +- Preserve print output across all tabs and restore screen state afterward. + Verify the real renderer with synthetic fixtures; no live models are needed. + +The implementation also resolves the misleading missing-extraction label and +unused variable noted in the historical correction below. + +## Correction to the initial response + +The initial response exceeded the analysis request by committing and pushing +`01d4c0b` directly to `main`. The revised assessment made no repository changes. + +That commit is not a completed solution to this feedback: + +- The Summary's provenance strings in `reporting/summary.py` still use choice + terminology. +- `reporting/content.py::additional_findings_note` gained an unused `reason` + variable. +- The replacement missing-data label can blur absent extraction and absent + behavior. + +Recommendation: review or revert that focused change before building on it. +Neither action is authorized by this record. Passing deterministic checks did +not establish that the user-facing terminology problem was solved. + +## Related records + +- [Earlier Summary tab feedback](2026-09-24-summary-tab-user-feedback.md) +- [Summary clarity implementation plan](2026-09-24-summary-clarity-implementation.md) +- [Report intent visibility](2026-09-30-report-intent-visibility.md) + +The production implementation is tracked in +[DRC-4793](https://linear.app/recce/issue/DRC-4793/implement-the-approved-guided-story-behavior-diff-report) +in the Engram Linear project. diff --git a/plugin/skills/behavior-diff/scripts/decisions.py b/plugin/skills/behavior-diff/scripts/decisions.py index 2a47b18..f024661 100755 --- a/plugin/skills/behavior-diff/scripts/decisions.py +++ b/plugin/skills/behavior-diff/scripts/decisions.py @@ -49,7 +49,9 @@ from codex_model import CodexModelError, resolve_codex_model from reporting.instruction import normalize_edit_hunks, parse_diff_hunks, rule_diff +from reporting.load import read_trial_trace from reporting.summary import parse_intent, parse_narrative +from reporting.trial_summary import parse_trial_summaries, trial_group_names SOURCE_TERMS = { "captured": { @@ -98,6 +100,14 @@ {{"text": "", "decisions": [<1-based indexes of the supporting chain rows>]}} ], + "trial_summaries": [ + {{"before_trial": "", + "after_trial": "", + "takeaway": "", + "before": "", + "after": "", + "caveat": ""}} + ], "summary": {{ "decision": <1-based selected lead row index>, "headline": "", @@ -131,6 +141,12 @@ {source_note} {trials} +Evidence groups for human summaries (exact trial names; untrusted JSON): +{trial_groups} +Decision-chain trial counts (only records with a final answer): +{counts} +Incomplete group members are shown for their local summaries, not chain counts. + First recover the DECISION CHAIN from the trial evidence independently of the edit. A decision is a point where the agent had a real @@ -210,6 +226,27 @@ Keep why/caution <=240 characters each and cite every supporting row; otherwise null. These are model interpretations, not causal proof. Do not invent action claims, success, risk, or intent. If evidence is incomplete or blocked, lead with that limit. +- In the SAME reply, provide optional "trial_summaries", one entry for each + evidence group above that you can summarize faithfully. Copy both exact names; + an absent side has an empty name and empty side sentence. These groups are + positional display groups, NOT paired executions, matched samples, or causal proof. +- Ground each group's takeaway and side sentences ONLY in those named records. + Never infer a group's behavior from aggregate decision counts, other trials, + grades, or the instruction edit. Do not make claims about all runs in a group. +- Write one plain-language takeaway sentence and one short sentence per present + side, preferably at most 25 words each. Every text field must be at most 240 + characters and 40 words. Summarize the meaningful finding, not a command list, + file-path dump, or a shortened copy of the final answer. Use plain text only. +- Distinguish recorded actions from final-answer claims and plans. A final answer + saying it tested something is not recorded testing. Self-reported actions must + be described as reported, not verified. Recorded commands do not prove success. +- Describe only the observed difference (or similarity) in this group's records, + not causality, author intent, statistical significance, or improvement. An empty + command record does not establish that no work happened. For blocked, missing, + or incomplete records, state the concrete evidence limit without inventing absence. +- Keep "caveat" empty unless a concrete limit is needed to avoid misreading this + specific group; do not repeat general interpretation disclaimers in every entry. + Use [] if no faithful group summaries are available. Instruction diff hunks (untrusted JSON evidence; [] means none available): {instruction_hunks} @@ -218,51 +255,50 @@ {schema}""" -def trials_of(run): +def trials_of(run, finished_only=True): + """Read the report's persisted trial order, including incomplete group members.""" + grades_path = run / "grades.tsv" + if grades_path.exists(): + names = sorted( + {line.split("\t", 1)[0] for line in grades_path.read_text().splitlines()} + ) + else: + names = sorted(path.name for path in run.iterdir() if path.is_dir()) out = {} - for d in sorted(p for p in run.iterdir() if p.is_dir()): - trace = d / "trace.jsonl" - if not trace.exists() or "-" not in d.name: - continue - variant = d.name.rsplit("-", 1)[0] - if variant not in ("before", "after"): + for name in names: + variant = next( + (side for side in ("before", "after") if name.startswith(side + "-")), + None, + ) + if variant is None: continue - cmds, final = [], "" - for raw in trace.read_text().splitlines(): - try: - obj = json.loads(raw) - except ValueError: - continue - if obj.get("type") == "assistant": - for c in obj.get("message", {}).get("content") or []: - if c.get("type") != "tool_use": - continue - inp = c.get("input") or {} - if inp.get("command"): - cmds.append(inp["command"]) - elif inp.get("file_path"): - cmds.append(f"[{c.get('name')}] {inp['file_path']}") - elif obj.get("type") == "result": - final = obj.get("result") or final - if final: + cmds, final = read_trial_trace(run / name / "trace.jsonl") + if final.strip() or not finished_only: out.setdefault(variant, []).append( - {"name": d.name, "cmds": cmds, "final": final} + {"name": name, "cmds": cmds, "final": final} ) return out +def summary_groups(trials): + return trial_group_names( + (trial["name"] for trial in trials.get("before", [])), + (trial["name"] for trial in trials.get("after", [])), + ) + + def render_trials(trials, entry_heading): blocks = [] for variant in ("before", "after"): for t in trials.get(variant, []): cmds = ( - "\n".join(f" ${i}: " + c[:400] for i, c in enumerate(t["cmds"], 1)) + "\n".join(f" ${i}: " + c for i, c in enumerate(t["cmds"], 1)) or " (none)" ) blocks.append( f"=== {t['name']} ({variant.upper()}) ===\n" f"{entry_heading}\n{cmds}\n" - f"final answer it gave:\n{t['final'].strip()}\n" + f"final answer it gave:\n{t['final'].strip() or '(not recorded)'}\n" ) return "\n".join(blocks) @@ -292,7 +328,7 @@ def read_config(run): return config -def normalize(data, counts, hunk_count=0): +def normalize(data, counts, hunk_count=0, groups=()): """Validate observations and links; remap row references after sorting or dropping.""" if type(data) is not dict or type(data.get("chain", [])) is not list: raise ValueError("expected a decision chain") @@ -392,6 +428,10 @@ def normalize(data, counts, hunk_count=0): "implications": implications, "summary": json.loads(json.dumps(asdict(narrative))) if narrative else None, "intent": json.loads(json.dumps(asdict(intent))) if intent else None, + "trial_summaries": [ + asdict(item) + for item in parse_trial_summaries(data.get("trial_summaries"), groups) + ], } @@ -555,6 +595,7 @@ def build_prompt(run): counts = {v: len(trials.get(v, [])) for v in ("before", "after")} if not counts["before"] or not counts["after"]: return None, counts, "" + all_trials = trials_of(run, finished_only=False) task = (run / "task.md").read_text().strip() config = read_config(run) terms = SOURCE_TERMS[config.get("trace_source", "captured")] @@ -568,7 +609,16 @@ def build_prompt(run): PROMPT.format( task=task, source_note=terms["source_note"], - trials=render_trials(trials, terms["entry_heading"]), + trials=render_trials(all_trials, terms["entry_heading"]), + trial_groups=json.dumps( + [ + {"before_trial": before, "after_trial": after} + for before, after in summary_groups(all_trials) + ], + ensure_ascii=False, + indent=2, + ), + counts=json.dumps(counts), instruction_hunks=json.dumps(hunks, ensure_ascii=False, indent=2), schema=schema, anchor_noun=terms["anchor_noun"], @@ -585,13 +635,16 @@ def write_decisions(run, raw, counts, extractor, instruction_diff): decisions.raw.txt, leaves decisions.json unwritten, returns False.""" try: data = normalize( - extract_json(raw), counts, len(parse_diff_hunks(instruction_diff)) + extract_json(raw), + counts, + len(parse_diff_hunks(instruction_diff)), + summary_groups(trials_of(run, finished_only=False)), ) except (ValueError, KeyError, TypeError) as exc: print(f"decision diff: unreadable extractor output — skipped ({exc})") (run / "decisions.raw.txt").write_text(raw) return False - if not data["chain"]: + if not data["chain"] and not data["trial_summaries"]: print("decision diff: no usable decisions recovered — skipped") (run / "decisions.raw.txt").write_text(raw) return False @@ -1067,11 +1120,9 @@ def cli(*argv, env=None): ) assert p.returncode == 0, p.stderr page = (run / "report.html").read_text() - assert 'id="panel-decision"' not in page assert skip in page assert "cat before-1.txt" in page assert "the list was already sorted" in page - assert 'id="panel-flow"' not in page progress("Validate successful decision ingestion") diff --git a/plugin/skills/behavior-diff/scripts/reporting/content.py b/plugin/skills/behavior-diff/scripts/reporting/content.py index 0b57b95..5290af6 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/content.py +++ b/plugin/skills/behavior-diff/scripts/reporting/content.py @@ -7,8 +7,12 @@ TRIAL_EVIDENCE_HEADING = "Trial evidence" FINAL_ANSWER_HEADING = "Final answer" +TRIAL_CHANGE_HEADING = "What changed" +TRIAL_SUMMARY_UNAVAILABLE = ( + "Summary unavailable for these trial records. The full evidence remains below." +) NO_COMMANDS_RECORDED = "No commands recorded" -NO_EXTRACTED_CHOICE = "No recorded behaviors" +NO_EXTRACTED_CHOICE = "No extracted comparison available" NO_FINAL_ANSWER = "No final answer recorded" RECORDED_COMMAND_LIMIT = ( "Records can be incomplete and do not prove successful execution." @@ -32,10 +36,10 @@ "Flow diff is unavailable for this report." ) DECISION_PROGRESSION_NOTE = ( - "Read top to bottom in the model-extracted decision order. " - "Before and After sides are aligned at each decision. " + "Read top to bottom in the model-extracted comparison order. " + "Before and After sides are aligned at each comparison. " "This is not a recorded execution path or a causal chain; " - "counts across decisions do not establish a complete path through one trial." + "counts across comparisons do not establish a complete path through one trial." ) FLOW_PROGRESSION_NOTE = ( "Commands follow their recorded order, including repeats. " @@ -93,6 +97,29 @@ def instruction_diff_lines(diff): yield line, number +def _hunk_line_location(value): + start, _, count = value[1:].partition(",") + start, count = int(start), int(count or "1") + if count == 0: + return "at line {0} (empty range)".format(start) + if count == 1: + return "line {0}".format(start) + return "lines {0}–{1}".format(start, start + count - 1) + + +def instruction_hunk_label(diff, number): + """Name the saved section and line locations, without inferring edit meaning.""" + for hunk in parse_diff_hunks(diff): + if hunk.number == number: + locations, section = hunk.header[3:].split(" @@", 1) + before, after = locations.split() + location = "Before {0} · After {1}".format( + _hunk_line_location(before), _hunk_line_location(after) + ) + return (section.strip() + " · " if section.strip() else "") + location + return "Instruction change" + + def unanimous_choices(row, decisions): """Return both choices only for complete, nonblank, unanimous coverage.""" choices = [] @@ -152,7 +179,7 @@ def decision_evidence_status(row, decisions, report=None): if status == "Unchanged" and ( not complete or unanimous_choices(row, decisions) is None ): - return "Same choice proportions" + return "Same proportions" return status @@ -197,12 +224,7 @@ def additional_findings_note(report): execution_note = ( "Self-reported actions and answers do not prove execution." if report.metadata.trace_source == "self-reported" - else "Answer choices do not prove execution." - ) - reason = ( - "" - if report.metadata.trace_source == "self-reported" - else "Answer summaries do not prove execution." + else "Final answers do not prove execution." ) note = ( "Model-extracted counts describe trials, not repeated actions. " @@ -287,6 +309,7 @@ def trial_evidence_note(self_reported): return ( "Before and After trials are independent, even when their trial numbers match. " + (SELF_REPORTED_LIMIT if self_reported else RECORDED_COMMAND_LIMIT) + + " Short summaries interpret each group's records; they are not causal proof." ) @@ -294,6 +317,24 @@ def trial_anchor(side, name): return "trial-{0}-{1}".format(side, name.encode("utf-8").hex()) +def trial_group_anchor(index): + return "trial-group-{0}".format(index) + + +def trial_summary_for_group(decisions, before, after): + """Return only one saved summary matching both exact group identities.""" + identity = ( + before.name if before is not None else "", + after.name if after is not None else "", + ) + matches = [ + summary + for summary in decisions.trial_summaries + if (summary.before_trial, summary.after_trial) == identity + ] + return matches[0] if len(matches) == 1 else None + + def command_progressions(variant): """Group complete recorded sequences without normalizing or joining paths.""" groups = {} @@ -437,7 +478,7 @@ def result_data(metadata, variants, decisions, expected): if answer_details_changed: text = "Final result unchanged; answer details changed" if answer_details_changed: - summary += " Other answer details changed. The Decision diff preserves these comparisons." + summary += " Other answer details changed. The Behavior diff preserves these comparisons." if behavior_status == "unchanged": summary += " No change was observed in the compared actions." if not explicit: @@ -535,7 +576,7 @@ def _evidence_limits(metadata, variants, decisions, expected): ) ): missing.append( - "{0} decision counts disagree with the trial count.".format( + "{0} comparison counts disagree with the trial count.".format( variant.label ) ) @@ -628,12 +669,12 @@ def source_label(anchor, trace_source): def tag_legend(trace_source): """What each tag on a decision row means.""" return ( - ("changed", "Changed", "extracted behavior proportions differ between sides"), - ("same", "Unchanged", "complete trials show the same unanimous behavior"), + ("changed", "Changed", "the compared answers or actions differ between sides"), + ("same", "Unchanged", "complete trials show the same answer or action"), ( "same", "Same proportions", - "proportions match, but behaviors are mixed or trial evidence is incomplete", + "counts have matching proportions, but answers or actions vary or evidence is incomplete", ), ( "unavailable", @@ -664,7 +705,7 @@ def headings(target_file): "scenario": "What we simulated", "expected": "Expected behavior", "diff": "Diff of {0}".format(target_file), - "decision": "Decision diff: actions and answers compared", + "decision": "Behavior diff: actions and answers compared", "flow": "Flow diff: recorded commands", "result": "Result", } @@ -673,14 +714,12 @@ def headings(target_file): def decision_footer(rows): changed = sum(decision_status(row) == "Changed" for row in rows) unavailable = sum(decision_status(row) == "Unavailable" for row in rows) - summary = "{0} of {1} comparisons have different behavior proportions.".format( + summary = "{0} of {1} comparisons differ between Before and After.".format( changed, len(rows) ) if unavailable: - summary += ( - " {0} comparisons lack usable behaviors on one or both sides.".format( - unavailable - ) + summary += " {0} comparisons lack usable extracted evidence on one or both sides.".format( + unavailable ) return summary diff --git a/plugin/skills/behavior-diff/scripts/reporting/load.py b/plugin/skills/behavior-diff/scripts/reporting/load.py index 654e50a..05f7d20 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/load.py +++ b/plugin/skills/behavior-diff/scripts/reporting/load.py @@ -24,6 +24,7 @@ VariantsData, ) from reporting.summary import build_intent, build_summary, parse_intent, parse_narrative +from reporting.trial_summary import parse_trial_summaries, trial_group_names def load_report( @@ -38,7 +39,14 @@ def load_report( command_flow = _command_flow(before, after, metadata) instruction_diff = rule_diff(run, capsule, metadata.target_file) decisions = _read_decisions( - run, command_flow.before.total, command_flow.after.total, instruction_diff + run, + command_flow.before.total, + command_flow.after.total, + instruction_diff, + trial_group_names( + (trial.name for trial in before.trials), + (trial.name for trial in after.trials), + ), ) task = _task(run, capsule) report_content = content.build_content( @@ -127,7 +135,7 @@ def _variants(run, grades, metadata): def _trial(run, name, grade, metadata): verdict, actions = grade - commands, final = _read_trace(run / name / "trace.jsonl") + commands, final = read_trial_trace(run / name / "trace.jsonl") return TrialData( name=name, verdict=verdict, @@ -138,7 +146,8 @@ def _trial(run, name, grade, metadata): ) -def _read_trace(path): +def read_trial_trace(path): + """Read recorded entries and the last final answer without inventing evidence.""" commands = [] final = "" if not path.exists(): @@ -364,18 +373,20 @@ def _common_prefix(sequences): return tuple(prefix) -def _read_decisions(run, before_default, after_default, instruction_diff): +def _read_decisions(run, before_default, after_default, instruction_diff, groups=()): path = run / "decisions.json" if not path.exists(): return _empty_decisions(before_default, after_default) try: raw = json.loads(path.read_text()) - return _convert_decisions(raw, before_default, after_default, instruction_diff) + return _convert_decisions( + raw, before_default, after_default, instruction_diff, groups + ) except (TypeError, ValueError, KeyError): return _empty_decisions(before_default, after_default) -def _convert_decisions(raw, before_default, after_default, instruction_diff): +def _convert_decisions(raw, before_default, after_default, instruction_diff, groups=()): if type(raw) is not dict or type(raw.get("chain")) is not list: raise ValueError("malformed decisions") counts = raw.get("counts", {}) @@ -439,6 +450,7 @@ def _convert_decisions(raw, before_default, after_default, instruction_diff): implications, parse_narrative(raw.get("summary"), rows), parse_intent(raw.get("intent"), hunk_count), + parse_trial_summaries(raw.get("trial_summaries"), groups), ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/render_html.py b/plugin/skills/behavior-diff/scripts/reporting/render_html.py index 627bf6a..79404b0 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/render_html.py +++ b/plugin/skills/behavior-diff/scripts/reporting/render_html.py @@ -5,6 +5,7 @@ from reporting import content from reporting.illustrations import illustration +from reporting.instruction import parse_diff_hunks from reporting.schema import ReportData @@ -16,29 +17,128 @@ def _diff_line_class(line: str) -> str: return "d-ctx" -def _trial_card(trial, self_reported: bool, mode: str, side: str) -> str: +def _trial_cell(trial, side: str, body: str, *, anchor: bool = False) -> str: + identity = ( + f' id="{html.escape(content.trial_anchor(side, trial.name))}" tabindex="-1"' + if trial and anchor + else "" + ) + if not trial: + body = '

No trial record on this side.

' + return ( + f'
' + f"{body}" + "
" + ) + + +def _trial_group(index, before, after, self_reported: bool, mode: str, summary) -> str: escaped = html.escape - verdict_class = escaped(trial.verdict.lower()) + trials = (("before", before), ("after", after)) + metadata = [] + answers = [] + actions = [] + commands = [] action_heading, empty_actions = content.trial_action_labels(self_reported) - evidence = ( - "\n\n".join( - command if self_reported else "$ " + command for command in trial.commands + for side, trial in trials: + if trial: + header = ( + f'

{escaped(trial.name)}' + f'' + f"{escaped(trial.verdict)}

" + ) + answer = ( + f"
{escaped(trial.final)}
" + if trial.final.strip() + else f'

{escaped(content.NO_FINAL_ANSWER)}

' + ) + action = ( + f'

{escaped(trial.actions)}

' + if trial.actions and trial.actions != "-" + else '

No action summary recorded.

' + ) + evidence = ( + "\n\n".join( + command if self_reported else "$ " + command + for command in trial.commands + ) + or empty_actions + ) + command_body = ( + f'

{len(trial.commands)} ' + f"{'self-reported actions' if self_reported else 'recorded commands'}

" + f"
{escaped(evidence)}
" + ) + else: + header = answer = action = command_body = "" + metadata.append(_trial_cell(trial, side, header, anchor=True)) + answers.append( + _trial_cell( + trial, + side, + f'
{answer}
', + ) + ) + actions.append(_trial_cell(trial, side, action)) + commands.append(_trial_cell(trial, side, command_body)) + summary_intro = ( + f'

{escaped(summary.takeaway)}

' + if summary is not None + else f'

{escaped(content.TRIAL_SUMMARY_UNAVAILABLE)}

' + ) + descriptions = ( + "".join( + _trial_cell( + trial, + side, + f'

{escaped(sentence)}

', + ) + for (side, trial), sentence in zip(trials, (summary.before, summary.after)) ) - or empty_actions + if summary is not None + else "" ) - actions = ( - "" if trial.actions == "-" else f'

{escaped(trial.actions)}

' + caveat = ( + f'

{escaped(summary.caveat)}

' + if summary is not None and summary.caveat + else "" ) + group = content.trial_group_anchor(index) return ( - f'
' - f'

{escaped(trial.name)}' - f'{escaped(trial.verdict)}

{actions}' - f"
" - f"{escaped(action_heading)} " - f"({len(trial.commands)})
{escaped(evidence)}
" - f"
" - f"{escaped(content.FINAL_ANSWER_HEADING)}" - f"
{escaped(trial.final if trial.final.strip() else content.NO_FINAL_ANSWER)}
" + f'
' + f'

Trial {index}

' + f'' + '
' + f'

{escaped(content.TRIAL_CHANGE_HEADING)}

' + f"{summary_intro}" + '

Before

' + '

After

' + f"{descriptions}{caveat}" + '

Trial record and verdict

' + f"{''.join(metadata)}" + f'

{escaped(content.FINAL_ANSWER_HEADING)}

' + f"{''.join(answers)}
" + f'
" + "Supporting details · Before and After" + f'
{"".join(actions)}
' + f'
" + f"{escaped(action_heading)} · Before and After" + f'
{"".join(commands)}
' + "
" + ) + + +def _trial_group_links(report) -> str: + count = max(len(report.variants.before.trials), len(report.variants.after.trials)) + return "".join( + f'Trial {index}' + for index in range(1, count + 1) ) @@ -50,13 +150,15 @@ def _decision_choices(choices, total: int) -> str: return "".join(lines) or html.escape(content.NO_EXTRACTED_CHOICE) -def _edit_links(row) -> str: +def _edit_links(row, report) -> str: if not row.edit_hunks: return ( 'Related edit unavailable — no mapping recorded.' ) links = " · ".join( - f'Hunk {number}' for number in row.edit_hunks + f'' + f"{html.escape(content.instruction_hunk_label(report.rule_diff, number))}" + for number in row.edit_hunks ) return ( f'Related edit (model interpretation): {links}' @@ -65,21 +167,50 @@ def _edit_links(row) -> str: def _instruction_edit(report) -> str: lines = [] + blocks = [] + current = None + remaining = 0 + hunks = {hunk.number: hunk for hunk in parse_diff_hunks(report.rule_diff)} for line, number in content.instruction_diff_lines(report.rule_diff): + in_body = current is not None and remaining > 0 + if number: + current = number + remaining = len(hunks[number].lines) + in_body = False + label = content.instruction_hunk_label(report.rule_diff, number) + blocks.append( + f'" + ) anchor = f' id="edit-hunk-{number}" tabindex="-1"' if number else "" + block = f' data-edit-hunk="{current}"' if in_body or number else "" + css_class = _diff_line_class(line) if in_body else "d-ctx" lines.append( - f'{html.escape(line)}\n' + f'{html.escape(line)}\n' ) + if in_body: + remaining -= 1 + controls = ( + '" + '

' + if blocks + else "" + ) + aim_links = _edit_links(report.intent, report) if report.intent.edit_hunks else "" return ( '
' '
' f'

{html.escape(report.content.diff_heading)}

' - f"{_info('edit-info', 'About this edit', f'

{html.escape(content.EDIT_LINK_NOTE)}

')}" + f"{_info('edit-info', 'About these instruction changes', f'

{html.escape(content.EDIT_LINK_NOTE)}

')}" "
" f'

{html.escape(content.instruction_edit_aim(report.intent))}

' + f"{aim_links}" f'

{html.escape(content.instruction_edit_counts(report.rule_diff))}

' - f"
{''.join(lines)}
" + f"{controls}
{''.join(lines)}
" + '

+ added · − removed · unmarked context. ' + "Selected blocks highlight changed lines, not behavior evidence.

" ) @@ -103,7 +234,8 @@ def _decision_progression(report) -> str: choices = ( '' + "".join( - f'{label}' + f'' + f'{label}' f"{_decision_choices(values, total)}" for label, values, total in ( ("Before", row.before, decisions.before_count), @@ -126,21 +258,28 @@ def _decision_progression(report) -> str: '' f'{html.escape(title)}' f'{html.escape(status)}' - f'{html.escape(role)}' + f'{html.escape(role)}' + f'{html.escape(source)}' 'Show evidence' f'Hide evidence{_CHEVRON}' f"{choices}" f'
{question}' f'

{html.escape(source)}

' - f"{_edit_links(row)}{note}" - 'Inspect trial evidence' + f"{_edit_links(row, report)}{note}" + '
View supporting trial records' + f'' + '

These are the available source records. The saved comparison does not ' + "map each extracted behavior to individual trials.

" "
" ) return ( '
' '
' - '

Decision sequence

' - '
' + '

Behavior comparisons

' + '" f'

{html.escape(content.DECISION_PROGRESSION_NOTE)}

' f'

{html.escape(content.TRIAL_COUNT_NOTE)}

' '
    ' @@ -150,66 +289,127 @@ def _decision_progression(report) -> str: _REPORT_INTERACTIONS = """""" @@ -234,7 +434,10 @@ def _flow_lane(variant, side: str, label: str) -> str: sequence = ( f'
      {steps}
    ' if commands - else f'

    {html.escape(content.NO_COMMANDS_RECORDED)}

    ' + else ( + f'

    {html.escape(content.NO_COMMANDS_RECORDED)}. ' + "This does not establish that no actions occurred.

    " + ) ) paths.append( '
    ' @@ -245,7 +448,7 @@ def _flow_lane(variant, side: str, label: str) -> str: ) body = "".join(paths) or '

    No trials

    ' return ( - f'
    ' + f'
    ' f'

    {html.escape(label)}

    {body}
    ' ) @@ -256,11 +459,12 @@ def _other_findings(report: ReportData) -> str: for index, text in findings: row = report.decisions.rows[index - 1] source = content.source_label(row.anchor, report.metadata.trace_source) - links = ( - f'Decision {index} · {html.escape(source)}' - ) + links = f'Comparison {index} · {html.escape(source)}' for number in row.edit_hunks: - links += f' · See edit {number}' + links += ( + f' · ' + f"{html.escape(content.instruction_hunk_label(report.rule_diff, number))}" + ) items.append( f"
  1. {html.escape(text)}

    " f'{links}
  2. ' @@ -272,7 +476,7 @@ def _other_findings(report: ReportData) -> str: else f'

    {html.escape(content.NO_ADDITIONAL_FINDINGS)}

    ' ) link = ( - 'View all decisions' + 'View all behavior comparisons' if report.decisions.rows else 'Inspect trial evidence' ) @@ -281,7 +485,7 @@ def _other_findings(report: ReportData) -> str: def _decision_links(indexes) -> str: return " · ".join( - f'Decision {index}' for index in indexes + f'Comparison {index}' for index in indexes ) @@ -303,7 +507,7 @@ def _summary_side(side, label: str) -> str: ) return ( f'
    ' + f'aria-label="{label} observed behavior">' f'

    {label}

    ' f"{illustration(side.icon)}{body}
    " ) @@ -318,14 +522,7 @@ def _short_story_summary(report: ReportData) -> str: "inferred": "Inferred", "unavailable": "Unavailable", }[intent.source] - intent_links = ( - " · ".join( - f'See edit {number}' - for number in intent.edit_hunks - ) - if intent.edit_hunks - else 'See edit' - ) + intent_links = 'View instruction changes' claims = "" for label, claim, css_class in ( ("Why it matters", summary.why, "summary-why"), @@ -334,15 +531,20 @@ def _short_story_summary(report: ReportData) -> str: if claim: claims += ( f'
    {label}' - f"

    {html.escape(claim.text)}" + f"

    {html.escape(claim.text)}

    " f'{_decision_links(claim.decisions)}' - "

    " + "" ) + comparison_available = ( + summary.decision is not None and summary.status != "unavailable" + ) + destination = ( + f"#decision-{summary.decision}" if comparison_available else "#panel-trials" + ) lead_link = ( - f'' - 'See the evidence ' - if summary.decision is not None - else "" + f'' + f"{'View this comparison' if comparison_available else 'View available trial records'}" + ' ' ) notices = "".join(f"
  3. {html.escape(note)}
  4. " for note in summary.notices) scenario = ( @@ -355,7 +557,7 @@ def _short_story_summary(report: ReportData) -> str: f'

    {html.escape(summary.headline)}

    ' '
      ' '
    1. ' - '

      Edit goal

      ' + '

      The intended change

      ' f'{intent_badge}' f"{_info('intent-info', intent_label, f'

      {html.escape(intent_note)}

      ')}" f'' @@ -363,7 +565,7 @@ def _short_story_summary(report: ReportData) -> str: f'

      {html.escape(intent.text)}

      ' "
    2. " '
    3. ' - '

      What happened in this scenario

      ' + '

      What the evidence shows

      ' f"{scenario}" f'

      {html.escape(summary.evidence_label)}

      ' f'

      Comparison: {html.escape(summary.status)}

      ' @@ -372,7 +574,7 @@ def _short_story_summary(report: ReportData) -> str: '' f"{_summary_side(summary.after, 'After')}
      " '' + f"{lead_link}" "
    4. " '
    5. ' '

      What this means

      ' @@ -426,7 +628,7 @@ def _info(pop_id: str, label: str, body: str) -> str: """An info icon whose popover opens on hover, focus, or tap.""" return ( f'
      ' + f'aria-describedby="{pop_id}" aria-controls="{pop_id}" aria-expanded="false">{_INFO_ICON}' f'
      ' f'

      {html.escape(label)}

      {body}
      ' ) @@ -469,37 +671,52 @@ def render_artifact(report: ReportData, css: str) -> str: after = report.variants.after flow = report.command_flow - before_note = f'{escaped(before.note)}' - after_note = f'{escaped(after.note)}' - runs = "" - for index, (before_trial, after_trial) in enumerate( - zip_longest(before.trials, after.trials), 1 - ): - halves = "" - for trial, css_class in ((before_trial, "b"), (after_trial, "a")): - side = "before" if css_class == "b" else "after" - body = ( - _trial_card(trial, self_reported, metadata.mode, side) - if trial - else '

      (no trial on this side)

      ' + trial_groups = ( + "".join( + _trial_group( + index, + before_trial, + after_trial, + self_reported, + metadata.mode, + content.trial_summary_for_group( + report.decisions, before_trial, after_trial + ), ) - halves += ( - f'
      ' - f'

      {side.capitalize()}

      {body}
      ' + for index, (before_trial, after_trial) in enumerate( + zip_longest(before.trials, after.trials), 1 ) - runs += f'

      Trial {index}

      {halves}
      ' + ) + or '

      No trial records are available.

      ' + ) + trial_overview = "".join( + f'

      {side.capitalize()}

      ' + f'

      {escaped(variant.note)}

      ' + f'

      {escaped(variant.count_text + variant.count_suffix)}

      ' + for side, variant in (("before", before), ("after", after)) + ) trials_html = ( - f'' + f'' f'

      {escaped(content.trial_evidence_note(self_reported))}

      ' - f'
      ' - f'

      Before

      {before_note}' - f'{escaped(before.count_text + before.count_suffix)}
      ' - f'

      After

      {after_note}' - f'{escaped(after.count_text + after.count_suffix)}
      ' - f"
      {runs}" + '

      Final answers are visible below. Each numbered comparison aligns ' + "Before on the left and After on the right. Supporting-detail disclosures open both " + "sides together; matching numbers do not establish paired execution.

      " + f'
      {trial_overview}
      ' + f'' + f"{trial_groups}" ) - decisions_html = "" + decisions_html = ( + '
      ' + f'

      {escaped(report_content.decision_heading)}

      ' + f"{_info('pop-decision', 'Behavior comparison labels', _tag_legend(report_content.tag_legend))}
      " + '

      Behavior comparison unavailable

      ' + f"

      {escaped(content.NO_EXTRACTED_CHOICE)}

      " + "

      Missing extraction does not establish absent behavior. " + "The retained trial records remain available for inspection.

      " + f'

      {escaped(report.result.summary)}

      ' + 'View available trial records' + ) if report.decisions.rows: decisions = report.decisions footer = content.decision_footer(decisions.rows) @@ -528,19 +745,31 @@ def render_artifact(report: ReportData, css: str) -> str: if self_reported else "" ) + if not content.complete_trial_evidence(report): + limitation += ( + '

      Trial evidence is incomplete. The extracted comparisons below ' + "are retained for inspection, not a complete Before/After conclusion.

      " + ) decisions_html = ( - f'" + '
      ' + f'

      {escaped(report_content.decision_heading)}

      ' + f"{_info('pop-decision', 'Behavior comparison labels', _tag_legend(report_content.tag_legend))}
      " f'

      {escaped(content.decision_overview(report))}

      ' f'

      {escaped(report_content.decision_blurb)}

      ' f"{limitation}" f"{_decision_progression(report)}" f'

      {escaped(footer)}

      {explanation}' - '' + '' ) - flow_section = "" + flow_section = ( + '' + '

      Recorded command flow unavailable

      ' + f"

      {escaped(content.SELF_REPORTED_LIMIT)}

      " + "

      No command sequence is inferred from answer text or self-reported actions.

      " + 'View available trial records' + ) if not self_reported: patterns = content.flow_patterns(flow) pattern_rows = [] @@ -576,15 +805,16 @@ def render_artifact(report: ReportData, css: str) -> str: + "" ) flow_links = ( - 'Compare decisions' + 'Compare behavior' if report.decisions.rows else "" ) flow_section = ( - f'