From b813d54a70a224e1ec2c4b586526d296cb879216 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 4 Oct 2026 13:55:41 +0000 Subject: [PATCH 1/3] perf(build,ship): scope local gates to the change's impact, defer the rest to CI What: add scripts/impact-scope.py (dev and factory), which lists the files changed since a base, the packages that own them, and a verdict: docs, impacted, or full (a lockfile, root config, toolchain pin, or CI workflow changed). ci-parity.md now scopes every local gate by that verdict: build's wave gate runs the Validation block at the wave's impact, the gauntlet and remediation re-runs at the fixes' impact, and the CI-parity runs at the branch's impact, recording what lies outside it as deferred to CI. A PR that carries a deferred check opens as a draft and is marked ready once its required checks are green; a run that opens no PR still runs the full set. Why: on a grown repository the full merge gate ran up to three times per change (wave gates, build's parity run, ship's parity run) and dominated wall clock, while the PR's CI runs it again anyway. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CJNyTNFr77RZj8omURZxyA --- dev/.claude-plugin/plugin.json | 2 +- dev/README.md | 4 +- dev/evals/README.md | 2 +- dev/evals/build.json | 21 ++- dev/evals/ship.json | 5 +- dev/evals/tests/test_impact_scope.py | 123 +++++++++++++ dev/references/ci-parity.md | 57 +++++- dev/scripts/impact-scope.py | 168 ++++++++++++++++++ dev/skills/build/SKILL.md | 5 +- dev/skills/build/references/parallel.md | 5 +- dev/skills/ship/SKILL.md | 4 +- dev/skills/ship/references/gauntlet.md | 11 +- dev/skills/ship/references/pull-request.md | 8 +- dev/skills/ship/references/remediation.md | 4 +- factory/.claude-plugin/plugin.json | 2 +- .../evals/tests/test_protocol_and_copies.py | 2 +- factory/phases/build/SKILL.md | 2 +- factory/phases/build/references/parallel.md | 5 +- factory/phases/ship/SKILL.md | 4 +- factory/phases/ship/references/gauntlet.md | 11 +- .../phases/ship/references/pull-request.md | 8 +- factory/phases/ship/references/remediation.md | 4 +- factory/references/ci-parity.md | 57 +++++- factory/scripts/impact-scope.py | 168 ++++++++++++++++++ package.json | 2 +- plugins/dev/.codex-plugin/plugin.json | 2 +- plugins/dev/references/ci-parity.md | 57 +++++- plugins/dev/scripts/impact-scope.py | 168 ++++++++++++++++++ plugins/dev/skills/build/SKILL.md | 5 +- .../dev/skills/build/references/parallel.md | 5 +- plugins/dev/skills/ship/SKILL.md | 4 +- .../dev/skills/ship/references/gauntlet.md | 11 +- .../skills/ship/references/pull-request.md | 8 +- .../dev/skills/ship/references/remediation.md | 4 +- plugins/factory/.codex-plugin/plugin.json | 2 +- plugins/factory/phases/build/SKILL.md | 2 +- .../phases/build/references/parallel.md | 5 +- plugins/factory/phases/ship/SKILL.md | 4 +- .../phases/ship/references/gauntlet.md | 11 +- .../phases/ship/references/pull-request.md | 8 +- .../phases/ship/references/remediation.md | 4 +- plugins/factory/references/ci-parity.md | 57 +++++- plugins/factory/scripts/impact-scope.py | 168 ++++++++++++++++++ 43 files changed, 1104 insertions(+), 105 deletions(-) create mode 100644 dev/evals/tests/test_impact_scope.py create mode 100755 dev/scripts/impact-scope.py create mode 100755 factory/scripts/impact-scope.py create mode 100755 plugins/dev/scripts/impact-scope.py create mode 100755 plugins/factory/scripts/impact-scope.py diff --git a/dev/.claude-plugin/plugin.json b/dev/.claude-plugin/plugin.json index 5733a85..530161c 100644 --- a/dev/.claude-plugin/plugin.json +++ b/dev/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "dev", - "version": "3.5.0", + "version": "3.6.0", "description": "Development workflow skills: scope changes with argued decisions, build across unit/integration/e2e with every scenario proven by tests, ship with a deterministic quality gauntlet and an adversarially verified review, create structured commits that feed a decision ledger, and render pitches or comprehension quizzes.", "author": { "name": "Tobrun" diff --git a/dev/README.md b/dev/README.md index ada9de0..5b3b854 100644 --- a/dev/README.md +++ b/dev/README.md @@ -32,7 +32,8 @@ Executes a spec's change sets at the layer each `tests:` scenario is tagged with Enforces outcomes rather than rituals: every scenario gets a real test at its tagged layer, written with the code and run to green, with no red step. Failing-test-first is reserved for bug fixes, where one red run is the proof the issue was actually reproduced. Runs independent change sets in parallel as waves of subagents batched by disjoint file lists in spec order, committing each change set and appending to a running `implementation-notes.md` that logs any deviations forced by an edge case. -Each subagent starts from a brief that `scripts/change-set-brief.py` cuts from the spec for its change set and runs only its own tests; the spec's Validation block runs once per wave as the gate before its commits, and e2e, benchmark, and coverage commands run once at the end. +Each subagent starts from a brief that `scripts/change-set-brief.py` cuts from the spec for its change set and runs only its own tests; the spec's Validation block runs once per wave as the gate before its commits, scoped by `scripts/impact-scope.py` to the packages the wave reaches, and e2e, benchmark, and coverage commands run once at the end. +The repository's required pull-request commands run locally only at the branch's impact; what lies outside it is deferred to the pull request's CI, unless a lockfile, root config, toolchain pin, or CI workflow changed, which makes every package impacted. Once every change set is committed, drives the real app against a mocked environment, loops until every e2e scenario passes, then renders the e2e report: screenshots per scenario for frontend systems, Test Scenario and Data Model State tables for everything else. Build never pushes or opens a PR; `ship` does, once the change is hardened and reviewed. @@ -50,6 +51,7 @@ Claude Code, Codex, and opencode use their native parallel subagent facilities. Renders `review_N.html` alongside the `review_N.md` file for reviewer handoff. Phase 3 commits what the gauntlet fixed, pushes, and opens the pull request automatically - a draft when the review verdict is BLOCK - with an Evidence section that shows the change working: screenshots from the e2e run for a system with a frontend, published to a `pr-evidence` branch so they render inline, or labeled before/after state otherwise, and for every bug fix the reproducing test shown red on the merge base and green on the branch. A deterministic check (`pr-evidence.py check`) gates the PR body, so a PR cannot open on a placeholder or a data URI, and the phase then follows required checks to green. +Local gates in every phase run at the change's impact, so the full merge gate is the PR's own CI: a PR carrying a check deferred to CI opens as a draft and is marked ready once its required checks pass, and a run that opens no PR runs the full set locally. ### commit diff --git a/dev/evals/README.md b/dev/evals/README.md index 12c3ece..38a00f2 100644 --- a/dev/evals/README.md +++ b/dev/evals/README.md @@ -6,7 +6,7 @@ Eval definitions for the `dev` plugin's skills: realistic prompts and objective - `{skill}.json` - one file per skill: the eval prompt(s), the fixture each expects, and the assertions to grade the output against. Covers all 7 skills: `scope`, `scope-review`, `commit`, `build`, `ship`, `to-pitch`, `to-quiz`. - `results.md` - the record of the most recent full run: scores, methodology, and findings. -- `tests/` - unit tests for the deterministic scripts the skills loop against (`lint-spec.py`, `change-set-brief.py`, `check-tests.py`), run by `scripts/validate.sh` as check D01. +- `tests/` - unit tests for the deterministic scripts the skills loop against (`lint-spec.py`, `change-set-brief.py`, `check-tests.py`, `impact-scope.py`), run by `scripts/validate.sh` as check D01. `build` runs in `"functional"` mode (a real fixture, a real subagent run, assertions checked against the actual output). `scope`, `scope-review`, `commit`, `ship`, `to-pitch`, and `to-quiz` run in `"comprehension"` mode instead - each depends on either an interactive question loop, a live codebase, or prior artifacts (a finished spec, implementation notes, an e2e report) that are too expensive to stage on every iteration, so these check policy comprehension of the skill text directly. diff --git a/dev/evals/build.json b/dev/evals/build.json index b6ac0a8..78b8117 100644 --- a/dev/evals/build.json +++ b/dev/evals/build.json @@ -80,17 +80,28 @@ { "id": "ci-parity-before-pr", "prompt": "Implement this spec.", - "fixture": "a repo whose spec Validation block runs unit/lint/build, while .github/workflows/pr.yml also requires a project-owned screenshot-matrix command. The active-month screenshot fails deterministically on the first UTC day, although the feature's tagged e2e scenarios pass.", + "fixture": "a repo whose spec Validation block runs unit/lint/build, while .github/workflows/pr.yml also requires a project-owned screenshot-matrix command over the UI package. The spec changes that UI package, and the active-month screenshot fails deterministically on the first UTC day, although the feature's tagged e2e scenarios pass.", "assertions": [ - "The pull-request workflow is inspected and the screenshot-matrix command is run before the user is offered a PR", + "The pull-request workflow is inspected and impact-scope.py is run before the required commands, and the screenshot-matrix command runs before the user is offered a PR because the change reaches its inputs", "A required command that already ran green on the unchanged final tree is recorded, not run a second time", - "The screenshot failure is diagnosed and fixed rather than labeled pre-existing, flaky, unrelated, or an accepted deviation", + "The screenshot failure is diagnosed and fixed rather than labeled pre-existing, flaky, unrelated, deferred to CI, or an accepted deviation", "The fix removes the time-dependent assumption without adding a retry, sleep, timeout increase, or looser assertion", - "The full screenshot-matrix command is rerun and green before the PR question", - "The discovered CI commands and outcomes are recorded in implementation-notes.md, each as its command finishes", + "The screenshot-matrix command is rerun and green before the PR question", + "The discovered CI commands, the scope each ran at, and their outcomes are recorded in implementation-notes.md, each as its command finishes", "If the user approves a PR, required checks are watched to a terminal state and a deterministic failure is fixed and pushed rather than merely reported as restarted" ] }, + { + "id": "impact-scoped-gates", + "prompt": "Implement this spec.", + "fixture": "an npm-workspaces repo with packages/api and packages/web, each with its own test script, and a pr.yml that runs every workspace's tests; the spec's two change sets edit only packages/api", + "assertions": [ + "Each wave gate runs impact-scope.py --base HEAD and runs the Validation commands over packages/api (and any workspace that depends on it), never the packages/web suite", + "The final CI-parity run is scoped by impact-scope.py against the merge base, and the packages/web tests are recorded in implementation-notes.md as deferred to CI: outside impact rather than run or silently dropped", + "No required command whose inputs the change reaches is deferred because it is slow", + "The closing message points at ship, whose PR's CI runs the deferred commands" + ] + }, { "id": "jira-disabled", "prompt": "Implement this spec.", diff --git a/dev/evals/ship.json b/dev/evals/ship.json index 271f3f4..91ad2ef 100644 --- a/dev/evals/ship.json +++ b/dev/evals/ship.json @@ -5,7 +5,7 @@ "evals": [ { "id": "gauntlet-policy", - "prompt": "Answer briefly, from the skill text only: (1) A mutation run leaves three surviving mutants and fixing them is tedious. May the fix agent delete the flaky-looking tests so the mutants no longer count, or lower the threshold to pass? (2) docs/dependencies.md forbids ui -> infra but the change needs data from infra. What are the legitimate fixes, and who may edit the rules file? (3) The repo has no mutation framework. Do you skip mutation testing, hand-roll a full harness, or something else? (4) The complexity threshold of 6 seems too strict for this codebase and the user agrees to 8. What must happen besides using 8? (5) What is the default scope of a run, and when does it widen? (6) A violation has survived two fix rounds on the same root cause. What happens next? (7) After the gauntlet passes, is the change verified? What happens next? (8) The spec E2E is green, but a required PR screenshot workflow is red and the same scenario was previously called flaky. Can ship proceed to review or call the diff PR-ready?", + "prompt": "Answer briefly, from the skill text only: (1) A mutation run leaves three surviving mutants and fixing them is tedious. May the fix agent delete the flaky-looking tests so the mutants no longer count, or lower the threshold to pass? (2) docs/dependencies.md forbids ui -> infra but the change needs data from infra. What are the legitimate fixes, and who may edit the rules file? (3) The repo has no mutation framework. Do you skip mutation testing, hand-roll a full harness, or something else? (4) The complexity threshold of 6 seems too strict for this codebase and the user agrees to 8. What must happen besides using 8? (5) What is the default scope of a run, and when does it widen? (6) A violation has survived two fix rounds on the same root cause. What happens next? (7) After the gauntlet passes, is the change verified? What happens next? (8) The spec E2E is green, but a required PR screenshot workflow is red and the same scenario was previously called flaky. Can ship proceed to review or call the diff PR-ready? (9) The diff edits only packages/api of an npm-workspaces repo, and pr.yml runs every workspace's tests. Which of those does the CI-parity exit run locally, what happens to the rest, and how does phase 3 open the PR? What changes if the diff also touched package-lock.json, or if the user asked for no PR?", "fixture": "none - policy questions answered from the skill text", "assertions": [ "Answer 1: no on both; agents never delete tests to moot a mutant, never weaken a check or edit thresholds to pass", @@ -15,7 +15,8 @@ "Answer 5: the branch diff against the default branch; full-repo only when the user asks", "Answer 6: it becomes a human call - presented as a real defect, a threshold worth changing, or a rule the spec should have amended; the loop does not grind on", "Answer 7: mechanics only, not meaning; by default the review phase runs next over the post-fix diff, then the pull request phase; commit is recommended at wrap up only when phase 3 did not run, never invoked", - "Answer 8: no; ship always runs required PR commands before review, the red screenshot is a violation even if called flaky or pre-existing, and pre-existing status requires merge-base proof plus a human call" + "Answer 8: no; ship always runs required PR commands before review, the red screenshot is a violation even if called flaky or pre-existing, and pre-existing status requires merge-base proof plus a human call", + "Answer 9: impact-scope.py scopes the run, so packages/api's tests (and any dependent workspace's) run locally and the packages/web tests are recorded deferred to CI on pr.md's CI parity line; the PR opens as a draft and is marked ready once its required checks are green; a lockfile change makes the verdict full, and a no-PR run has no CI to defer to, so both run the full set locally" ] }, { diff --git a/dev/evals/tests/test_impact_scope.py b/dev/evals/tests/test_impact_scope.py new file mode 100644 index 0000000..17cbf14 --- /dev/null +++ b/dev/evals/tests/test_impact_scope.py @@ -0,0 +1,123 @@ +"""Unit tests for impact-scope.py, the check build and ship loop against to +decide whether a local gate runs at the impacted scope or in full. + +Run from the repo root: python3 -m unittest discover -s dev/evals/tests -t . +Each test builds a scratch git repository and runs the script as a subprocess, +so it proves the CLI contract the skills read. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCRIPT = REPO_ROOT / "dev" / "scripts" / "impact-scope.py" + +BASE_TREE = { + "package.json": "{}", + "package-lock.json": "{}", + "tsconfig.json": "{}", + "README.md": "# repo", + "packages/api/package.json": "{}", + "packages/api/src/index.ts": "export {}", + "packages/web/package.json": "{}", + "packages/web/src/app.ts": "export {}", + "scripts/release.sh": "echo", + ".github/workflows/pr.yml": "on: pull_request", +} + + +def git(cwd: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True) + + +class ImpactScopeTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.repo = Path(self.tmp.name) + git(self.repo, "init", "-q", "-b", "main") + git(self.repo, "config", "user.email", "t@example.com") + git(self.repo, "config", "user.name", "t") + for path, body in BASE_TREE.items(): + self.write(path, body) + git(self.repo, "add", "-A") + git(self.repo, "commit", "-q", "-m", "base") + git(self.repo, "checkout", "-q", "-b", "work") + + def tearDown(self) -> None: + self.tmp.cleanup() + + def write(self, path: str, body: str) -> None: + target = self.repo / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + + def scope(self, *args: str) -> dict: + result = subprocess.run( + [sys.executable, str(SCRIPT), "--json", *args], + cwd=self.repo, check=False, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + def test_package_source_change_is_impacted_on_that_package(self) -> None: + self.write("packages/api/src/index.ts", "export const a = 1") + report = self.scope() + self.assertEqual(report["verdict"], "impacted") + self.assertEqual(report["packages"], ["packages/api"]) + + def test_untracked_file_counts(self) -> None: + self.write("packages/web/src/new.ts", "export {}") + report = self.scope() + self.assertEqual(report["packages"], ["packages/web"]) + + def test_file_outside_any_package_belongs_to_root(self) -> None: + self.write("scripts/release.sh", "echo 2") + self.assertEqual(self.scope()["packages"], ["."]) + + def test_docs_only_change_owes_no_run(self) -> None: + self.write("README.md", "# changed") + self.write("docs/guide.md", "new") + self.assertEqual(self.scope()["verdict"], "docs") + + def test_no_change_is_docs(self) -> None: + self.assertEqual(self.scope()["verdict"], "docs") + + def test_lockfile_widens_to_full(self) -> None: + self.write("package-lock.json", '{"v": 2}') + report = self.scope() + self.assertEqual(report["verdict"], "full") + self.assertIn("package-lock.json", report["reasons"][0]) + + def test_ci_workflow_widens_to_full(self) -> None: + self.write(".github/workflows/pr.yml", "on: push") + self.assertEqual(self.scope()["verdict"], "full") + + def test_root_config_and_root_manifest_widen_to_full(self) -> None: + self.write("tsconfig.json", '{"strict": true}') + self.assertEqual(self.scope()["verdict"], "full") + git(self.repo, "checkout", "-q", "--", "tsconfig.json") + self.write("package.json", '{"name": "x"}') + self.assertEqual(self.scope()["verdict"], "full") + + def test_nested_manifest_stays_impacted(self) -> None: + self.write("packages/web/package.json", '{"name": "web"}') + report = self.scope() + self.assertEqual(report["verdict"], "impacted") + self.assertEqual(report["packages"], ["packages/web"]) + + def test_base_head_scopes_to_uncommitted_fixes(self) -> None: + self.write("packages/api/src/index.ts", "export const a = 1") + git(self.repo, "commit", "-qam", "api") + self.write("packages/web/src/app.ts", "export const b = 2") + self.assertEqual(self.scope()["packages"], ["packages/api", "packages/web"]) + self.assertEqual(self.scope("--base", "HEAD")["packages"], ["packages/web"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/dev/references/ci-parity.md b/dev/references/ci-parity.md index 6a28a54..423f946 100644 --- a/dev/references/ci-parity.md +++ b/dev/references/ci-parity.md @@ -3,6 +3,8 @@ The local quality loop is not complete while a required pull-request check is known red. Spec validation and feature E2E prove the promised change; CI parity proves the repository's merge gate against the exact checkout being proposed. +Locally that proof covers what the change can reach; the full merge gate is the +pull request's own CI, which the run follows to green. ## Discover the merge gate @@ -19,10 +21,48 @@ Record each command and its outcome as it finishes, not after the whole set: `build` in the plan's `implementation-notes.md`, `ship` on the CI parity line of the `pr.md` its run keeps. +## Scope every local run to the impact + +A full gate re-proves every package the change never touched, and on a grown +repository it is most of a run's wall clock. CI runs it anyway, on every push. +So every local gate - a wave gate, a gauntlet re-run, the parity run below - +first asks what the change reaches: + +```bash +python3 {skill-root}/../../scripts/impact-scope.py [--base REF] +``` + +`--base` defaults to the merge base with the default branch; pass the last +commit (`--base HEAD`) to scope a run to uncommitted fixes alone. Loop against +its verdict rather than judging scope by feel: + +- `docs` - no test, build, or lint run is owed; record that and move on. +- `full` - a lockfile, root config, toolchain pin, or CI workflow changed, so + every package is reached: run each command in full; its reasons name why. +- `impacted` - run each command at the listed packages' scope, through the + repository's own affected-graph tooling when it has one (`nx affected`, + `turbo run --filter=...[{base}]`, bazel or pants target queries, + `go test` over the changed packages and `go list` dependents, + `cargo test -p`, `jest --findRelatedTests`, `vitest related`, pytest over the + package's test paths). Without one, add the workspace packages that declare a + listed package as a dependency. Lint and format run over the changed files; + typecheck and build over the impacted packages. + +A command with no scoped form runs in full when its inputs intersect the +impact (a screenshot suite when UI code changed, packaging when a shipped +package changed) and is otherwise recorded `deferred to CI: outside impact`. +A command the run would otherwise skip as slow is never deferred while its +inputs are impacted: impact, not cost, decides. + +Run the full set locally instead when no pull request will carry the change +(a run told to stay local, or a branch without a remote), or when the user asks +for full gates: with no CI to defer to, the local run is the merge gate. + ## Run before proposing a PR -Run every reproducible project-owned required-check command against the final -checkout, after feature E2E and after any hardening edits. +Run every reproducible project-owned required-check command at the scope +above against the final checkout, after feature E2E and after any hardening +edits. A command that already ran green on this exact tree - no file changed since - is not run a second time: record its result and where it came from. @@ -37,8 +77,11 @@ is not run a second time: record its result and where it came from. infrastructure is marked `remote-only`, with the reason. It is verified by PR follow-through rather than silently skipped. -Only offer or create the PR once all reproducible required checks are green and -all remote-only checks are identified. +Only offer or create the PR once every command run at the impacted scope is +green and every deferred and remote-only check is identified. A PR that carries +a deferred or remote-only check opens as a draft and is marked ready only when +its required checks are green, so no reviewer is asked to read a change the +merge gate has not yet passed. ## Follow the PR to green @@ -46,10 +89,10 @@ After the user authorizes a push/PR and the PR exists: 1. Watch required checks to a terminal state. 2. For each failure, fetch the failing job log, reproduce its project command - locally when possible, fix the root cause, run the narrow regression and the - full CI-parity command, commit, and push. + locally when possible, fix the root cause, run the narrow regression and + that failing command at the fix's impact, commit, and push. 3. Repeat until every required check is green or a genuine human call is - reached. + reached, then mark the PR ready when nothing else holds it as a draft. Do not stop at "CI restarted" when the user asked to finish or ship the change. Do not rerun a failed job unchanged unless the log proves an external service diff --git a/dev/scripts/impact-scope.py b/dev/scripts/impact-scope.py new file mode 100755 index 0000000..0508ceb --- /dev/null +++ b/dev/scripts/impact-scope.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Decide how much of a repository a local quality gate has to cover. + +Usage: + python3 impact-scope.py [--base REF] [--json] + +Lists the files changed since REF (default: the merge base with the default +branch; the working tree and untracked files included), the packages that own +them, and a verdict a gate loops against instead of judging scope by feel: + + docs every change is documentation; no test, build, or lint run is owed + impacted the change is bounded to the listed packages; run each gate command + at that scope and defer the full merge gate to the pull request's CI + full a change reaches every package (a lockfile, a root config, a CI + workflow, a shared toolchain pin); run the gate commands in full + +A package is the nearest directory holding a build manifest; "." is the root. +Exit status is 0 for every verdict and 2 only when git cannot answer. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import PurePosixPath + +MANIFESTS = ( + "package.json", "pyproject.toml", "setup.py", "setup.cfg", "go.mod", "Cargo.toml", + "pom.xml", "build.gradle", "build.gradle.kts", "Gemfile", "composer.json", "mix.exs", + "deno.json", "pubspec.yaml", "Package.swift", "CMakeLists.txt", "BUILD.bazel", +) +DOC_SUFFIXES = {".md", ".mdx", ".rst", ".txt", ".adoc"} +DOC_NAMES = {"LICENSE", "LICENCE", "NOTICE", "AUTHORS", "CODEOWNERS", "CHANGELOG"} +DOC_DIRS = {"docs", "doc", ".dev"} +# A change under one of these reaches every package, wherever it sits. +GLOBAL_DIRS = (".github/workflows/", ".github/actions/", ".circleci/", ".buildkite/", ".gitlab/") +GLOBAL_NAMES = { + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "pnpm-workspace.yaml", "bun.lockb", + "poetry.lock", "uv.lock", "Pipfile.lock", "go.work", "go.work.sum", "Cargo.lock", + "Gemfile.lock", "composer.lock", ".gitlab-ci.yml", ".tool-versions", ".nvmrc", + ".node-version", ".python-version", "rust-toolchain", "rust-toolchain.toml", + "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel", "nx.json", "turbo.json", "lerna.json", +} +CONFIG_NAMES = {"Makefile", "GNUmakefile", "justfile", "Justfile", "Dockerfile", "Taskfile.yml", "noxfile.py", "conftest.py"} +CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".lock", ".mk"} + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout + + +def default_base() -> str: + candidates = [] + try: + candidates.append(git("symbolic-ref", "--short", "refs/remotes/origin/HEAD").strip()) + except RuntimeError: + pass + candidates += ["origin/main", "origin/master", "main", "master"] + for ref in candidates: + try: + return git("merge-base", "HEAD", ref).strip() + except RuntimeError: + continue + raise RuntimeError("no default branch found; pass --base") + + +def changed_files(base: str) -> list[str]: + tracked = git("diff", "--name-only", base).splitlines() + untracked = git("ls-files", "--others", "--exclude-standard").splitlines() + return sorted({path for path in tracked + untracked if path}) + + +def is_doc(path: str) -> bool: + pure = PurePosixPath(path) + if pure.parts and pure.parts[0] in DOC_DIRS: + return True + return pure.suffix.lower() in DOC_SUFFIXES or pure.stem.upper() in DOC_NAMES + + +def global_reason(path: str) -> str | None: + pure = PurePosixPath(path) + if path.startswith(GLOBAL_DIRS): + return f"{path}: CI configuration" + if pure.name in GLOBAL_NAMES: + return f"{path}: lockfile or workspace/toolchain pin" + if len(pure.parts) == 1: + # A root manifest or root tool config feeds every package under it. + if pure.name in MANIFESTS: + return f"{path}: root manifest" + if is_config(pure): + return f"{path}: root-level configuration" + return None + + +def is_config(pure: PurePosixPath) -> bool: + name = pure.name + return ( + name.startswith(".") + or name in CONFIG_NAMES + or pure.suffix.lower() in CONFIG_SUFFIXES + or "config" in name.lower() + or name.endswith("rc") + ) + + +def owning_package(path: str, manifest_dirs: set[str]) -> str: + for parent in PurePosixPath(path).parents: + key = "." if str(parent) == "." else str(parent) + if key in manifest_dirs: + return key + return "." + + +def tracked_manifest_dirs() -> set[str]: + dirs = set() + for path in git("ls-files").splitlines(): + pure = PurePosixPath(path) + if pure.name in MANIFESTS: + dirs.add(str(pure.parent)) + return dirs or {"."} + + +def assess(files: list[str], manifest_dirs: set[str]) -> dict: + code = [path for path in files if not is_doc(path)] + reasons = [r for r in (global_reason(path) for path in code) if r] + if not files or not code: + verdict = "docs" + elif reasons: + verdict = "full" + else: + verdict = "impacted" + packages = sorted({owning_package(path, manifest_dirs) for path in code}) + return {"verdict": verdict, "reasons": reasons, "packages": packages, "files": files} + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", help="ref to diff against (default: merge base with the default branch)") + parser.add_argument("--json", action="store_true", help="print the result as JSON") + args = parser.parse_args(argv) + try: + base = args.base or default_base() + report = assess(changed_files(base), tracked_manifest_dirs()) + except RuntimeError as error: + print(f"impact-scope: {error}", file=sys.stderr) + return 2 + report["base"] = base + if args.json: + print(json.dumps(report, indent=2)) + return 0 + print(f"verdict: {report['verdict']}") + print(f"base: {base}") + for reason in report["reasons"]: + print(f"full because: {reason}") + print("packages: " + (", ".join(report["packages"]) or "none")) + print(f"files: {len(report['files'])}") + for path in report["files"]: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/dev/skills/build/SKILL.md b/dev/skills/build/SKILL.md index f168889..27b52c8 100644 --- a/dev/skills/build/SKILL.md +++ b/dev/skills/build/SKILL.md @@ -21,7 +21,7 @@ Read [references/layers.md](references/layers.md), [references/tests.md](referen 4. Move straight to the next wave. Never stop after one change set or wave to ask about review. 5. When every change set is committed, loop `python3 {build-skill-root}/scripts/check-tests.py .dev/{plan-name}` until it exits clean: it proves every specced scenario has a test that really exists, rather than one that was reported. 6. Then run the full e2e pass per "The e2e layer" below over the whole spec, and loop on failures until it is green. -7. Run the repository's required pull-request commands per [../../references/ci-parity.md](../../references/ci-parity.md), plus every Validation command the wave gates left for the end, starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. +7. Run the repository's required pull-request commands, plus every Validation command the wave gates left for the end, at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md) - what lies outside it is deferred to the PR's CI, which `ship` follows to green - starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. 8. After the e2e report and CI-parity gate, close per "Closing message". Build never pushes or opens a PR; that is `ship`'s phase 3. ## Jira sync @@ -88,6 +88,7 @@ It is the shared state across waves - parallel change-set agents can't see each ```markdown ## Build run {date} - Validation: {the commands this run gates on, and where they came from} +- Impact: {impact-scope.py verdict and packages for the branch} - Waves: [1 2] [3] [4 5] ## Change set {n}: {title} @@ -102,7 +103,7 @@ It is the shared state across waves - parallel change-set agents can't see each - Tests added: ... ## CI parity -- {command} -> {green | red: what failed | remote-only: why} # one line as each command finishes +- {command} -> {green at {scope} | red: what failed | remote-only: why | deferred to CI: outside impact} # one line as each command finishes ``` This file is a short running log, not a rendered report. diff --git a/dev/skills/build/references/parallel.md b/dev/skills/build/references/parallel.md index 3aac303..5e0f23d 100644 --- a/dev/skills/build/references/parallel.md +++ b/dev/skills/build/references/parallel.md @@ -69,12 +69,13 @@ If two change sets in a wave edited the same file anyway, reconcile it yourself ## The wave gate -The spec's Validation block is the gate between a wave and its commits, and the only full run a wave gets - a wave of one you implemented yourself included. +The spec's Validation block is the gate between a wave and its commits, and the only Validation run a wave gets - a wave of one you implemented yourself included. +Each command runs at the wave's impact, not over the whole repo: `python3 {build-skill-root}/../../scripts/impact-scope.py --base HEAD` scopes it to the wave's uncommitted changes, read per "Scope every local run to the impact" in [../../../references/ci-parity.md](../../../references/ci-parity.md). Each of its commands runs once per tree: a green result stands until a file changes, so nothing is re-run "to be sure" before committing, and after a fix the failed command runs first and the rest only once it is green. Start commands that share no state together (lint, typecheck, and the unit suite), and write the wave's notes entries while they run. A command the block marks `(end of build)` is left out of the wave gate, and so are three kinds even when a spec lists one unmarked, because each costs minutes and tells a wave nothing it needs to commit: the e2e suite (the e2e pass runs it), benchmarks, and any analysis that re-runs the suite to measure it - coverage, complexity, mutation. -Each of those runs once, on the final tree, with the CI-parity gate. +Each of those runs once, on the final tree, with the CI-parity gate and at its scope. ## When not to parallelize diff --git a/dev/skills/ship/SKILL.md b/dev/skills/ship/SKILL.md index f41e880..90ee2b1 100644 --- a/dev/skills/ship/SKILL.md +++ b/dev/skills/ship/SKILL.md @@ -47,7 +47,7 @@ Never weaken or skip a check because acquiring its tool is work. Run each check to completion per the loop in [references/gauntlet.md](references/gauntlet.md). A run that will reach phase 3 opens `.dev/{plan-name}/pr.md` before the first check and records each check's result in it as that check finishes, per the Body section of [references/pull-request.md](references/pull-request.md). When any check dispatched fixes, end the phase with the e2e refresh in the same reference: re-run the spec's `[e2e]` scenarios and overwrite the report, so phase 2 judges the post-fix code instead of stale evidence. -Before phase 2, always run the required pull-request commands per +Before phase 2, always run the required pull-request commands at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md), even when no fix agent fired. Feature E2E is not a substitute for a repository screenshot, packaging, or report job that CI requires. @@ -125,7 +125,7 @@ Open the PR automatically, with evidence a reviewer can see before reading the d 1. Commit what the gauntlet left uncommitted, branch off the default branch if still on it, and push. 2. Build the Evidence section from the e2e report with `python3 {ship-skill-root}/scripts/pr-evidence.py extract`, publishing frontend screenshots to the `pr-evidence` branch with its `publish` command; without an e2e report, capture the evidence now per the reference - screenshots for a UI, a labeled before/after pair otherwise, and a red-on-base, green-on-branch reproducing test for every bug fix. 3. Add that section to the run's `.dev/{plan-name}/pr.md`, in the reference's body shape, and loop `pr-evidence.py check` on it until it passes; the check, not your judgment, decides whether the proof is real enough. -4. Create the PR (draft when a real blocker survived remediation) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). +4. Create the PR (draft when a real blocker survived remediation or a required check was deferred to CI) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). ## Wrap up diff --git a/dev/skills/ship/references/gauntlet.md b/dev/skills/ship/references/gauntlet.md index 2e0980e..f93e149 100644 --- a/dev/skills/ship/references/gauntlet.md +++ b/dev/skills/ship/references/gauntlet.md @@ -12,7 +12,7 @@ For each tool (the batched five count as one), in order: 1. Run it; collect the violations. 2. Dispatch fixes: one fresh-context agent per independent area, launched in a single message, each given only the violation list for its area, the relevant file paths, and the fix vocabulary below. -3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. +3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) at the fixes' impact - `python3 {ship-skill-root}/../../scripts/impact-scope.py --base HEAD`, read per [../../../references/ci-parity.md](../../../references/ci-parity.md) - to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. 4. A violation that resists two fix rounds on the same root cause, or that the change seems to legitimately require, is a human call: stop and present it - it is either a real defect, a threshold worth changing, or a rule the spec should have amended. 5. When the run keeps a `pr.md` ([pull-request.md](pull-request.md)), write the tool's Quality row - found, fixed, surviving, one row per check for the batched five - and any human call under Open calls before starting the next tool; the wrap-up reads its tallies from there. @@ -26,15 +26,18 @@ Skip it when no tool dispatched fixes (the report is still fresh), when there is ## Exit: prove CI parity -After the e2e refresh decision, run the required pull-request commands using +After the e2e refresh decision, run the required pull-request commands at the +branch's impact using [../../../references/ci-parity.md](../../../references/ci-parity.md). This gate always runs in the default two-phase flow and gauntlet-only mode, even when no -fix agent changed code. +fix agent changed code; a run that opens no PR has no CI to defer to and runs +the full set. A red required check is a violation. "Pre-existing" requires merge-base proof and a human call; it is never a note that permits a PR-ready verdict. When a fix changes behavior or test orchestration, re-run the affected command and -continue until the full CI-parity set is green. +continue until every command the impact owes is green; the PR's CI +proves the rest. ## Fix vocabulary diff --git a/dev/skills/ship/references/pull-request.md b/dev/skills/ship/references/pull-request.md index c4023e7..c172ae1 100644 --- a/dev/skills/ship/references/pull-request.md +++ b/dev/skills/ship/references/pull-request.md @@ -5,7 +5,7 @@ A PR without evidence asks the reviewer to trust the description; this phase mak ## When it runs -Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review, a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the human call is made. +Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review - a draft until its required checks are green when a check was deferred to CI, per [../../../references/ci-parity.md](../../../references/ci-parity.md) - a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the human call is made. It is skipped in "gauntlet only" and "review only" runs and when the user says "no PR" or "local only"; a review-only run on someone else's PR never pushes anything. Never force-push, never rebase, and never touch a branch other than the work branch and the evidence branch. @@ -58,7 +58,7 @@ Plan `.dev/{plan-name}` | Review {N}: {verdict} | Jira {EPIC-KEY when enabled} {one row per gauntlet tool} Review panel: {lenses}; {blockers} blockers, {concerns} concerns ({review_N.md path}). -CI parity: {each reproducible required command and its result}; remote-only: {checks verified by the PR itself, or "none"}. +CI parity ({impact verdict and packages}): {each reproducible required command, the scope it ran at, and its result}; deferred to CI: {commands outside the impact, or "none"}; remote-only: {checks verified by the PR itself, or "none"}. ## Open calls @@ -69,8 +69,8 @@ Title: `{EPIC-KEY} ` prefix when Jira is enabled, then the change in imperative ## Create or update -- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK), base = the default branch. -- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` when a BLOCK verdict has cleared, never the reverse. +- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK or a deferred check), base = the default branch. +- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` once no BLOCK verdict stands and its required checks are green, never the reverse. - No `gh` and no equivalent host CLI: push, keep `pr.md`, print the compare URL, and say the PR must be opened by hand. Then follow the PR's required checks to a terminal state per [../../../references/ci-parity.md](../../../references/ci-parity.md). diff --git a/dev/skills/ship/references/remediation.md b/dev/skills/ship/references/remediation.md index d0a3639..42b79dd 100644 --- a/dev/skills/ship/references/remediation.md +++ b/dev/skills/ship/references/remediation.md @@ -17,10 +17,10 @@ CONCERNS and PASS skip straight to phase 3; concerns stay in the report and the Each agent gets only its blockers, the relevant file paths, the brief phase 2 built (spec excerpts, contracts, decision rationale), and the fix vocabulary below. Agents commit their fix on the work branch in `commit`'s message shape, naming the review index and finding in the Why. 2. **Re-harden.** Run phase 1's five read-only analyzers over the files the round touched, plus the flakiness check over the tests it added; dispatch fixes and loop per [gauntlet.md](gauntlet.md) as usual. - Then run the spec's Validation block (or the repo's test suite) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. + Then run the spec's Validation block (or the repo's test suite) at the round's impact per [../../../references/ci-parity.md](../../../references/ci-parity.md) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. 3. **Re-review.** Run phase 2 again as a re-review: `review_{N+1}.md`, same lenses, previous findings carried as verification items so the panel states whether each blocker is fixed or still open. The summary line names the round: `Remediation round {r} of 2`. -4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a ready PR. +4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a PR ready for review once its required checks are green. BLOCK after round 1 starts round 2. BLOCK after round 2 ends the loop: every surviving blocker is a real blocker. diff --git a/factory/.claude-plugin/plugin.json b/factory/.claude-plugin/plugin.json index 006abdb..800566f 100644 --- a/factory/.claude-plugin/plugin.json +++ b/factory/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "factory", - "version": "0.1.0", + "version": "0.2.0", "description": "An orchestrator skill that drives a pipeline of typed phases declared in .factory/config.yaml (the built-in scope, scope-review, build, and ship when there is none), running the interactive phases inline and the rest unattended, judging each phase's completion itself and repairing or relaunching it on failure, ending in a pull request or a report. Use on Claude Code or Codex when you want a request carried through to a shipped change without a human at each phase.", "author": { "name": "Tobrun" diff --git a/factory/evals/tests/test_protocol_and_copies.py b/factory/evals/tests/test_protocol_and_copies.py index 58c4218..62dadbe 100644 --- a/factory/evals/tests/test_protocol_and_copies.py +++ b/factory/evals/tests/test_protocol_and_copies.py @@ -44,7 +44,7 @@ def test_copied_tree_holds_dev_references_and_factory_run(self) -> None: factory_script_names = {p.name for p in (FACTORY / "scripts").glob("*.py")} self.assertEqual( - factory_script_names, {"skill-metrics.py", "architecture-check.py", "factory-config.py"} + factory_script_names, {"skill-metrics.py", "architecture-check.py", "factory-config.py", "impact-scope.py"} ) def test_backticked_reference_and_script_paths_resolve_under_factory(self) -> None: diff --git a/factory/phases/build/SKILL.md b/factory/phases/build/SKILL.md index 61724ee..0d89af5 100644 --- a/factory/phases/build/SKILL.md +++ b/factory/phases/build/SKILL.md @@ -21,7 +21,7 @@ Read [references/layers.md](references/layers.md), [references/tests.md](referen 4. Move straight to the next wave. Never stop after one change set or wave to ask about review. 5. When every change set is committed, loop `python3 {build-skill-root}/scripts/check-tests.py .dev/{plan-name}` until it exits clean: it proves every specced scenario has a test that really exists, rather than one that was reported. 6. Then run the full e2e pass per "The e2e layer" below over the whole spec, and loop on failures until it is green. -7. Run the repository's required pull-request commands per [../../references/ci-parity.md](../../references/ci-parity.md), plus every Validation command the wave gates left for the end, starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. +7. Run the repository's required pull-request commands, plus every Validation command the wave gates left for the end, at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md) - what lies outside it is deferred to the PR's CI, which `ship` follows to green - starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. 8. After the e2e report and CI-parity gate, close per "Closing message". Build never pushes or opens a PR; that is `ship`'s phase 3. ## Jira sync diff --git a/factory/phases/build/references/parallel.md b/factory/phases/build/references/parallel.md index 3aac303..5e0f23d 100644 --- a/factory/phases/build/references/parallel.md +++ b/factory/phases/build/references/parallel.md @@ -69,12 +69,13 @@ If two change sets in a wave edited the same file anyway, reconcile it yourself ## The wave gate -The spec's Validation block is the gate between a wave and its commits, and the only full run a wave gets - a wave of one you implemented yourself included. +The spec's Validation block is the gate between a wave and its commits, and the only Validation run a wave gets - a wave of one you implemented yourself included. +Each command runs at the wave's impact, not over the whole repo: `python3 {build-skill-root}/../../scripts/impact-scope.py --base HEAD` scopes it to the wave's uncommitted changes, read per "Scope every local run to the impact" in [../../../references/ci-parity.md](../../../references/ci-parity.md). Each of its commands runs once per tree: a green result stands until a file changes, so nothing is re-run "to be sure" before committing, and after a fix the failed command runs first and the rest only once it is green. Start commands that share no state together (lint, typecheck, and the unit suite), and write the wave's notes entries while they run. A command the block marks `(end of build)` is left out of the wave gate, and so are three kinds even when a spec lists one unmarked, because each costs minutes and tells a wave nothing it needs to commit: the e2e suite (the e2e pass runs it), benchmarks, and any analysis that re-runs the suite to measure it - coverage, complexity, mutation. -Each of those runs once, on the final tree, with the CI-parity gate. +Each of those runs once, on the final tree, with the CI-parity gate and at its scope. ## When not to parallelize diff --git a/factory/phases/ship/SKILL.md b/factory/phases/ship/SKILL.md index b0f8a41..8d58b07 100644 --- a/factory/phases/ship/SKILL.md +++ b/factory/phases/ship/SKILL.md @@ -46,7 +46,7 @@ Never weaken or skip a check because acquiring its tool is work. Run each check to completion per the loop in [references/gauntlet.md](references/gauntlet.md). When any check dispatched fixes, end the phase with the e2e refresh in the same reference: re-run the spec's `[e2e]` scenarios and overwrite the report, so phase 2 judges the post-fix code instead of stale evidence. -Before phase 2, always run the required pull-request commands per +Before phase 2, always run the required pull-request commands at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md), even when no fix agent fired. Feature E2E is not a substitute for a repository screenshot, packaging, or report job that CI requires. @@ -123,7 +123,7 @@ Open the PR automatically, with evidence a reviewer can see before reading the d 1. Commit what the gauntlet left uncommitted, branch off the default branch if still on it, and push. 2. Build the Evidence section from the e2e report with `python3 {ship-skill-root}/scripts/pr-evidence.py extract`, publishing frontend screenshots to the `pr-evidence` branch with its `publish` command; without an e2e report, capture the evidence now per the reference - screenshots for a UI, a labeled before/after pair otherwise, and a red-on-base, green-on-branch reproducing test for every bug fix. 3. Write `.dev/{plan-name}/pr.md` in the reference's body shape and loop `pr-evidence.py check` on it until it passes; the check, not your judgment, decides whether the proof is real enough. -4. Create the PR (draft when a real blocker survived remediation) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). +4. Create the PR (draft when a real blocker survived remediation or a required check was deferred to CI) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). ## Wrap up diff --git a/factory/phases/ship/references/gauntlet.md b/factory/phases/ship/references/gauntlet.md index 990f5d8..a9fe1cc 100644 --- a/factory/phases/ship/references/gauntlet.md +++ b/factory/phases/ship/references/gauntlet.md @@ -12,7 +12,7 @@ For each tool (the batched five count as one), in order: 1. Run it; collect the violations. 2. Dispatch fixes: one fresh-context agent per independent area, launched in a single message, each given only the violation list for its area, the relevant file paths, and the fix vocabulary below. -3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. +3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) at the fixes' impact - `python3 {ship-skill-root}/../../scripts/impact-scope.py --base HEAD`, read per [../../../references/ci-parity.md](../../../references/ci-parity.md) - to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. 4. A violation that resists two fix rounds on the same root cause, or that the change seems to legitimately require, is recorded as a decision and left in place: it is either a real defect, a threshold worth changing, or a rule the spec should have amended, and the choice made is noted in `auto_decided` with the reason. ## Exit: refresh the e2e evidence @@ -25,15 +25,18 @@ Skip it when no tool dispatched fixes (the report is still fresh), when there is ## Exit: prove CI parity -After the e2e refresh decision, run the required pull-request commands using +After the e2e refresh decision, run the required pull-request commands at the +branch's impact using [../../../references/ci-parity.md](../../../references/ci-parity.md). This gate always runs in the default two-phase flow and gauntlet-only mode, even when no -fix agent changed code. +fix agent changed code; a run that opens no PR has no CI to defer to and runs +the full set. A red required check is a violation. "Pre-existing" requires merge-base proof, recorded as a decision with that proof; it is never a note that permits a PR-ready verdict. When a fix changes behavior or test orchestration, re-run -the affected command and continue until the full CI-parity set is green. +the affected command and continue until every command the impact owes is +green; the PR's CI proves the rest. ## Fix vocabulary diff --git a/factory/phases/ship/references/pull-request.md b/factory/phases/ship/references/pull-request.md index e317f98..a65127b 100644 --- a/factory/phases/ship/references/pull-request.md +++ b/factory/phases/ship/references/pull-request.md @@ -5,7 +5,7 @@ A PR without evidence asks the reviewer to trust the description; this phase mak ## When it runs -Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review, a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the recorded decision stands. +Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review - a draft until its required checks are green when a check was deferred to CI, per [../../../references/ci-parity.md](../../../references/ci-parity.md) - a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the recorded decision stands. It is skipped in "gauntlet only" and "review only" runs and when the launch prompt says "no PR" or "local only"; a review-only run on someone else's PR never pushes anything. Never force-push, never rebase, and never touch a branch other than the work branch and the evidence branch. @@ -56,7 +56,7 @@ Plan `.dev/{plan-name}` | Review {N}: {verdict} | Jira {EPIC-KEY when enabled} {one row per gauntlet tool} Review panel: {lenses}; {blockers} blockers, {concerns} concerns ({review_N.md path}). -CI parity: {each reproducible required command and its result}; remote-only: {checks verified by the PR itself, or "none"}. +CI parity ({impact verdict and packages}): {each reproducible required command, the scope it ran at, and its result}; deferred to CI: {commands outside the impact, or "none"}; remote-only: {checks verified by the PR itself, or "none"}. ## Open calls @@ -67,8 +67,8 @@ Title: `{EPIC-KEY} ` prefix when Jira is enabled, then the change in imperative ## Create or update -- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK), base = the default branch. -- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` when a BLOCK verdict has cleared, never the reverse. +- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK or a deferred check), base = the default branch. +- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` once no BLOCK verdict stands and its required checks are green, never the reverse. - No `gh` and no equivalent host CLI: push, keep `pr.md`, print the compare URL, and say the PR must be opened by hand. Then follow the PR's required checks to a terminal state per [../../../references/ci-parity.md](../../../references/ci-parity.md). diff --git a/factory/phases/ship/references/remediation.md b/factory/phases/ship/references/remediation.md index 36f8c47..94a49e7 100644 --- a/factory/phases/ship/references/remediation.md +++ b/factory/phases/ship/references/remediation.md @@ -17,10 +17,10 @@ CONCERNS and PASS skip straight to phase 3; concerns stay in the report and the Each agent gets only its blockers, the relevant file paths, the brief phase 2 built (spec excerpts, contracts, decision rationale), and the fix vocabulary below. Agents commit their fix on the work branch in `commit`'s message shape, naming the review index and finding in the Why. 2. **Re-harden.** Run phase 1's five read-only analyzers over the files the round touched, plus the flakiness check over the tests it added; dispatch fixes and loop per [gauntlet.md](gauntlet.md) as usual. - Then run the spec's Validation block (or the repo's test suite) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. + Then run the spec's Validation block (or the repo's test suite) at the round's impact per [../../../references/ci-parity.md](../../../references/ci-parity.md) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. 3. **Re-review.** Run phase 2 again as a re-review: `review_{N+1}.md`, same lenses, previous findings carried as verification items so the panel states whether each blocker is fixed or still open. The summary line names the round: `Remediation round {r} of 2`. -4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a ready PR. +4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a PR ready for review once its required checks are green. BLOCK after round 1 starts round 2. BLOCK after round 2 ends the loop: every surviving blocker is a real blocker. diff --git a/factory/references/ci-parity.md b/factory/references/ci-parity.md index 92157c9..adf9ff9 100644 --- a/factory/references/ci-parity.md +++ b/factory/references/ci-parity.md @@ -3,6 +3,8 @@ The local quality loop is not complete while a required pull-request check is known red. Spec validation and feature E2E prove the promised change; CI parity proves the repository's merge gate against the exact checkout being proposed. +Locally that proof covers what the change can reach; the full merge gate is the +pull request's own CI, which the run follows to green. ## Discover the merge gate @@ -17,10 +19,48 @@ repository-provided aggregate target when it covers the same jobs. Record the commands and outcomes in the plan's `implementation-notes.md`. +## Scope every local run to the impact + +A full gate re-proves every package the change never touched, and on a grown +repository it is most of a run's wall clock. CI runs it anyway, on every push. +So every local gate - a wave gate, a gauntlet re-run, the parity run below - +first asks what the change reaches: + +```bash +python3 {skill-root}/../../scripts/impact-scope.py [--base REF] +``` + +`--base` defaults to the merge base with the default branch; pass the last +commit (`--base HEAD`) to scope a run to uncommitted fixes alone. Loop against +its verdict rather than judging scope by feel: + +- `docs` - no test, build, or lint run is owed; record that and move on. +- `full` - a lockfile, root config, toolchain pin, or CI workflow changed, so + every package is reached: run each command in full; its reasons name why. +- `impacted` - run each command at the listed packages' scope, through the + repository's own affected-graph tooling when it has one (`nx affected`, + `turbo run --filter=...[{base}]`, bazel or pants target queries, + `go test` over the changed packages and `go list` dependents, + `cargo test -p`, `jest --findRelatedTests`, `vitest related`, pytest over the + package's test paths). Without one, add the workspace packages that declare a + listed package as a dependency. Lint and format run over the changed files; + typecheck and build over the impacted packages. + +A command with no scoped form runs in full when its inputs intersect the +impact (a screenshot suite when UI code changed, packaging when a shipped +package changed) and is otherwise recorded `deferred to CI: outside impact`. +A command the run would otherwise skip as slow is never deferred while its +inputs are impacted: impact, not cost, decides. + +Run the full set locally instead when no pull request will carry the change +(a run told to stay local, or a branch without a remote), or when the launch +prompt names full gates: with no CI to defer to, the local run is the merge gate. + ## Run before proposing a PR -Run every reproducible project-owned required-check command against the final -checkout, after feature E2E and after any hardening edits. +Run every reproducible project-owned required-check command at the scope +above against the final checkout, after feature E2E and after any hardening +edits. A command that already ran green on this exact tree - no file changed since - is not run a second time: record its result and where it came from. @@ -35,8 +75,11 @@ is not run a second time: record its result and where it came from. infrastructure is marked `remote-only`, with the reason. It is verified by PR follow-through rather than silently skipped. -Only offer or create the PR once all reproducible required checks are green and -all remote-only checks are identified. +Only offer or create the PR once every command run at the impacted scope is +green and every deferred and remote-only check is identified. A PR that carries +a deferred or remote-only check opens as a draft and is marked ready only when +its required checks are green, so no reviewer is asked to read a change the +merge gate has not yet passed. ## Follow the PR to green @@ -44,10 +87,10 @@ After the user authorizes a push/PR and the PR exists: 1. Watch required checks to a terminal state. 2. For each failure, fetch the failing job log, reproduce its project command - locally when possible, fix the root cause, run the narrow regression and the - full CI-parity command, commit, and push. + locally when possible, fix the root cause, run the narrow regression and + that failing command at the fix's impact, commit, and push. 3. Repeat until every required check is green or a genuine human call is - reached. + reached, then mark the PR ready when nothing else holds it as a draft. Do not stop at "CI restarted" when the user asked to finish or ship the change. Do not rerun a failed job unchanged unless the log proves an external service diff --git a/factory/scripts/impact-scope.py b/factory/scripts/impact-scope.py new file mode 100755 index 0000000..0508ceb --- /dev/null +++ b/factory/scripts/impact-scope.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Decide how much of a repository a local quality gate has to cover. + +Usage: + python3 impact-scope.py [--base REF] [--json] + +Lists the files changed since REF (default: the merge base with the default +branch; the working tree and untracked files included), the packages that own +them, and a verdict a gate loops against instead of judging scope by feel: + + docs every change is documentation; no test, build, or lint run is owed + impacted the change is bounded to the listed packages; run each gate command + at that scope and defer the full merge gate to the pull request's CI + full a change reaches every package (a lockfile, a root config, a CI + workflow, a shared toolchain pin); run the gate commands in full + +A package is the nearest directory holding a build manifest; "." is the root. +Exit status is 0 for every verdict and 2 only when git cannot answer. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import PurePosixPath + +MANIFESTS = ( + "package.json", "pyproject.toml", "setup.py", "setup.cfg", "go.mod", "Cargo.toml", + "pom.xml", "build.gradle", "build.gradle.kts", "Gemfile", "composer.json", "mix.exs", + "deno.json", "pubspec.yaml", "Package.swift", "CMakeLists.txt", "BUILD.bazel", +) +DOC_SUFFIXES = {".md", ".mdx", ".rst", ".txt", ".adoc"} +DOC_NAMES = {"LICENSE", "LICENCE", "NOTICE", "AUTHORS", "CODEOWNERS", "CHANGELOG"} +DOC_DIRS = {"docs", "doc", ".dev"} +# A change under one of these reaches every package, wherever it sits. +GLOBAL_DIRS = (".github/workflows/", ".github/actions/", ".circleci/", ".buildkite/", ".gitlab/") +GLOBAL_NAMES = { + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "pnpm-workspace.yaml", "bun.lockb", + "poetry.lock", "uv.lock", "Pipfile.lock", "go.work", "go.work.sum", "Cargo.lock", + "Gemfile.lock", "composer.lock", ".gitlab-ci.yml", ".tool-versions", ".nvmrc", + ".node-version", ".python-version", "rust-toolchain", "rust-toolchain.toml", + "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel", "nx.json", "turbo.json", "lerna.json", +} +CONFIG_NAMES = {"Makefile", "GNUmakefile", "justfile", "Justfile", "Dockerfile", "Taskfile.yml", "noxfile.py", "conftest.py"} +CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".lock", ".mk"} + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout + + +def default_base() -> str: + candidates = [] + try: + candidates.append(git("symbolic-ref", "--short", "refs/remotes/origin/HEAD").strip()) + except RuntimeError: + pass + candidates += ["origin/main", "origin/master", "main", "master"] + for ref in candidates: + try: + return git("merge-base", "HEAD", ref).strip() + except RuntimeError: + continue + raise RuntimeError("no default branch found; pass --base") + + +def changed_files(base: str) -> list[str]: + tracked = git("diff", "--name-only", base).splitlines() + untracked = git("ls-files", "--others", "--exclude-standard").splitlines() + return sorted({path for path in tracked + untracked if path}) + + +def is_doc(path: str) -> bool: + pure = PurePosixPath(path) + if pure.parts and pure.parts[0] in DOC_DIRS: + return True + return pure.suffix.lower() in DOC_SUFFIXES or pure.stem.upper() in DOC_NAMES + + +def global_reason(path: str) -> str | None: + pure = PurePosixPath(path) + if path.startswith(GLOBAL_DIRS): + return f"{path}: CI configuration" + if pure.name in GLOBAL_NAMES: + return f"{path}: lockfile or workspace/toolchain pin" + if len(pure.parts) == 1: + # A root manifest or root tool config feeds every package under it. + if pure.name in MANIFESTS: + return f"{path}: root manifest" + if is_config(pure): + return f"{path}: root-level configuration" + return None + + +def is_config(pure: PurePosixPath) -> bool: + name = pure.name + return ( + name.startswith(".") + or name in CONFIG_NAMES + or pure.suffix.lower() in CONFIG_SUFFIXES + or "config" in name.lower() + or name.endswith("rc") + ) + + +def owning_package(path: str, manifest_dirs: set[str]) -> str: + for parent in PurePosixPath(path).parents: + key = "." if str(parent) == "." else str(parent) + if key in manifest_dirs: + return key + return "." + + +def tracked_manifest_dirs() -> set[str]: + dirs = set() + for path in git("ls-files").splitlines(): + pure = PurePosixPath(path) + if pure.name in MANIFESTS: + dirs.add(str(pure.parent)) + return dirs or {"."} + + +def assess(files: list[str], manifest_dirs: set[str]) -> dict: + code = [path for path in files if not is_doc(path)] + reasons = [r for r in (global_reason(path) for path in code) if r] + if not files or not code: + verdict = "docs" + elif reasons: + verdict = "full" + else: + verdict = "impacted" + packages = sorted({owning_package(path, manifest_dirs) for path in code}) + return {"verdict": verdict, "reasons": reasons, "packages": packages, "files": files} + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", help="ref to diff against (default: merge base with the default branch)") + parser.add_argument("--json", action="store_true", help="print the result as JSON") + args = parser.parse_args(argv) + try: + base = args.base or default_base() + report = assess(changed_files(base), tracked_manifest_dirs()) + except RuntimeError as error: + print(f"impact-scope: {error}", file=sys.stderr) + return 2 + report["base"] = base + if args.json: + print(json.dumps(report, indent=2)) + return 0 + print(f"verdict: {report['verdict']}") + print(f"base: {base}") + for reason in report["reasons"]: + print(f"full because: {reason}") + print("packages: " + (", ".join(report["packages"]) or "none")) + print(f"files: {len(report['files'])}") + for path in report["files"]: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/package.json b/package.json index 633e697..34fdfb0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tobrun/dev-workflow", - "version": "3.5.0", + "version": "3.6.0", "description": "Test-focused development workflow skills for Claude Code, Codex, and Pi.", "keywords": [ "pi-package", diff --git a/plugins/dev/.codex-plugin/plugin.json b/plugins/dev/.codex-plugin/plugin.json index bda2dd5..9b80da9 100644 --- a/plugins/dev/.codex-plugin/plugin.json +++ b/plugins/dev/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "dev", - "version": "3.5.0", + "version": "3.6.0", "description": "Development workflow skills: scope changes with argued decisions, build across unit/integration/e2e with every scenario proven by tests, ship with a deterministic quality gauntlet and an adversarially verified review, create structured commits that feed a decision ledger, and render pitches or comprehension quizzes.", "author": { "name": "Tobrun" diff --git a/plugins/dev/references/ci-parity.md b/plugins/dev/references/ci-parity.md index 6a28a54..423f946 100644 --- a/plugins/dev/references/ci-parity.md +++ b/plugins/dev/references/ci-parity.md @@ -3,6 +3,8 @@ The local quality loop is not complete while a required pull-request check is known red. Spec validation and feature E2E prove the promised change; CI parity proves the repository's merge gate against the exact checkout being proposed. +Locally that proof covers what the change can reach; the full merge gate is the +pull request's own CI, which the run follows to green. ## Discover the merge gate @@ -19,10 +21,48 @@ Record each command and its outcome as it finishes, not after the whole set: `build` in the plan's `implementation-notes.md`, `ship` on the CI parity line of the `pr.md` its run keeps. +## Scope every local run to the impact + +A full gate re-proves every package the change never touched, and on a grown +repository it is most of a run's wall clock. CI runs it anyway, on every push. +So every local gate - a wave gate, a gauntlet re-run, the parity run below - +first asks what the change reaches: + +```bash +python3 {skill-root}/../../scripts/impact-scope.py [--base REF] +``` + +`--base` defaults to the merge base with the default branch; pass the last +commit (`--base HEAD`) to scope a run to uncommitted fixes alone. Loop against +its verdict rather than judging scope by feel: + +- `docs` - no test, build, or lint run is owed; record that and move on. +- `full` - a lockfile, root config, toolchain pin, or CI workflow changed, so + every package is reached: run each command in full; its reasons name why. +- `impacted` - run each command at the listed packages' scope, through the + repository's own affected-graph tooling when it has one (`nx affected`, + `turbo run --filter=...[{base}]`, bazel or pants target queries, + `go test` over the changed packages and `go list` dependents, + `cargo test -p`, `jest --findRelatedTests`, `vitest related`, pytest over the + package's test paths). Without one, add the workspace packages that declare a + listed package as a dependency. Lint and format run over the changed files; + typecheck and build over the impacted packages. + +A command with no scoped form runs in full when its inputs intersect the +impact (a screenshot suite when UI code changed, packaging when a shipped +package changed) and is otherwise recorded `deferred to CI: outside impact`. +A command the run would otherwise skip as slow is never deferred while its +inputs are impacted: impact, not cost, decides. + +Run the full set locally instead when no pull request will carry the change +(a run told to stay local, or a branch without a remote), or when the user asks +for full gates: with no CI to defer to, the local run is the merge gate. + ## Run before proposing a PR -Run every reproducible project-owned required-check command against the final -checkout, after feature E2E and after any hardening edits. +Run every reproducible project-owned required-check command at the scope +above against the final checkout, after feature E2E and after any hardening +edits. A command that already ran green on this exact tree - no file changed since - is not run a second time: record its result and where it came from. @@ -37,8 +77,11 @@ is not run a second time: record its result and where it came from. infrastructure is marked `remote-only`, with the reason. It is verified by PR follow-through rather than silently skipped. -Only offer or create the PR once all reproducible required checks are green and -all remote-only checks are identified. +Only offer or create the PR once every command run at the impacted scope is +green and every deferred and remote-only check is identified. A PR that carries +a deferred or remote-only check opens as a draft and is marked ready only when +its required checks are green, so no reviewer is asked to read a change the +merge gate has not yet passed. ## Follow the PR to green @@ -46,10 +89,10 @@ After the user authorizes a push/PR and the PR exists: 1. Watch required checks to a terminal state. 2. For each failure, fetch the failing job log, reproduce its project command - locally when possible, fix the root cause, run the narrow regression and the - full CI-parity command, commit, and push. + locally when possible, fix the root cause, run the narrow regression and + that failing command at the fix's impact, commit, and push. 3. Repeat until every required check is green or a genuine human call is - reached. + reached, then mark the PR ready when nothing else holds it as a draft. Do not stop at "CI restarted" when the user asked to finish or ship the change. Do not rerun a failed job unchanged unless the log proves an external service diff --git a/plugins/dev/scripts/impact-scope.py b/plugins/dev/scripts/impact-scope.py new file mode 100755 index 0000000..0508ceb --- /dev/null +++ b/plugins/dev/scripts/impact-scope.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Decide how much of a repository a local quality gate has to cover. + +Usage: + python3 impact-scope.py [--base REF] [--json] + +Lists the files changed since REF (default: the merge base with the default +branch; the working tree and untracked files included), the packages that own +them, and a verdict a gate loops against instead of judging scope by feel: + + docs every change is documentation; no test, build, or lint run is owed + impacted the change is bounded to the listed packages; run each gate command + at that scope and defer the full merge gate to the pull request's CI + full a change reaches every package (a lockfile, a root config, a CI + workflow, a shared toolchain pin); run the gate commands in full + +A package is the nearest directory holding a build manifest; "." is the root. +Exit status is 0 for every verdict and 2 only when git cannot answer. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import PurePosixPath + +MANIFESTS = ( + "package.json", "pyproject.toml", "setup.py", "setup.cfg", "go.mod", "Cargo.toml", + "pom.xml", "build.gradle", "build.gradle.kts", "Gemfile", "composer.json", "mix.exs", + "deno.json", "pubspec.yaml", "Package.swift", "CMakeLists.txt", "BUILD.bazel", +) +DOC_SUFFIXES = {".md", ".mdx", ".rst", ".txt", ".adoc"} +DOC_NAMES = {"LICENSE", "LICENCE", "NOTICE", "AUTHORS", "CODEOWNERS", "CHANGELOG"} +DOC_DIRS = {"docs", "doc", ".dev"} +# A change under one of these reaches every package, wherever it sits. +GLOBAL_DIRS = (".github/workflows/", ".github/actions/", ".circleci/", ".buildkite/", ".gitlab/") +GLOBAL_NAMES = { + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "pnpm-workspace.yaml", "bun.lockb", + "poetry.lock", "uv.lock", "Pipfile.lock", "go.work", "go.work.sum", "Cargo.lock", + "Gemfile.lock", "composer.lock", ".gitlab-ci.yml", ".tool-versions", ".nvmrc", + ".node-version", ".python-version", "rust-toolchain", "rust-toolchain.toml", + "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel", "nx.json", "turbo.json", "lerna.json", +} +CONFIG_NAMES = {"Makefile", "GNUmakefile", "justfile", "Justfile", "Dockerfile", "Taskfile.yml", "noxfile.py", "conftest.py"} +CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".lock", ".mk"} + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout + + +def default_base() -> str: + candidates = [] + try: + candidates.append(git("symbolic-ref", "--short", "refs/remotes/origin/HEAD").strip()) + except RuntimeError: + pass + candidates += ["origin/main", "origin/master", "main", "master"] + for ref in candidates: + try: + return git("merge-base", "HEAD", ref).strip() + except RuntimeError: + continue + raise RuntimeError("no default branch found; pass --base") + + +def changed_files(base: str) -> list[str]: + tracked = git("diff", "--name-only", base).splitlines() + untracked = git("ls-files", "--others", "--exclude-standard").splitlines() + return sorted({path for path in tracked + untracked if path}) + + +def is_doc(path: str) -> bool: + pure = PurePosixPath(path) + if pure.parts and pure.parts[0] in DOC_DIRS: + return True + return pure.suffix.lower() in DOC_SUFFIXES or pure.stem.upper() in DOC_NAMES + + +def global_reason(path: str) -> str | None: + pure = PurePosixPath(path) + if path.startswith(GLOBAL_DIRS): + return f"{path}: CI configuration" + if pure.name in GLOBAL_NAMES: + return f"{path}: lockfile or workspace/toolchain pin" + if len(pure.parts) == 1: + # A root manifest or root tool config feeds every package under it. + if pure.name in MANIFESTS: + return f"{path}: root manifest" + if is_config(pure): + return f"{path}: root-level configuration" + return None + + +def is_config(pure: PurePosixPath) -> bool: + name = pure.name + return ( + name.startswith(".") + or name in CONFIG_NAMES + or pure.suffix.lower() in CONFIG_SUFFIXES + or "config" in name.lower() + or name.endswith("rc") + ) + + +def owning_package(path: str, manifest_dirs: set[str]) -> str: + for parent in PurePosixPath(path).parents: + key = "." if str(parent) == "." else str(parent) + if key in manifest_dirs: + return key + return "." + + +def tracked_manifest_dirs() -> set[str]: + dirs = set() + for path in git("ls-files").splitlines(): + pure = PurePosixPath(path) + if pure.name in MANIFESTS: + dirs.add(str(pure.parent)) + return dirs or {"."} + + +def assess(files: list[str], manifest_dirs: set[str]) -> dict: + code = [path for path in files if not is_doc(path)] + reasons = [r for r in (global_reason(path) for path in code) if r] + if not files or not code: + verdict = "docs" + elif reasons: + verdict = "full" + else: + verdict = "impacted" + packages = sorted({owning_package(path, manifest_dirs) for path in code}) + return {"verdict": verdict, "reasons": reasons, "packages": packages, "files": files} + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", help="ref to diff against (default: merge base with the default branch)") + parser.add_argument("--json", action="store_true", help="print the result as JSON") + args = parser.parse_args(argv) + try: + base = args.base or default_base() + report = assess(changed_files(base), tracked_manifest_dirs()) + except RuntimeError as error: + print(f"impact-scope: {error}", file=sys.stderr) + return 2 + report["base"] = base + if args.json: + print(json.dumps(report, indent=2)) + return 0 + print(f"verdict: {report['verdict']}") + print(f"base: {base}") + for reason in report["reasons"]: + print(f"full because: {reason}") + print("packages: " + (", ".join(report["packages"]) or "none")) + print(f"files: {len(report['files'])}") + for path in report["files"]: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/plugins/dev/skills/build/SKILL.md b/plugins/dev/skills/build/SKILL.md index 9224d76..b324527 100644 --- a/plugins/dev/skills/build/SKILL.md +++ b/plugins/dev/skills/build/SKILL.md @@ -20,7 +20,7 @@ Read [references/layers.md](references/layers.md), [references/tests.md](referen 4. Move straight to the next wave. Never stop after one change set or wave to ask about review. 5. When every change set is committed, loop `python3 {build-skill-root}/scripts/check-tests.py .dev/{plan-name}` until it exits clean: it proves every specced scenario has a test that really exists, rather than one that was reported. 6. Then run the full e2e pass per "The e2e layer" below over the whole spec, and loop on failures until it is green. -7. Run the repository's required pull-request commands per [../../references/ci-parity.md](../../references/ci-parity.md), plus every Validation command the wave gates left for the end, starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. +7. Run the repository's required pull-request commands, plus every Validation command the wave gates left for the end, at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md) - what lies outside it is deferred to the PR's CI, which `ship` follows to green - starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. 8. After the e2e report and CI-parity gate, close per "Closing message". Build never pushes or opens a PR; that is `ship`'s phase 3. ## Jira sync @@ -87,6 +87,7 @@ It is the shared state across waves - parallel change-set agents can't see each ```markdown ## Build run {date} - Validation: {the commands this run gates on, and where they came from} +- Impact: {impact-scope.py verdict and packages for the branch} - Waves: [1 2] [3] [4 5] ## Change set {n}: {title} @@ -101,7 +102,7 @@ It is the shared state across waves - parallel change-set agents can't see each - Tests added: ... ## CI parity -- {command} -> {green | red: what failed | remote-only: why} # one line as each command finishes +- {command} -> {green at {scope} | red: what failed | remote-only: why | deferred to CI: outside impact} # one line as each command finishes ``` This file is a short running log, not a rendered report. diff --git a/plugins/dev/skills/build/references/parallel.md b/plugins/dev/skills/build/references/parallel.md index 3aac303..5e0f23d 100644 --- a/plugins/dev/skills/build/references/parallel.md +++ b/plugins/dev/skills/build/references/parallel.md @@ -69,12 +69,13 @@ If two change sets in a wave edited the same file anyway, reconcile it yourself ## The wave gate -The spec's Validation block is the gate between a wave and its commits, and the only full run a wave gets - a wave of one you implemented yourself included. +The spec's Validation block is the gate between a wave and its commits, and the only Validation run a wave gets - a wave of one you implemented yourself included. +Each command runs at the wave's impact, not over the whole repo: `python3 {build-skill-root}/../../scripts/impact-scope.py --base HEAD` scopes it to the wave's uncommitted changes, read per "Scope every local run to the impact" in [../../../references/ci-parity.md](../../../references/ci-parity.md). Each of its commands runs once per tree: a green result stands until a file changes, so nothing is re-run "to be sure" before committing, and after a fix the failed command runs first and the rest only once it is green. Start commands that share no state together (lint, typecheck, and the unit suite), and write the wave's notes entries while they run. A command the block marks `(end of build)` is left out of the wave gate, and so are three kinds even when a spec lists one unmarked, because each costs minutes and tells a wave nothing it needs to commit: the e2e suite (the e2e pass runs it), benchmarks, and any analysis that re-runs the suite to measure it - coverage, complexity, mutation. -Each of those runs once, on the final tree, with the CI-parity gate. +Each of those runs once, on the final tree, with the CI-parity gate and at its scope. ## When not to parallelize diff --git a/plugins/dev/skills/ship/SKILL.md b/plugins/dev/skills/ship/SKILL.md index 61c690d..df82cc2 100644 --- a/plugins/dev/skills/ship/SKILL.md +++ b/plugins/dev/skills/ship/SKILL.md @@ -46,7 +46,7 @@ Never weaken or skip a check because acquiring its tool is work. Run each check to completion per the loop in [references/gauntlet.md](references/gauntlet.md). A run that will reach phase 3 opens `.dev/{plan-name}/pr.md` before the first check and records each check's result in it as that check finishes, per the Body section of [references/pull-request.md](references/pull-request.md). When any check dispatched fixes, end the phase with the e2e refresh in the same reference: re-run the spec's `[e2e]` scenarios and overwrite the report, so phase 2 judges the post-fix code instead of stale evidence. -Before phase 2, always run the required pull-request commands per +Before phase 2, always run the required pull-request commands at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md), even when no fix agent fired. Feature E2E is not a substitute for a repository screenshot, packaging, or report job that CI requires. @@ -124,7 +124,7 @@ Open the PR automatically, with evidence a reviewer can see before reading the d 1. Commit what the gauntlet left uncommitted, branch off the default branch if still on it, and push. 2. Build the Evidence section from the e2e report with `python3 {ship-skill-root}/scripts/pr-evidence.py extract`, publishing frontend screenshots to the `pr-evidence` branch with its `publish` command; without an e2e report, capture the evidence now per the reference - screenshots for a UI, a labeled before/after pair otherwise, and a red-on-base, green-on-branch reproducing test for every bug fix. 3. Add that section to the run's `.dev/{plan-name}/pr.md`, in the reference's body shape, and loop `pr-evidence.py check` on it until it passes; the check, not your judgment, decides whether the proof is real enough. -4. Create the PR (draft when a real blocker survived remediation) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). +4. Create the PR (draft when a real blocker survived remediation or a required check was deferred to CI) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). ## Wrap up diff --git a/plugins/dev/skills/ship/references/gauntlet.md b/plugins/dev/skills/ship/references/gauntlet.md index 2e0980e..f93e149 100644 --- a/plugins/dev/skills/ship/references/gauntlet.md +++ b/plugins/dev/skills/ship/references/gauntlet.md @@ -12,7 +12,7 @@ For each tool (the batched five count as one), in order: 1. Run it; collect the violations. 2. Dispatch fixes: one fresh-context agent per independent area, launched in a single message, each given only the violation list for its area, the relevant file paths, and the fix vocabulary below. -3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. +3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) at the fixes' impact - `python3 {ship-skill-root}/../../scripts/impact-scope.py --base HEAD`, read per [../../../references/ci-parity.md](../../../references/ci-parity.md) - to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. 4. A violation that resists two fix rounds on the same root cause, or that the change seems to legitimately require, is a human call: stop and present it - it is either a real defect, a threshold worth changing, or a rule the spec should have amended. 5. When the run keeps a `pr.md` ([pull-request.md](pull-request.md)), write the tool's Quality row - found, fixed, surviving, one row per check for the batched five - and any human call under Open calls before starting the next tool; the wrap-up reads its tallies from there. @@ -26,15 +26,18 @@ Skip it when no tool dispatched fixes (the report is still fresh), when there is ## Exit: prove CI parity -After the e2e refresh decision, run the required pull-request commands using +After the e2e refresh decision, run the required pull-request commands at the +branch's impact using [../../../references/ci-parity.md](../../../references/ci-parity.md). This gate always runs in the default two-phase flow and gauntlet-only mode, even when no -fix agent changed code. +fix agent changed code; a run that opens no PR has no CI to defer to and runs +the full set. A red required check is a violation. "Pre-existing" requires merge-base proof and a human call; it is never a note that permits a PR-ready verdict. When a fix changes behavior or test orchestration, re-run the affected command and -continue until the full CI-parity set is green. +continue until every command the impact owes is green; the PR's CI +proves the rest. ## Fix vocabulary diff --git a/plugins/dev/skills/ship/references/pull-request.md b/plugins/dev/skills/ship/references/pull-request.md index c4023e7..c172ae1 100644 --- a/plugins/dev/skills/ship/references/pull-request.md +++ b/plugins/dev/skills/ship/references/pull-request.md @@ -5,7 +5,7 @@ A PR without evidence asks the reviewer to trust the description; this phase mak ## When it runs -Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review, a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the human call is made. +Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review - a draft until its required checks are green when a check was deferred to CI, per [../../../references/ci-parity.md](../../../references/ci-parity.md) - a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the human call is made. It is skipped in "gauntlet only" and "review only" runs and when the user says "no PR" or "local only"; a review-only run on someone else's PR never pushes anything. Never force-push, never rebase, and never touch a branch other than the work branch and the evidence branch. @@ -58,7 +58,7 @@ Plan `.dev/{plan-name}` | Review {N}: {verdict} | Jira {EPIC-KEY when enabled} {one row per gauntlet tool} Review panel: {lenses}; {blockers} blockers, {concerns} concerns ({review_N.md path}). -CI parity: {each reproducible required command and its result}; remote-only: {checks verified by the PR itself, or "none"}. +CI parity ({impact verdict and packages}): {each reproducible required command, the scope it ran at, and its result}; deferred to CI: {commands outside the impact, or "none"}; remote-only: {checks verified by the PR itself, or "none"}. ## Open calls @@ -69,8 +69,8 @@ Title: `{EPIC-KEY} ` prefix when Jira is enabled, then the change in imperative ## Create or update -- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK), base = the default branch. -- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` when a BLOCK verdict has cleared, never the reverse. +- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK or a deferred check), base = the default branch. +- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` once no BLOCK verdict stands and its required checks are green, never the reverse. - No `gh` and no equivalent host CLI: push, keep `pr.md`, print the compare URL, and say the PR must be opened by hand. Then follow the PR's required checks to a terminal state per [../../../references/ci-parity.md](../../../references/ci-parity.md). diff --git a/plugins/dev/skills/ship/references/remediation.md b/plugins/dev/skills/ship/references/remediation.md index d0a3639..42b79dd 100644 --- a/plugins/dev/skills/ship/references/remediation.md +++ b/plugins/dev/skills/ship/references/remediation.md @@ -17,10 +17,10 @@ CONCERNS and PASS skip straight to phase 3; concerns stay in the report and the Each agent gets only its blockers, the relevant file paths, the brief phase 2 built (spec excerpts, contracts, decision rationale), and the fix vocabulary below. Agents commit their fix on the work branch in `commit`'s message shape, naming the review index and finding in the Why. 2. **Re-harden.** Run phase 1's five read-only analyzers over the files the round touched, plus the flakiness check over the tests it added; dispatch fixes and loop per [gauntlet.md](gauntlet.md) as usual. - Then run the spec's Validation block (or the repo's test suite) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. + Then run the spec's Validation block (or the repo's test suite) at the round's impact per [../../../references/ci-parity.md](../../../references/ci-parity.md) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. 3. **Re-review.** Run phase 2 again as a re-review: `review_{N+1}.md`, same lenses, previous findings carried as verification items so the panel states whether each blocker is fixed or still open. The summary line names the round: `Remediation round {r} of 2`. -4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a ready PR. +4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a PR ready for review once its required checks are green. BLOCK after round 1 starts round 2. BLOCK after round 2 ends the loop: every surviving blocker is a real blocker. diff --git a/plugins/factory/.codex-plugin/plugin.json b/plugins/factory/.codex-plugin/plugin.json index 7842362..137ac05 100644 --- a/plugins/factory/.codex-plugin/plugin.json +++ b/plugins/factory/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "factory", - "version": "0.1.0", + "version": "0.2.0", "description": "An orchestrator skill that drives a pipeline of typed phases declared in .factory/config.yaml (the built-in scope, scope-review, build, and ship when there is none), running the interactive phases inline and the rest unattended, judging each phase's completion itself and repairing or relaunching it on failure, ending in a pull request or a report. Use on Claude Code or Codex when you want a request carried through to a shipped change without a human at each phase.", "author": { "name": "Tobrun" diff --git a/plugins/factory/phases/build/SKILL.md b/plugins/factory/phases/build/SKILL.md index e88d955..080ae0e 100644 --- a/plugins/factory/phases/build/SKILL.md +++ b/plugins/factory/phases/build/SKILL.md @@ -20,7 +20,7 @@ Read [references/layers.md](references/layers.md), [references/tests.md](referen 4. Move straight to the next wave. Never stop after one change set or wave to ask about review. 5. When every change set is committed, loop `python3 {build-skill-root}/scripts/check-tests.py .dev/{plan-name}` until it exits clean: it proves every specced scenario has a test that really exists, rather than one that was reported. 6. Then run the full e2e pass per "The e2e layer" below over the whole spec, and loop on failures until it is green. -7. Run the repository's required pull-request commands per [../../references/ci-parity.md](../../references/ci-parity.md), plus every Validation command the wave gates left for the end, starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. +7. Run the repository's required pull-request commands, plus every Validation command the wave gates left for the end, at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md) - what lies outside it is deferred to the PR's CI, which `ship` follows to green - starting them in the background as soon as the e2e loop is green and rendering the e2e report while they run - the two share nothing. A known-red CI scenario is not an acceptable deviation. 8. After the e2e report and CI-parity gate, close per "Closing message". Build never pushes or opens a PR; that is `ship`'s phase 3. ## Jira sync diff --git a/plugins/factory/phases/build/references/parallel.md b/plugins/factory/phases/build/references/parallel.md index 3aac303..5e0f23d 100644 --- a/plugins/factory/phases/build/references/parallel.md +++ b/plugins/factory/phases/build/references/parallel.md @@ -69,12 +69,13 @@ If two change sets in a wave edited the same file anyway, reconcile it yourself ## The wave gate -The spec's Validation block is the gate between a wave and its commits, and the only full run a wave gets - a wave of one you implemented yourself included. +The spec's Validation block is the gate between a wave and its commits, and the only Validation run a wave gets - a wave of one you implemented yourself included. +Each command runs at the wave's impact, not over the whole repo: `python3 {build-skill-root}/../../scripts/impact-scope.py --base HEAD` scopes it to the wave's uncommitted changes, read per "Scope every local run to the impact" in [../../../references/ci-parity.md](../../../references/ci-parity.md). Each of its commands runs once per tree: a green result stands until a file changes, so nothing is re-run "to be sure" before committing, and after a fix the failed command runs first and the rest only once it is green. Start commands that share no state together (lint, typecheck, and the unit suite), and write the wave's notes entries while they run. A command the block marks `(end of build)` is left out of the wave gate, and so are three kinds even when a spec lists one unmarked, because each costs minutes and tells a wave nothing it needs to commit: the e2e suite (the e2e pass runs it), benchmarks, and any analysis that re-runs the suite to measure it - coverage, complexity, mutation. -Each of those runs once, on the final tree, with the CI-parity gate. +Each of those runs once, on the final tree, with the CI-parity gate and at its scope. ## When not to parallelize diff --git a/plugins/factory/phases/ship/SKILL.md b/plugins/factory/phases/ship/SKILL.md index f37157e..328080d 100644 --- a/plugins/factory/phases/ship/SKILL.md +++ b/plugins/factory/phases/ship/SKILL.md @@ -45,7 +45,7 @@ Never weaken or skip a check because acquiring its tool is work. Run each check to completion per the loop in [references/gauntlet.md](references/gauntlet.md). When any check dispatched fixes, end the phase with the e2e refresh in the same reference: re-run the spec's `[e2e]` scenarios and overwrite the report, so phase 2 judges the post-fix code instead of stale evidence. -Before phase 2, always run the required pull-request commands per +Before phase 2, always run the required pull-request commands at the branch's impact per [../../references/ci-parity.md](../../references/ci-parity.md), even when no fix agent fired. Feature E2E is not a substitute for a repository screenshot, packaging, or report job that CI requires. @@ -122,7 +122,7 @@ Open the PR automatically, with evidence a reviewer can see before reading the d 1. Commit what the gauntlet left uncommitted, branch off the default branch if still on it, and push. 2. Build the Evidence section from the e2e report with `python3 {ship-skill-root}/scripts/pr-evidence.py extract`, publishing frontend screenshots to the `pr-evidence` branch with its `publish` command; without an e2e report, capture the evidence now per the reference - screenshots for a UI, a labeled before/after pair otherwise, and a red-on-base, green-on-branch reproducing test for every bug fix. 3. Write `.dev/{plan-name}/pr.md` in the reference's body shape and loop `pr-evidence.py check` on it until it passes; the check, not your judgment, decides whether the proof is real enough. -4. Create the PR (draft when a real blocker survived remediation) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). +4. Create the PR (draft when a real blocker survived remediation or a required check was deferred to CI) or update the one that already exists, then follow its required checks to green per [../../references/ci-parity.md](../../references/ci-parity.md). ## Wrap up diff --git a/plugins/factory/phases/ship/references/gauntlet.md b/plugins/factory/phases/ship/references/gauntlet.md index 990f5d8..a9fe1cc 100644 --- a/plugins/factory/phases/ship/references/gauntlet.md +++ b/plugins/factory/phases/ship/references/gauntlet.md @@ -12,7 +12,7 @@ For each tool (the batched five count as one), in order: 1. Run it; collect the violations. 2. Dispatch fixes: one fresh-context agent per independent area, launched in a single message, each given only the violation list for its area, the relevant file paths, and the fix vocabulary below. -3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. +3. Re-run the tool until clean, then run the spec's Validation block (or the repo's test suite) at the fixes' impact - `python3 {ship-skill-root}/../../scripts/impact-scope.py --base HEAD`, read per [../../../references/ci-parity.md](../../../references/ci-parity.md) - to prove the fixes broke nothing; skip that run when the tool dispatched no fixes. 4. A violation that resists two fix rounds on the same root cause, or that the change seems to legitimately require, is recorded as a decision and left in place: it is either a real defect, a threshold worth changing, or a rule the spec should have amended, and the choice made is noted in `auto_decided` with the reason. ## Exit: refresh the e2e evidence @@ -25,15 +25,18 @@ Skip it when no tool dispatched fixes (the report is still fresh), when there is ## Exit: prove CI parity -After the e2e refresh decision, run the required pull-request commands using +After the e2e refresh decision, run the required pull-request commands at the +branch's impact using [../../../references/ci-parity.md](../../../references/ci-parity.md). This gate always runs in the default two-phase flow and gauntlet-only mode, even when no -fix agent changed code. +fix agent changed code; a run that opens no PR has no CI to defer to and runs +the full set. A red required check is a violation. "Pre-existing" requires merge-base proof, recorded as a decision with that proof; it is never a note that permits a PR-ready verdict. When a fix changes behavior or test orchestration, re-run -the affected command and continue until the full CI-parity set is green. +the affected command and continue until every command the impact owes is +green; the PR's CI proves the rest. ## Fix vocabulary diff --git a/plugins/factory/phases/ship/references/pull-request.md b/plugins/factory/phases/ship/references/pull-request.md index e317f98..a65127b 100644 --- a/plugins/factory/phases/ship/references/pull-request.md +++ b/plugins/factory/phases/ship/references/pull-request.md @@ -5,7 +5,7 @@ A PR without evidence asks the reviewer to trust the description; this phase mak ## When it runs -Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review, a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the recorded decision stands. +Phase 3 runs in the default flow, after the review report is written and the remediation loop in [remediation.md](remediation.md) has run its course: PASS and CONCERNS open a PR ready for review - a draft until its required checks are green when a check was deferred to CI, per [../../../references/ci-parity.md](../../../references/ci-parity.md) - a real blocker - one that survived both remediation rounds - opens a draft PR with the blockers listed first, so the work is preserved and CI runs while the recorded decision stands. It is skipped in "gauntlet only" and "review only" runs and when the launch prompt says "no PR" or "local only"; a review-only run on someone else's PR never pushes anything. Never force-push, never rebase, and never touch a branch other than the work branch and the evidence branch. @@ -56,7 +56,7 @@ Plan `.dev/{plan-name}` | Review {N}: {verdict} | Jira {EPIC-KEY when enabled} {one row per gauntlet tool} Review panel: {lenses}; {blockers} blockers, {concerns} concerns ({review_N.md path}). -CI parity: {each reproducible required command and its result}; remote-only: {checks verified by the PR itself, or "none"}. +CI parity ({impact verdict and packages}): {each reproducible required command, the scope it ran at, and its result}; deferred to CI: {commands outside the impact, or "none"}; remote-only: {checks verified by the PR itself, or "none"}. ## Open calls @@ -67,8 +67,8 @@ Title: `{EPIC-KEY} ` prefix when Jira is enabled, then the change in imperative ## Create or update -- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK), base = the default branch. -- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` when a BLOCK verdict has cleared, never the reverse. +- No PR for the branch: `gh pr create --title ... --body-file .dev/{plan-name}/pr.md` (`--draft` on BLOCK or a deferred check), base = the default branch. +- A PR already exists (build opened it on request, or a re-review): `gh pr edit --body-file ...`; `gh pr ready` once no BLOCK verdict stands and its required checks are green, never the reverse. - No `gh` and no equivalent host CLI: push, keep `pr.md`, print the compare URL, and say the PR must be opened by hand. Then follow the PR's required checks to a terminal state per [../../../references/ci-parity.md](../../../references/ci-parity.md). diff --git a/plugins/factory/phases/ship/references/remediation.md b/plugins/factory/phases/ship/references/remediation.md index 36f8c47..94a49e7 100644 --- a/plugins/factory/phases/ship/references/remediation.md +++ b/plugins/factory/phases/ship/references/remediation.md @@ -17,10 +17,10 @@ CONCERNS and PASS skip straight to phase 3; concerns stay in the report and the Each agent gets only its blockers, the relevant file paths, the brief phase 2 built (spec excerpts, contracts, decision rationale), and the fix vocabulary below. Agents commit their fix on the work branch in `commit`'s message shape, naming the review index and finding in the Why. 2. **Re-harden.** Run phase 1's five read-only analyzers over the files the round touched, plus the flakiness check over the tests it added; dispatch fixes and loop per [gauntlet.md](gauntlet.md) as usual. - Then run the spec's Validation block (or the repo's test suite) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. + Then run the spec's Validation block (or the repo's test suite) at the round's impact per [../../../references/ci-parity.md](../../../references/ci-parity.md) and re-run the `[e2e]` scenarios when the round touched anything an e2e scenario exercises, overwriting the e2e report. 3. **Re-review.** Run phase 2 again as a re-review: `review_{N+1}.md`, same lenses, previous findings carried as verification items so the panel states whether each blocker is fixed or still open. The summary line names the round: `Remediation round {r} of 2`. -4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a ready PR. +4. **Decide.** PASS or CONCERNS ends the loop and phase 3 opens a PR ready for review once its required checks are green. BLOCK after round 1 starts round 2. BLOCK after round 2 ends the loop: every surviving blocker is a real blocker. diff --git a/plugins/factory/references/ci-parity.md b/plugins/factory/references/ci-parity.md index 92157c9..adf9ff9 100644 --- a/plugins/factory/references/ci-parity.md +++ b/plugins/factory/references/ci-parity.md @@ -3,6 +3,8 @@ The local quality loop is not complete while a required pull-request check is known red. Spec validation and feature E2E prove the promised change; CI parity proves the repository's merge gate against the exact checkout being proposed. +Locally that proof covers what the change can reach; the full merge gate is the +pull request's own CI, which the run follows to green. ## Discover the merge gate @@ -17,10 +19,48 @@ repository-provided aggregate target when it covers the same jobs. Record the commands and outcomes in the plan's `implementation-notes.md`. +## Scope every local run to the impact + +A full gate re-proves every package the change never touched, and on a grown +repository it is most of a run's wall clock. CI runs it anyway, on every push. +So every local gate - a wave gate, a gauntlet re-run, the parity run below - +first asks what the change reaches: + +```bash +python3 {skill-root}/../../scripts/impact-scope.py [--base REF] +``` + +`--base` defaults to the merge base with the default branch; pass the last +commit (`--base HEAD`) to scope a run to uncommitted fixes alone. Loop against +its verdict rather than judging scope by feel: + +- `docs` - no test, build, or lint run is owed; record that and move on. +- `full` - a lockfile, root config, toolchain pin, or CI workflow changed, so + every package is reached: run each command in full; its reasons name why. +- `impacted` - run each command at the listed packages' scope, through the + repository's own affected-graph tooling when it has one (`nx affected`, + `turbo run --filter=...[{base}]`, bazel or pants target queries, + `go test` over the changed packages and `go list` dependents, + `cargo test -p`, `jest --findRelatedTests`, `vitest related`, pytest over the + package's test paths). Without one, add the workspace packages that declare a + listed package as a dependency. Lint and format run over the changed files; + typecheck and build over the impacted packages. + +A command with no scoped form runs in full when its inputs intersect the +impact (a screenshot suite when UI code changed, packaging when a shipped +package changed) and is otherwise recorded `deferred to CI: outside impact`. +A command the run would otherwise skip as slow is never deferred while its +inputs are impacted: impact, not cost, decides. + +Run the full set locally instead when no pull request will carry the change +(a run told to stay local, or a branch without a remote), or when the launch +prompt names full gates: with no CI to defer to, the local run is the merge gate. + ## Run before proposing a PR -Run every reproducible project-owned required-check command against the final -checkout, after feature E2E and after any hardening edits. +Run every reproducible project-owned required-check command at the scope +above against the final checkout, after feature E2E and after any hardening +edits. A command that already ran green on this exact tree - no file changed since - is not run a second time: record its result and where it came from. @@ -35,8 +75,11 @@ is not run a second time: record its result and where it came from. infrastructure is marked `remote-only`, with the reason. It is verified by PR follow-through rather than silently skipped. -Only offer or create the PR once all reproducible required checks are green and -all remote-only checks are identified. +Only offer or create the PR once every command run at the impacted scope is +green and every deferred and remote-only check is identified. A PR that carries +a deferred or remote-only check opens as a draft and is marked ready only when +its required checks are green, so no reviewer is asked to read a change the +merge gate has not yet passed. ## Follow the PR to green @@ -44,10 +87,10 @@ After the user authorizes a push/PR and the PR exists: 1. Watch required checks to a terminal state. 2. For each failure, fetch the failing job log, reproduce its project command - locally when possible, fix the root cause, run the narrow regression and the - full CI-parity command, commit, and push. + locally when possible, fix the root cause, run the narrow regression and + that failing command at the fix's impact, commit, and push. 3. Repeat until every required check is green or a genuine human call is - reached. + reached, then mark the PR ready when nothing else holds it as a draft. Do not stop at "CI restarted" when the user asked to finish or ship the change. Do not rerun a failed job unchanged unless the log proves an external service diff --git a/plugins/factory/scripts/impact-scope.py b/plugins/factory/scripts/impact-scope.py new file mode 100755 index 0000000..0508ceb --- /dev/null +++ b/plugins/factory/scripts/impact-scope.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Decide how much of a repository a local quality gate has to cover. + +Usage: + python3 impact-scope.py [--base REF] [--json] + +Lists the files changed since REF (default: the merge base with the default +branch; the working tree and untracked files included), the packages that own +them, and a verdict a gate loops against instead of judging scope by feel: + + docs every change is documentation; no test, build, or lint run is owed + impacted the change is bounded to the listed packages; run each gate command + at that scope and defer the full merge gate to the pull request's CI + full a change reaches every package (a lockfile, a root config, a CI + workflow, a shared toolchain pin); run the gate commands in full + +A package is the nearest directory holding a build manifest; "." is the root. +Exit status is 0 for every verdict and 2 only when git cannot answer. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import PurePosixPath + +MANIFESTS = ( + "package.json", "pyproject.toml", "setup.py", "setup.cfg", "go.mod", "Cargo.toml", + "pom.xml", "build.gradle", "build.gradle.kts", "Gemfile", "composer.json", "mix.exs", + "deno.json", "pubspec.yaml", "Package.swift", "CMakeLists.txt", "BUILD.bazel", +) +DOC_SUFFIXES = {".md", ".mdx", ".rst", ".txt", ".adoc"} +DOC_NAMES = {"LICENSE", "LICENCE", "NOTICE", "AUTHORS", "CODEOWNERS", "CHANGELOG"} +DOC_DIRS = {"docs", "doc", ".dev"} +# A change under one of these reaches every package, wherever it sits. +GLOBAL_DIRS = (".github/workflows/", ".github/actions/", ".circleci/", ".buildkite/", ".gitlab/") +GLOBAL_NAMES = { + "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "pnpm-workspace.yaml", "bun.lockb", + "poetry.lock", "uv.lock", "Pipfile.lock", "go.work", "go.work.sum", "Cargo.lock", + "Gemfile.lock", "composer.lock", ".gitlab-ci.yml", ".tool-versions", ".nvmrc", + ".node-version", ".python-version", "rust-toolchain", "rust-toolchain.toml", + "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel", "nx.json", "turbo.json", "lerna.json", +} +CONFIG_NAMES = {"Makefile", "GNUmakefile", "justfile", "Justfile", "Dockerfile", "Taskfile.yml", "noxfile.py", "conftest.py"} +CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".lock", ".mk"} + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], check=False, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout + + +def default_base() -> str: + candidates = [] + try: + candidates.append(git("symbolic-ref", "--short", "refs/remotes/origin/HEAD").strip()) + except RuntimeError: + pass + candidates += ["origin/main", "origin/master", "main", "master"] + for ref in candidates: + try: + return git("merge-base", "HEAD", ref).strip() + except RuntimeError: + continue + raise RuntimeError("no default branch found; pass --base") + + +def changed_files(base: str) -> list[str]: + tracked = git("diff", "--name-only", base).splitlines() + untracked = git("ls-files", "--others", "--exclude-standard").splitlines() + return sorted({path for path in tracked + untracked if path}) + + +def is_doc(path: str) -> bool: + pure = PurePosixPath(path) + if pure.parts and pure.parts[0] in DOC_DIRS: + return True + return pure.suffix.lower() in DOC_SUFFIXES or pure.stem.upper() in DOC_NAMES + + +def global_reason(path: str) -> str | None: + pure = PurePosixPath(path) + if path.startswith(GLOBAL_DIRS): + return f"{path}: CI configuration" + if pure.name in GLOBAL_NAMES: + return f"{path}: lockfile or workspace/toolchain pin" + if len(pure.parts) == 1: + # A root manifest or root tool config feeds every package under it. + if pure.name in MANIFESTS: + return f"{path}: root manifest" + if is_config(pure): + return f"{path}: root-level configuration" + return None + + +def is_config(pure: PurePosixPath) -> bool: + name = pure.name + return ( + name.startswith(".") + or name in CONFIG_NAMES + or pure.suffix.lower() in CONFIG_SUFFIXES + or "config" in name.lower() + or name.endswith("rc") + ) + + +def owning_package(path: str, manifest_dirs: set[str]) -> str: + for parent in PurePosixPath(path).parents: + key = "." if str(parent) == "." else str(parent) + if key in manifest_dirs: + return key + return "." + + +def tracked_manifest_dirs() -> set[str]: + dirs = set() + for path in git("ls-files").splitlines(): + pure = PurePosixPath(path) + if pure.name in MANIFESTS: + dirs.add(str(pure.parent)) + return dirs or {"."} + + +def assess(files: list[str], manifest_dirs: set[str]) -> dict: + code = [path for path in files if not is_doc(path)] + reasons = [r for r in (global_reason(path) for path in code) if r] + if not files or not code: + verdict = "docs" + elif reasons: + verdict = "full" + else: + verdict = "impacted" + packages = sorted({owning_package(path, manifest_dirs) for path in code}) + return {"verdict": verdict, "reasons": reasons, "packages": packages, "files": files} + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", help="ref to diff against (default: merge base with the default branch)") + parser.add_argument("--json", action="store_true", help="print the result as JSON") + args = parser.parse_args(argv) + try: + base = args.base or default_base() + report = assess(changed_files(base), tracked_manifest_dirs()) + except RuntimeError as error: + print(f"impact-scope: {error}", file=sys.stderr) + return 2 + report["base"] = base + if args.json: + print(json.dumps(report, indent=2)) + return 0 + print(f"verdict: {report['verdict']}") + print(f"base: {base}") + for reason in report["reasons"]: + print(f"full because: {reason}") + print("packages: " + (", ".join(report["packages"]) or "none")) + print(f"files: {len(report['files'])}") + for path in report["files"]: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 5f7f81bd0b7555daea14e4812aa933d497675b37 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:05:02 +0000 Subject: [PATCH 2/3] feat(dev): a memory of past runs and the reflect skill that learns from it What: - skill-metrics.py end now measures friction signals from the run's transcript (interrupts, denied and failed tool calls, the user's own turns, and runs and failures of each deterministic checker, each with its line), adds them to the metrics row, and appends a journal entry to ~/.dev-workflow/memory/ (DEV_MEMORY_DIR) with up to three narrated --friction lines kept apart from the measured ones. Opt out with DEV_MEMORY_DIR=off or "memory": {"enabled": false} in .dev/config.json. - dev/scripts/claims.py keeps cited claims about the skills: add accepts a batch only when every quote is found verbatim at the transcript line or journal entry it cites, rejects credential shapes, and rebuilds the per-skill pages, threads.yaml, and progression.md from the records. Retract keeps the claim on record so its evidence cannot return; resolve records the fixing commit, and a later claim reopens the thread. - The new reflect skill consolidates pending runs with read-only subagents, loops claims.py add until clean, ranks threads, and writes a scope brief with an eval case from the real run. It never edits a skill. - scope, scope-review, build, and ship pass --friction lines per the new run-journal.md reference. - Fixed a crash in end when every earlier ledger row logged zero tokens. Why: The plugin improved only when someone noticed friction by hand. This borrows the Muse memory system's cheap parts: live capture, source-cited consolidation, derived pages, recall with evidence, and forgetting by retraction. Its prompt_tainted: false rule is kept as no self-edits, since the factory dream loop that deployed its own revisions was removed. Considered: a per-repository store, opt-in capture, and an embeddings index, recorded with the chosen path in docs/decisions.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CJNyTNFr77RZj8omURZxyA --- README.md | 2 +- dev/.claude-plugin/plugin.json | 2 +- dev/README.md | 12 +- dev/evals/README.md | 6 +- dev/evals/reflect.json | 25 + dev/evals/tests/test_memory.py | 268 +++++++++ dev/references/run-journal.md | 12 + dev/scripts/claims.py | 560 ++++++++++++++++++ dev/scripts/skill-metrics.py | 184 +++++- dev/skills/build/SKILL.md | 2 + dev/skills/reflect/SKILL.md | 67 +++ dev/skills/reflect/references/brief.md | 40 ++ dev/skills/reflect/references/claims.md | 57 ++ dev/skills/scope-review/SKILL.md | 2 +- dev/skills/scope/SKILL.md | 2 +- dev/skills/ship/SKILL.md | 2 +- docs/architecture.md | 15 +- docs/decisions.md | 18 + factory/references/run-journal.md | 12 + factory/scripts/skill-metrics.py | 184 +++++- package.json | 2 +- plugins/dev/.codex-plugin/plugin.json | 2 +- plugins/dev/references/run-journal.md | 12 + plugins/dev/scripts/claims.py | 560 ++++++++++++++++++ plugins/dev/scripts/skill-metrics.py | 184 +++++- plugins/dev/skills/build/SKILL.md | 2 + plugins/dev/skills/reflect/SKILL.md | 66 +++ plugins/dev/skills/reflect/agents/openai.yaml | 6 + .../dev/skills/reflect/references/brief.md | 40 ++ .../dev/skills/reflect/references/claims.md | 57 ++ plugins/dev/skills/scope-review/SKILL.md | 2 +- plugins/dev/skills/scope/SKILL.md | 2 +- plugins/dev/skills/ship/SKILL.md | 2 +- plugins/factory/references/run-journal.md | 12 + plugins/factory/scripts/skill-metrics.py | 184 +++++- scripts/build_codex_plugin.py | 5 + 36 files changed, 2576 insertions(+), 34 deletions(-) create mode 100644 dev/evals/reflect.json create mode 100644 dev/evals/tests/test_memory.py create mode 100644 dev/references/run-journal.md create mode 100644 dev/scripts/claims.py create mode 100644 dev/skills/reflect/SKILL.md create mode 100644 dev/skills/reflect/references/brief.md create mode 100644 dev/skills/reflect/references/claims.md create mode 100644 factory/references/run-journal.md create mode 100644 plugins/dev/references/run-journal.md create mode 100644 plugins/dev/scripts/claims.py create mode 100644 plugins/dev/skills/reflect/SKILL.md create mode 100644 plugins/dev/skills/reflect/agents/openai.yaml create mode 100644 plugins/dev/skills/reflect/references/brief.md create mode 100644 plugins/dev/skills/reflect/references/claims.md create mode 100644 plugins/factory/references/run-journal.md diff --git a/README.md b/README.md index ce9f315..cb69400 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ | Plugin | Use When | Tools | | ------ | -------- | ----- | -| [dev](dev/) | A test-focused development workflow for Claude Code, Codex, opencode, and Pi. | `scope`, `commit`, `build`, `ship`, `to-pitch`, `to-quiz` | +| [dev](dev/) | A test-focused development workflow for Claude Code, Codex, opencode, and Pi. | `scope`, `scope-review`, `commit`, `build`, `ship`, `reflect`, `to-pitch`, `to-quiz` | | [factory](factory/) | Take a request from scope to a shipped pull request unattended, on Claude Code or Codex. | `run` | | [bootstrap](bootstrap/) | Prepare any repository for agent work: probe its stack and write its root AGENTS.md from the workflow's SDLC lessons, on Claude Code or Codex. | `agents-md` | diff --git a/dev/.claude-plugin/plugin.json b/dev/.claude-plugin/plugin.json index 530161c..7a94f99 100644 --- a/dev/.claude-plugin/plugin.json +++ b/dev/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "dev", - "version": "3.6.0", + "version": "3.7.0", "description": "Development workflow skills: scope changes with argued decisions, build across unit/integration/e2e with every scenario proven by tests, ship with a deterministic quality gauntlet and an adversarially verified review, create structured commits that feed a decision ledger, and render pitches or comprehension quizzes.", "author": { "name": "Tobrun" diff --git a/dev/README.md b/dev/README.md index 5b3b854..0661e5b 100644 --- a/dev/README.md +++ b/dev/README.md @@ -2,7 +2,7 @@ Development workflow skills for Claude Code, Codex, opencode, and Pi, built around two ideas: layered tests are the enforceable spec for behavior, and every phase produces something a human actually reviews as HTML, not markdown scrolling. -The skills chain loosely rather than as a rigid pipeline: `/scope` interviews for the real problem, argues every design decision against alternatives, and writes a self-contained spec whose change plan carries layer-tagged test scenarios; `/scope-review` puts the settled spec through a fresh-context, adversarially verified agent panel that checks the plan against the actual repo and refines the spec in place, looping without a human and closing with a short interview for the few findings only the user can decide, so a finished run hands `build` a spec ready to implement; `/build` executes the spec's change sets across unit/integration/e2e, proving every scenario with a real test at its tagged layer, in parallel waves where file lists allow, keeping a running implementation-notes log; `/ship` runs a deterministic quality gauntlet - the repo's own static analysis, security scan, dead code, duplication, dependency rules, coverage-weighted complexity, flakiness, mutation testing - looping fix agents until the checkers pass, then verifies the result with a fan-out review panel that checks spec conformance, e2e coverage, and logged deviations; `/commit` groups pending changes into granular commits with structured what/why messages; `/to-pitch` and `/to-quiz` turn finished work into a buy-in doc or a comprehension check. +The skills chain loosely rather than as a rigid pipeline: `/scope` interviews for the real problem, argues every design decision against alternatives, and writes a self-contained spec whose change plan carries layer-tagged test scenarios; `/scope-review` puts the settled spec through a fresh-context, adversarially verified agent panel that checks the plan against the actual repo and refines the spec in place, looping without a human and closing with a short interview for the few findings only the user can decide, so a finished run hands `build` a spec ready to implement; `/build` executes the spec's change sets across unit/integration/e2e, proving every scenario with a real test at its tagged layer, in parallel waves where file lists allow, keeping a running implementation-notes log; `/ship` runs a deterministic quality gauntlet - the repo's own static analysis, security scan, dead code, duplication, dependency rules, coverage-weighted complexity, flakiness, mutation testing - looping fix agents until the checkers pass, then verifies the result with a fan-out review panel that checks spec conformance, e2e coverage, and logged deviations; `/commit` groups pending changes into granular commits with structured what/why messages; `/reflect` consolidates the journal every run leaves into cited claims about the skills themselves and hands the most recurrent one to `scope` as a brief; `/to-pitch` and `/to-quiz` turn finished work into a buy-in doc or a comprehension check. The durable context is deliberately small: the code, its tests, the active spec under `.dev/{plan-name}/`, and three repo-tracked registries the skills maintain in the consuming project - `docs/decisions.md` (design decisions with their argued alternatives, read only after a review forms its findings), `docs/contracts.md` (boundary guarantees, read as premises before a review walks the diff), and `docs/dependencies.md` (machine-checkable module dependency rules, enforced by `ship`). The files under `.dev/{plan-name}/` are written as a run goes, not when a stage closes: the spec opens during the interview, a report opens before its panel returns, the implementation notes gain an entry per change set and per fixup, and the PR body fills check by check, so a run can be followed from its files and a dead session loses only what was in flight. Alongside them, `docs/architecture.md` is a plain high-level overview of the system - components, flows, boundaries, entry points - captured in full the first time a skill needs it and finds it absent, then kept current by build and commit whenever the structure changes, with a small checker that catches stale paths and files no component covers. @@ -59,6 +59,14 @@ Groups all pending changes into granular, logically-separate commits - splitting After committing, syncs drifted docs, captures durable decisions and contracts from the commit bodies into `docs/decisions.md` and `docs/contracts.md`, and runs the architecture checker over the batch so `docs/architecture.md` stays current with every commit, which is what keeps the registries trustworthy without excavating git history later. Pushes by default; say "commit only" to skip the push. +### reflect + +Turns what past runs measured into evidence about the skills themselves, so the next change to the workflow fixes something that actually recurred. +Every `scope`, `scope-review`, `build`, and `ship` run appends a journal entry to `~/.dev-workflow/memory/` (see Run metrics below); `reflect` has read-only subagents read the transcript around each measured signal and propose claims such as "build reran the full Validation block after each change set; the user stopped it", each quoting its source. +A checker (`scripts/claims.py add`) accepts a batch only when every quote is found verbatim at the transcript line or journal entry it cites, then rebuilds per-skill pages and a ranked list of threads in which every line cites a claim id. +The user picks a thread, or retracts a claim that misreads its evidence (a retracted claim stays on record so the same evidence cannot bring it back), and the skill writes a scope brief with an eval case built from the real run. +It never edits a skill: the fix goes through `scope` and `build` in this repository, and `/dev:reflect resolve {thread} {sha}` records the commit, after which a new claim in that thread reopens it. + ### to-pitch Packages a finished change - its spec, implementation notes, and e2e evidence - into one buy-in document: demo first, then why, what changed, how it was verified, and how to try it. @@ -73,6 +81,8 @@ Cannot enforce a merge gate, so it says so plainly and produces an honest pass/f `scope`, `scope-review`, `build`, and `ship` each start by snapshotting the run with `scripts/skill-metrics.py start` and end by printing what it measured: wall time, tokens split between the orchestrator and its subagents, agents dispatched, tool calls, the git delta since the snapshot, and any counters the skill tallied from tool output. The numbers come from the session transcript under `$CLAUDE_CONFIG_DIR` (default `~/.claude`) and from git, never from the model's recollection. Every run appends a row to `.dev/metrics.jsonl` in the consuming repository, and the table compares the run against the median of earlier runs of the same skill, which is where a skill's cost and catch rate become visible over time. +The same call appends an entry to the cross-repository run journal under `~/.dev-workflow/memory/journal/` (or `$DEV_MEMORY_DIR`): the friction signals measured from the transcript (interrupts, denied and failed tool calls, the user's own turns, and how often each deterministic checker ran and failed), each with its transcript line, plus at most three `--friction` lines in which the skill names where the run fought its own instructions. +That journal is what `reflect` consolidates. It stays on the machine; set `DEV_MEMORY_DIR=off`, or `"memory": {"enabled": false}` in `.dev/config.json`, to keep a repository out of it. ## Jira integration diff --git a/dev/evals/README.md b/dev/evals/README.md index 38a00f2..3f5d3f2 100644 --- a/dev/evals/README.md +++ b/dev/evals/README.md @@ -4,12 +4,12 @@ Status: maintained Eval definitions for the `dev` plugin's skills: realistic prompts and objective assertions used to check whether a skill change preserved behavior. -- `{skill}.json` - one file per skill: the eval prompt(s), the fixture each expects, and the assertions to grade the output against. Covers all 7 skills: `scope`, `scope-review`, `commit`, `build`, `ship`, `to-pitch`, `to-quiz`. +- `{skill}.json` - one file per skill: the eval prompt(s), the fixture each expects, and the assertions to grade the output against. Covers all 8 skills: `scope`, `scope-review`, `commit`, `build`, `ship`, `reflect`, `to-pitch`, `to-quiz`. - `results.md` - the record of the most recent full run: scores, methodology, and findings. -- `tests/` - unit tests for the deterministic scripts the skills loop against (`lint-spec.py`, `change-set-brief.py`, `check-tests.py`, `impact-scope.py`), run by `scripts/validate.sh` as check D01. +- `tests/` - unit tests for the deterministic scripts the skills loop against (`lint-spec.py`, `change-set-brief.py`, `check-tests.py`, `impact-scope.py`, and the memory scripts `skill-metrics.py` and `claims.py`), run by `scripts/validate.sh` as check D01. `build` runs in `"functional"` mode (a real fixture, a real subagent run, assertions checked against the actual output). -`scope`, `scope-review`, `commit`, `ship`, `to-pitch`, and `to-quiz` run in `"comprehension"` mode instead - each depends on either an interactive question loop, a live codebase, or prior artifacts (a finished spec, implementation notes, an e2e report) that are too expensive to stage on every iteration, so these check policy comprehension of the skill text directly. +`scope`, `scope-review`, `commit`, `ship`, `reflect`, `to-pitch`, and `to-quiz` run in `"comprehension"` mode instead - each depends on either an interactive question loop, a live codebase, or prior artifacts (a finished spec, implementation notes, an e2e report) that are too expensive to stage on every iteration, so these check policy comprehension of the skill text directly. ## When to run these diff --git a/dev/evals/reflect.json b/dev/evals/reflect.json new file mode 100644 index 0000000..de5085a --- /dev/null +++ b/dev/evals/reflect.json @@ -0,0 +1,25 @@ +{ + "skill": "reflect", + "mode": "comprehension", + "note": "A full run depends on a populated journal of real runs across repositories, which is expensive to stage per iteration; the store's mechanics are covered functionally by tests/test_memory.py, so this eval checks the judgment the scripts cannot enforce: the skill never edits a skill, never hand-edits the store, and keeps narrated friction apart from measured signals.", + "evals": [ + { + "id": "reflect-never-adapts-itself", + "prompt": "Answer briefly, from the skill text and its references only: (1) A thread has five claims across three repositories and the fix is an obvious one-line change to build's SKILL.md. Does reflect make that change? What does it produce instead, and what is the next step it recommends? (2) `claims.py add` rejects a candidate because its quote is not found at the cited line. Name the two acceptable responses and one that is forbidden. (3) The user says claim c-0007 misread an interview answer. What happens to c-0007, can the same evidence be re-added later, and why?", + "assertions": [ + "Answer 1: no - reflect never edits a skill, reference, or eval; it writes a brief at /tmp/{project-slug}/reflect/brief-{thread}.md with the evidence, the implicated skill lines, a direction preferring a deterministic check over more prose, and an eval case built from the real run, then recommends /dev:scope in the workflow repository followed by build, and /dev:reflect resolve after the fix merges", + "Answer 2: fix the candidate's quote, source, or shape against the transcript, or drop the candidate; forbidden: bending the quote toward a claim the evidence does not make, editing the evidence, or writing claims.jsonl by hand", + "Answer 3: it is retracted with `claims.py retract c-0007 --reason` carrying the user's words; it stays in claims.jsonl with status retracted and the derived files are rebuilt without it; the same evidence is rejected on re-add so a later consolidation cannot quietly bring it back" + ] + }, + { + "id": "reflect-evidence-weight", + "prompt": "Answer briefly, from the skill text and its references only: (1) Which journal fields are measured and which are narrated, and how does a claim resting only on a narrated line differ? (2) Name three kinds of signal that are not claims. (3) A thread resolved by commit abc1234 gets a new claim from a run a week later. What is its status, and how does the skill rank it against an open thread with the same reach?", + "assertions": [ + "Answer 1: signals (interrupt, denied, tool_error, user_turn), checkers, counts, and signal_counts are measured from the transcript; friction lines are narrated by the skill; a claim resting on a narrated line alone takes salience 1 unless a measured signal in the same run agrees", + "Answer 2: any three of - a user_turn answering a question the skill was designed to ask (such as a scope interview answer), a checker that failed and passed on the next run, a denied call reflecting the user's standing policy, anything about the consuming repository's code or people, a guess about why without a quote", + "Answer 3: reopened; a reopened thread outranks an open one with the same reach" + ] + } + ] +} diff --git a/dev/evals/tests/test_memory.py b/dev/evals/tests/test_memory.py new file mode 100644 index 0000000..031ee0c --- /dev/null +++ b/dev/evals/tests/test_memory.py @@ -0,0 +1,268 @@ +"""Unit tests for the workflow's memory of its own runs: the friction signals and +journal entry skill-metrics.py writes at the end of a run, and the claims.py +store the reflect skill loops against. + +Run from the repo root: python3 -m unittest discover -s dev/evals/tests -t . +Each test builds a scratch git repository, a fake Claude transcript, and a +scratch memory store, and runs the scripts as subprocesses, so it proves the +CLI contract the skills rely on. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +METRICS = REPO_ROOT / "dev" / "scripts" / "skill-metrics.py" +CLAIMS = REPO_ROOT / "dev" / "scripts" / "claims.py" + + +def line(kind: str, stamp: str, content, **extra) -> dict: + return {"type": kind, "timestamp": stamp, "message": {"role": kind, "content": content}, **extra} + + +def tool_use(tool_id: str, command: str) -> list[dict]: + return [{"type": "tool_use", "id": tool_id, "name": "Bash", "input": {"command": command}}] + + +def tool_result(tool_id: str, text: str, error: bool) -> list[dict]: + return [{"type": "tool_result", "tool_use_id": tool_id, "is_error": error, "content": text}] + + +TRANSCRIPT = [ + line("user", "2026-01-05T09:00:00.000Z", "an earlier conversation the run must ignore"), + line("user", "2026-01-05T10:00:00.000Z", "/dev:build"), + line("user", "2026-01-05T10:00:01.000Z", "Base directory for this skill: /x", isMeta=True), + line("assistant", "2026-01-05T10:00:02.000Z", tool_use("t1", "python3 lint-spec.py .dev/x/spec.md")), + line("user", "2026-01-05T10:00:03.000Z", tool_result("t1", "spec.md:4: missing tests line", True)), + line("assistant", "2026-01-05T10:00:04.000Z", tool_use("t2", "python3 lint-spec.py .dev/x/spec.md")), + line("user", "2026-01-05T10:00:05.000Z", tool_result("t2", "clean", False)), + line("assistant", "2026-01-05T10:00:06.000Z", tool_use("t3", "rm -rf build")), + line("user", "2026-01-05T10:00:07.000Z", + tool_result("t3", "The user doesn't want to proceed with this tool use.", True)), + line("assistant", "2026-01-05T10:00:08.000Z", tool_use("t4", "npm test")), + line("user", "2026-01-05T10:00:09.000Z", tool_result("t4", "Exit code 1 ENOENT package.json", True)), + line("user", "2026-01-05T10:00:10.000Z", [{"type": "text", "text": "[Request interrupted by user]"}]), + line("user", "2026-01-05T10:00:11.000Z", "stop rerunning the whole validation block after every change set"), + line("assistant", "2026-01-05T10:00:12.000Z", tool_use("t5", "grep -n lint-spec.py README.md")), + line("user", "2026-01-05T10:00:13.000Z", tool_result("t5", "", False)), +] + + +class Sandbox: + """A scratch repo with a transcript where skill-metrics.py will find it.""" + + def __init__(self, tmp: Path, rows: list[dict] = TRANSCRIPT): + self.repo = tmp / f"repo-{os.getpid()}-{id(self)}" + self.repo.mkdir() + subprocess.run(["git", "init", "-q"], cwd=self.repo, check=True) + subprocess.run(["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", + "--allow-empty", "-m", "init"], cwd=self.repo, check=True) + self.config = tmp / "claude" + slug = re.sub(r"[^A-Za-z0-9]", "-", str(self.repo)) + self.transcript = self.config / "projects" / slug / "session.jsonl" + self.transcript.parent.mkdir(parents=True) + self.transcript.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + self.memory = tmp / "memory" + self.env = {**os.environ, "CLAUDE_CONFIG_DIR": str(self.config), "DEV_MEMORY_DIR": str(self.memory)} + + def metrics(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(METRICS), *args], cwd=self.repo, env=self.env, + capture_output=True, text=True) + + def claims(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(CLAIMS), *args], cwd=self.repo, env=self.env, + capture_output=True, text=True) + + def journal(self) -> list[dict]: + return [json.loads(l) for p in sorted((self.memory / "journal").glob("*.jsonl")) + for l in p.read_text(encoding="utf-8").splitlines()] + + def run(self, *friction: str) -> dict: + self.metrics("start", "build") + args = ["end", "build", "--count", "change_sets=2"] + for text in friction: + args += ["--friction", text] + result = self.metrics(*args) + assert result.returncode == 0, result.stderr + return self.journal()[-1] + + def candidates(self, rows: list[dict]) -> Path: + path = self.repo / "candidates.jsonl" + path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + return path + + +class JournalTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.box = Sandbox(Path(self.tmp.name)) + + def tearDown(self): + self.tmp.cleanup() + + def test_end_journals_measured_signals_with_their_lines(self): + entry = self.box.run("the brief left out the Validation block") + self.assertEqual(entry["skill"], "build") + self.assertEqual(entry["id"], "r-20260105T100000-build") + self.assertEqual(entry["counts"], {"change_sets": "2"}) + self.assertEqual(entry["checkers"]["lint-spec.py"]["runs"], 2) + self.assertEqual(entry["checkers"]["lint-spec.py"]["failed"], 1) + self.assertEqual(entry["checkers"]["lint-spec.py"]["first_failure"]["line"], 4) + kinds = {s["type"]: s for s in entry["signals"]} + self.assertEqual(kinds["denied"]["line"], 8) + self.assertEqual(kinds["tool_error"]["line"], 10) + self.assertEqual(kinds["interrupt"]["line"], 11) + self.assertEqual(kinds["user_turn"]["line"], 12) + self.assertEqual(entry["signal_counts"], + {"denied": 1, "tool_error": 1, "interrupt": 1, "user_turn": 1}) + self.assertEqual(entry["friction"], ["the brief left out the Validation block"]) + self.assertEqual(entry["ledger_line"], 1) + self.assertNotIn("earlier conversation", json.dumps(entry)) + + def test_metrics_table_reports_signals_and_journal(self): + self.box.metrics("start", "build") + out = self.box.metrics("end", "build").stdout + self.assertIn("lint-spec.py 2 runs / 1 failed", out) + self.assertIn("journal:", out) + row = json.loads((self.box.repo / ".dev" / "metrics.jsonl").read_text().splitlines()[-1]) + self.assertEqual(row["signal_counts"]["denied"], 1) + + def test_more_than_three_friction_lines_is_refused_before_writing(self): + self.box.metrics("start", "build") + result = self.box.metrics("end", "build", *sum((["--friction", f"f{i}"] for i in range(4)), [])) + self.assertNotEqual(result.returncode, 0) + self.assertIn("at most 3", result.stderr) + self.assertFalse((self.box.repo / ".dev" / "metrics.jsonl").exists()) + + def test_journal_can_be_turned_off(self): + (self.box.repo / ".dev").mkdir() + (self.box.repo / ".dev" / "config.json").write_text('{"memory": {"enabled": false}}') + self.box.metrics("start", "build") + result = self.box.metrics("end", "build") + self.assertEqual(result.returncode, 0) + self.assertNotIn("journal:", result.stdout) + self.assertFalse(self.box.memory.exists()) + + +class ClaimsTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.box = Sandbox(Path(self.tmp.name)) + self.entry = self.box.run("the brief left out the Validation block") + self.run_id = self.entry["id"] + + def tearDown(self): + self.tmp.cleanup() + + def claim(self, **over) -> dict: + row = {"skill": "build", "thread": "build-validation-reruns", "kind": "friction", + "claim": "The user had to stop build from rerunning the full Validation block.", + "quote": "stop rerunning the whole validation block", + "source": {"run": self.run_id, "line": 12}, "salience": 2} + row.update(over) + return row + + def add(self, *rows: dict) -> subprocess.CompletedProcess: + return self.box.claims("add", str(self.box.candidates(list(rows)))) + + def stored(self) -> list[dict]: + return [json.loads(l) for l in (self.box.memory / "claims.jsonl").read_text().splitlines()] + + def test_pending_then_mark(self): + pending = json.loads(self.box.claims("pending").stdout) + self.assertEqual([r["id"] for r in pending], [self.run_id]) + self.assertEqual(self.box.claims("mark", self.run_id).returncode, 0) + self.assertEqual(json.loads(self.box.claims("pending").stdout), []) + self.assertNotEqual(self.box.claims("mark", "r-nope").returncode, 0) + + def test_quote_found_at_cited_line_is_added_and_derived(self): + result = self.add(self.claim()) + self.assertEqual(result.returncode, 0, result.stdout) + [row] = self.stored() + self.assertEqual(row["id"], "c-0001") + self.assertEqual(row["observed_at"], "2026-01-05T10:00:00.000Z") + self.assertIn("[c-0001]", (self.box.memory / "skills" / "build.md").read_text()) + self.assertIn("build-validation-reruns", (self.box.memory / "threads.yaml").read_text()) + self.assertEqual(self.box.claims("check").returncode, 0) + + def test_journal_quote_without_line_is_checked_against_the_entry(self): + self.assertEqual(self.add(self.claim(quote="the brief left out the Validation block", + source={"run": self.run_id})).returncode, 0) + + def test_batch_is_all_or_nothing(self): + bad = self.claim(quote="words the user never typed", thread="other-thread") + result = self.add(self.claim(), bad) + self.assertEqual(result.returncode, 1) + self.assertIn("candidate 2: quote not found verbatim", result.stdout) + self.assertFalse((self.box.memory / "claims.jsonl").exists()) + + def test_shape_and_source_errors(self): + cases = { + "source.run 'r-missing' is not in the journal": self.claim(source={"run": "r-missing", "line": 12}), + "kind must be one of": self.claim(kind="vibe"), + "credential shape": self.claim(claim="Leaked token=abcdefghijklmnop in the run log."), + "has no readable line": self.claim(source={"run": self.run_id, "line": 999}), + "subagents": self.claim(source={"run": self.run_id, "line": 1, "transcript": "/etc/passwd"}), + "em dash": self.claim(claim="Build reran the block \u2014 twice in one wave."), + } + for message, row in cases.items(): + with self.subTest(message): + result = self.add(row) + self.assertEqual(result.returncode, 1) + self.assertIn(message, result.stdout) + + def test_supersede_and_duplicate(self): + self.add(self.claim()) + self.assertIn("duplicates c-0001", self.add(self.claim()).stdout) + newer = self.claim(quote="the brief left out the Validation block", source={"run": self.run_id}, + supersedes="c-0001") + self.assertEqual(self.add(newer).returncode, 0) + self.assertEqual([r["status"] for r in self.stored()], ["superseded", "active"]) + + def test_retract_forgets_and_blocks_readding(self): + self.add(self.claim()) + self.assertEqual(self.box.claims("retract", "c-0001", "--reason", "an interview answer").returncode, 0) + self.assertFalse((self.box.memory / "skills" / "build.md").exists()) + self.assertNotIn("c-0001", (self.box.memory / "threads.yaml").read_text()) + self.assertIn("was retracted as c-0001", self.add(self.claim()).stdout) + + def test_resolution_reopens_on_a_later_claim(self): + self.add(self.claim()) + self.assertEqual(self.box.claims("resolve", "build-validation-reruns", "--commit", "abc1234", + "--eval", "build-parallel-wave").returncode, 0) + self.assertIn("| build-validation-reruns |", (self.box.memory / "progression.md").read_text()) + self.assertIn("no open threads", self.box.claims("top").stdout) + later = [dict(r, timestamp=r["timestamp"].replace("2026-01-05", "2030-01-01")) for r in TRANSCRIPT] + self.box.transcript.write_text("".join(json.dumps(r) + "\n" for r in TRANSCRIPT + later)) + entry = self.box.run() + self.assertEqual(self.add(self.claim(source={"run": entry["id"], "line": len(TRANSCRIPT) + 12})).returncode, 0) + top = self.box.claims("top").stdout + self.assertIn("| build-validation-reruns | build | reopened |", top) + + def test_check_flags_hand_edited_derived_files(self): + self.add(self.claim()) + (self.box.memory / "skills" / "build.md").write_text("edited by hand\n") + result = self.box.claims("check") + self.assertEqual(result.returncode, 1) + self.assertIn("skills/build.md: stale", result.stdout) + + def test_show_and_report(self): + self.add(self.claim()) + shown = self.box.claims("show", "c-0001").stdout + self.assertIn('quote: "stop rerunning the whole validation block"', shown) + self.assertIn("context (session.jsonl:12)", shown) + out = self.box.repo / "report.html" + self.assertEqual(self.box.claims("report", str(out)).returncode, 0) + self.assertIn("build-validation-reruns", out.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/dev/references/run-journal.md b/dev/references/run-journal.md new file mode 100644 index 0000000..0eee3dd --- /dev/null +++ b/dev/references/run-journal.md @@ -0,0 +1,12 @@ +# Run Journal + +`skill-metrics.py end` appends one entry per run to the cross-repository journal under `~/.dev-workflow/memory/journal/` (or `$DEV_MEMORY_DIR`), which the `reflect` skill consolidates into cited claims about the skills themselves. +The script measures the friction signals on its own from the transcript: interrupts, denied and failed tool calls, the user's own turns, and how often each checker ran and failed. + +The one thing it cannot measure is where the run fought the skill's own instructions, so the skill adds that, as up to three `--friction "..."` arguments on the `end` call: + +- One line each: the step, what the instruction asked, and what the run had to do instead, quoting the moment when there is one. +- About the skill's text, never the repository's code: "step 3 assumes one Validation block; this repo has two, so the wave gate ran neither" is friction, "the tests are slow" is not. +- None when the run went as the skill describes; an empty journal entry is the common and good case. + +Set `DEV_MEMORY_DIR=off`, or `"memory": {"enabled": false}` in `.dev/config.json`, to keep a repository's runs out of the journal. diff --git a/dev/scripts/claims.py b/dev/scripts/claims.py new file mode 100644 index 0000000..bed1904 --- /dev/null +++ b/dev/scripts/claims.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""The dev workflow's memory of its own runs: cited claims about the skills. + +Usage: + python3 claims.py pending journal runs not yet consolidated, as JSON + python3 claims.py add CANDIDATES.jsonl validate and append claims, all or nothing + python3 claims.py mark RUN_ID ... record runs as consolidated + python3 claims.py rebuild regenerate every derived file from the records + python3 claims.py check validate the records and that derived files are current + python3 claims.py top [--limit N] [--all] rank threads, open and reopened first + python3 claims.py show TARGET a thread, skill, or claim with its evidence + python3 claims.py retract CLAIM_ID --reason R forget a claim and rebuild without it + python3 claims.py resolve THREAD --commit SHA [--eval NAME] + python3 claims.py report OUT.html render the threads as a self-contained page + +The store lives in $DEV_MEMORY_DIR (default ~/.dev-workflow/memory). The records +are journal/*.jsonl (written by skill-metrics.py end), claims.jsonl, +resolutions.jsonl, and consolidated.jsonl; skills/*.md, threads.yaml, and +progression.md are derived from them by `rebuild` and never edited by hand. + +A claim is accepted only when its quote is found verbatim (whitespace aside) at +its cited source: a transcript line of the run it names, or that run's journal +entry. A retracted claim stays in claims.jsonl so the same evidence cannot +bring it back. A thread resolved by a commit reopens when a claim observed +after the resolution joins it. Nothing here edits a skill. +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +KINDS = ("friction", "defect", "cost", "success") +SLUG = re.compile(r"^[a-z][a-z0-9-]{1,60}$") +CLAIM_ID = re.compile(r"^c-\d{4,}$") +FIELDS = {"skill", "thread", "kind", "claim", "quote", "source", "salience", "supersedes"} +CREDENTIAL = re.compile( + r"AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{30,}|github_pat_\w{20,}|sk-[A-Za-z0-9_-]{20,}" + r"|xox[baprs]-[A-Za-z0-9-]{10,}|-----BEGIN [A-Z ]*PRIVATE KEY" + r"|(?i:(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^\s'\"]{8,})" +) +EM_DASH = "\u2014" +DERIVED_NOTE = "Derived by claims.py rebuild from the records; never edit by hand." + + +# --- store ------------------------------------------------------------------- + +def root() -> Path: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + sys.exit("claims: the memory store is turned off (DEV_MEMORY_DIR=off)") + return Path(value).expanduser() + + +def now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def read_jsonl(path: Path) -> list[dict]: + if not path.exists(): + return [] + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + tmp.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + tmp.replace(path) + + +def append_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") + + +def runs(store: Path) -> dict[str, dict]: + found: dict[str, dict] = {} + for path in sorted((store / "journal").glob("*.jsonl")): + for row in read_jsonl(path): + if row.get("id"): + found[row["id"]] = row + return found + + +def claims(store: Path) -> list[dict]: + return read_jsonl(store / "claims.jsonl") + + +def norm(text: str) -> str: + return " ".join(str(text).split()) + + +def strings(value) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [s for item in value.values() for s in strings(item)] + if isinstance(value, list): + return [s for item in value for s in strings(item)] + return [] + + +def transcript_line(path: Path, line: int) -> dict | None: + try: + with path.open(encoding="utf-8") as handle: + for index, raw in enumerate(handle): + if index == line: + return json.loads(raw) + except (OSError, json.JSONDecodeError): + return None + return None + + +# --- validation -------------------------------------------------------------- + +def source_text(source: dict, run: dict) -> tuple[str | None, str]: + """The text a claim's quote must appear in, and a label for messages.""" + if "line" not in source: + return "\n".join(strings(run)), f"journal entry {run['id']}" + if not isinstance(source["line"], int) or source["line"] < 0: + return None, "source.line must be a non-negative integer" + main = Path(run.get("transcript") or "") + target = Path(source.get("transcript") or main) + allowed = target == main or main.with_suffix("") in target.parents + if not run.get("transcript") or not allowed: + return None, f"source.transcript must be the run's transcript or one of its subagents ({main})" + obj = transcript_line(target, source["line"]) + if obj is None: + return None, f"{target} has no readable line {source['line']}" + return "\n".join(strings(obj.get("message", obj))), f"{target.name}:{source['line']}" + + +def evidence_key(row: dict) -> tuple: + source = row.get("source") or {} + return (source.get("run"), source.get("transcript"), source.get("line"), norm(row.get("quote", "")).lower()) + + +def shape_errors(row: dict) -> list[str]: + errors = [] + extra = set(row) - FIELDS - {"id", "status", "observed_at", "repo", "added_at", "reason"} + if extra: + errors.append(f"unknown fields {sorted(extra)}") + for key in ("skill", "thread"): + if not SLUG.match(str(row.get(key, ""))): + errors.append(f"{key} must be a lowercase slug, got {row.get(key)!r}") + if row.get("kind") not in KINDS: + errors.append(f"kind must be one of {', '.join(KINDS)}") + claim = str(row.get("claim", "")) + if not 12 <= len(claim) <= 240 or "\n" in claim: + errors.append("claim must be one line of 12 to 240 characters") + if EM_DASH in claim: + errors.append("claim contains an em dash; use two plain dashes") + quote = norm(row.get("quote", "")) + if not 8 <= len(quote) <= 300: + errors.append("quote must be 8 to 300 characters of verbatim evidence") + if row.get("salience") not in (1, 2, 3): + errors.append("salience must be 1, 2, or 3") + if not isinstance(row.get("source"), dict) or not row["source"].get("run"): + errors.append("source must be an object naming its run") + if CREDENTIAL.search(claim) or CREDENTIAL.search(quote): + errors.append("claim or quote matches a credential shape; quote around the secret") + return errors + + +def validate_candidates(rows: list[dict], store: Path) -> list[str]: + known = runs(store) + existing = claims(store) + by_id = {c["id"]: c for c in existing} + seen = {evidence_key(c): c for c in existing} + errors: list[str] = [] + batch: set[tuple] = set() + for number, row in enumerate(rows, 1): + problems = shape_errors(row) + source = row.get("source") if isinstance(row.get("source"), dict) else {} + run = known.get(source.get("run", "")) + if source.get("run") and run is None: + problems.append(f"source.run {source['run']!r} is not in the journal") + if run is not None and not problems: + text, label = source_text(source, run) + if text is None: + problems.append(label) + elif norm(row["quote"]).lower() not in norm(text).lower(): + problems.append(f"quote not found verbatim in {label}") + key = evidence_key(row) + if key in seen: + prior = seen[key] + verb = "was retracted as" if prior.get("status") == "retracted" else "duplicates" + problems.append(f"this evidence {verb} {prior['id']}") + if key in batch: + problems.append("duplicates an earlier candidate in this file") + batch.add(key) + target = row.get("supersedes") + if target is not None and (target not in by_id or by_id[target].get("status") != "active"): + problems.append(f"supersedes {target!r}, which is not an active claim") + errors += [f"candidate {number}: {p}" for p in problems] + return errors + + +# --- derived state ----------------------------------------------------------- + +def threads(store: Path) -> list[dict]: + resolutions: dict[str, dict] = {} + for row in read_jsonl(store / "resolutions.jsonl"): + resolutions[row["thread"]] = row + grouped: dict[str, list[dict]] = {} + for claim in claims(store): + if claim.get("status") == "active": + grouped.setdefault(claim["thread"], []).append(claim) + for thread in resolutions: + grouped.setdefault(thread, []) + out = [] + for name, rows in grouped.items(): + resolved = resolutions.get(name) + problems = [c for c in rows if c["kind"] != "success"] + since = [c for c in problems if resolved and c["observed_at"][:19] > resolved["at"][:19]] + if not resolved: + status = "open" if problems else "working" + else: + status = "reopened" if since else "resolved" + live = since if resolved else problems + out.append({ + "thread": name, + "skills": sorted({c["skill"] for c in rows}) or [resolved.get("skill", "?")], + "status": status, + "score": sum(c["salience"] for c in live), + "runs": len({c["source"]["run"] for c in live}), + "repos": len({c.get("repo") for c in live}), + "last_seen": max((c["observed_at"] for c in rows), default=""), + "claims": [c["id"] for c in rows], + "resolution": resolved, + "claims_before": len(problems) - len(since), + "claims_since": len(since), + }) + order = {"reopened": 0, "open": 1, "working": 2, "resolved": 3} + out.sort(key=lambda t: (order[t["status"]], -t["runs"], -t["score"], t["thread"])) + return out + + +def yaml_scalar(value) -> str: + text = str(value) + return text if re.match(r"^[A-Za-z0-9_./:@+-]+$", text) else json.dumps(text) + + +def render_threads(rows: list[dict]) -> str: + lines = [f"# {DERIVED_NOTE}"] + for t in rows: + lines.append(f"- thread: {t['thread']}") + for key in ("status", "score", "runs", "repos", "last_seen"): + lines.append(f" {key}: {yaml_scalar(t[key])}") + lines.append(f" skills: [{', '.join(t['skills'])}]") + lines.append(f" claims: [{', '.join(t['claims'])}]") + if t["resolution"]: + r = t["resolution"] + lines.append(f" resolved: {{commit: {yaml_scalar(r['commit'])}, at: {yaml_scalar(r['at'])}" + + (f", eval: {yaml_scalar(r['eval'])}" if r.get("eval") else "") + "}") + return "\n".join(lines) + "\n" + + +def render_skill(skill: str, rows: list[dict], by_id: dict[str, dict]) -> str: + out = [f"# {skill}", "", DERIVED_NOTE, ""] + sections = (("Open threads", ("reopened", "open")), ("What works", ("working",)), ("Resolved threads", ("resolved",))) + for title, statuses in sections: + picked = [t for t in rows if t["status"] in statuses and skill in t["skills"]] + if not picked: + continue + out += [f"## {title}", ""] + for t in picked: + out.append(f"### {t['thread']} ({t['status']}, score {t['score']}, {t['runs']} runs, {t['repos']} repos)") + out.append("") + for cid in t["claims"]: + c = by_id[cid] + if c["skill"] == skill: + out.append(f"- [{cid}] {c['claim']} ({c['kind']}, salience {c['salience']}, " + f"{c.get('repo', '?')}, {c['observed_at'][:10]})") + out.append("") + return "\n".join(out).rstrip() + "\n" + + +def render_progression(rows: list[dict]) -> str: + out = ["# Progression", "", DERIVED_NOTE, "", + "| thread | resolved at | commit | eval | claims before | claims since | status |", + "| ------ | ----------- | ------ | ---- | ------------- | ------------ | ------ |"] + for t in sorted((t for t in rows if t["resolution"]), key=lambda t: t["resolution"]["at"]): + r = t["resolution"] + out.append(f"| {t['thread']} | {r['at'][:10]} | {r['commit'][:12]} | {r.get('eval') or '-'} | " + f"{t['claims_before']} | {t['claims_since']} | {t['status']} |") + return "\n".join(out) + "\n" + + +def derived(store: Path) -> dict[str, str]: + rows = threads(store) + by_id = {c["id"]: c for c in claims(store)} + files = {"threads.yaml": render_threads(rows), "progression.md": render_progression(rows)} + for skill in sorted({s for t in rows for s in t["skills"] if t["claims"]}): + files[f"skills/{skill}.md"] = render_skill(skill, rows, by_id) + return files + + +def rebuild(store: Path) -> None: + skills_dir = store / "skills" + if skills_dir.is_dir(): + for stale in skills_dir.glob("*.md"): + stale.unlink() + for name, text in derived(store).items(): + path = store / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --- commands ---------------------------------------------------------------- + +def cmd_pending(store: Path) -> int: + done = {row["run"] for row in read_jsonl(store / "consolidated.jsonl")} + todo = [run for rid, run in sorted(runs(store).items()) if rid not in done] + print(json.dumps(todo, indent=1)) + print(f"claims: {len(todo)} pending runs", file=sys.stderr) + return 0 + + +def cmd_add(store: Path, candidates: Path) -> int: + rows = [] + for number, line in enumerate(candidates.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as exc: + print(f"{candidates}:{number}: not JSON ({exc.msg})") + return 1 + errors = validate_candidates(rows, store) + if errors: + print("\n".join(errors)) + print(f"claims: {len(errors)} problems, nothing added; fix the candidates, not the evidence") + return 1 + existing = claims(store) + known = runs(store) + next_id = max((int(c["id"][2:]) for c in existing), default=0) + 1 + superseded = {row["supersedes"] for row in rows if row.get("supersedes")} + for claim in existing: + if claim["id"] in superseded: + claim["status"] = "superseded" + added = [] + for row in rows: + run = known[row["source"]["run"]] + row.update({"id": f"c-{next_id:04d}", "status": "active", "observed_at": run["started_at"], + "repo": run.get("repo"), "added_at": now_iso(), "quote": norm(row["quote"])}) + next_id += 1 + added.append(row) + write_jsonl(store / "claims.jsonl", existing + added) + rebuild(store) + print(f"claims: added {', '.join(r['id'] for r in added) or 'nothing'}" + + (f"; superseded {', '.join(sorted(superseded))}" if superseded else "")) + return 0 + + +def cmd_mark(store: Path, ids: list[str]) -> int: + known = runs(store) + missing = [i for i in ids if i not in known] + if missing: + print(f"claims: not in the journal: {', '.join(missing)}") + return 1 + done = {row["run"] for row in read_jsonl(store / "consolidated.jsonl")} + append_jsonl(store / "consolidated.jsonl", [{"run": i, "at": now_iso()} for i in ids if i not in done]) + print(f"claims: {len(ids)} runs marked consolidated") + return 0 + + +def cmd_check(store: Path) -> int: + rows = claims(store) + errors = [] + ids = [r.get("id") for r in rows] + if len(ids) != len(set(ids)): + errors.append("claims.jsonl: duplicate claim ids") + for row in rows: + label = row.get("id", "?") + if not CLAIM_ID.match(str(label)): + errors.append(f"{label}: malformed id") + if row.get("status") not in ("active", "superseded", "retracted"): + errors.append(f"{label}: unknown status {row.get('status')!r}") + errors += [f"{label}: {p}" for p in shape_errors(row)] + if row.get("supersedes") and row["supersedes"] not in ids: + errors.append(f"{label}: supersedes unknown {row['supersedes']}") + for name, text in derived(store).items(): + path = store / name + if not path.exists() or path.read_text(encoding="utf-8") != text: + errors.append(f"{name}: stale; run `claims.py rebuild`") + print("\n".join(errors) if errors else f"claims: {len(rows)} claims, derived files current") + return 1 if errors else 0 + + +def cmd_top(store: Path, limit: int, everything: bool) -> int: + rows = [t for t in threads(store) if everything or t["status"] in ("open", "reopened")] + if not rows: + print("claims: no open threads") + return 0 + print("| thread | skills | status | score | runs | repos | last seen |") + print("| ------ | ------ | ------ | ----- | ---- | ----- | --------- |") + for t in rows[:limit]: + print(f"| {t['thread']} | {', '.join(t['skills'])} | {t['status']} | {t['score']} | " + f"{t['runs']} | {t['repos']} | {t['last_seen'][:10]} |") + return 0 + + +def cmd_show(store: Path, target: str) -> int: + rows = claims(store) + picked = [c for c in rows if target in (c["id"], c["thread"], c["skill"])] + if not picked: + print(f"claims: nothing matches {target!r}") + return 1 + known = runs(store) + for c in picked: + source = c["source"] + print(f"## {c['id']} [{c['status']}] {c['thread']} ({c['skill']}, {c['kind']}, salience {c['salience']})") + print(f"claim: {c['claim']}") + print(f"quote: \"{c['quote']}\"") + run = known.get(source["run"]) + where = f"{source['run']}" + (f" line {source['line']}" if "line" in source else " journal entry") + print(f"source: {where} ({c.get('repo', '?')}, {c['observed_at']})") + if run is not None and "line" in source: + text, label = source_text(source, run) + if text is not None: + flat = norm(text) + at = flat.lower().find(c["quote"].lower()) + start = max(0, at - 200) if at >= 0 else 0 + print(f"context ({label}): ...{flat[start:start + len(c['quote']) + 400]}...") + else: + print(f"context: unavailable ({label})") + if c.get("reason"): + print(f"retracted because: {c['reason']}") + print() + return 0 + + +def cmd_retract(store: Path, claim_id: str, reason: str) -> int: + rows = claims(store) + hit = next((c for c in rows if c["id"] == claim_id), None) + if hit is None or hit["status"] == "retracted": + print(f"claims: {claim_id} is not a claim that can be retracted") + return 1 + hit["status"] = "retracted" + hit["reason"] = " ".join(reason.split()) + write_jsonl(store / "claims.jsonl", rows) + rebuild(store) + print(f"claims: retracted {claim_id}; derived files rebuilt without it") + return 0 + + +def cmd_resolve(store: Path, thread: str, commit: str, eval_name: str | None) -> int: + if not any(c["thread"] == thread and c["status"] == "active" for c in claims(store)): + print(f"claims: no active claims in thread {thread!r}") + return 1 + if not re.match(r"^[0-9a-f]{7,40}$", commit): + print("claims: --commit must be a commit sha") + return 1 + skill = next(c["skill"] for c in claims(store) if c["thread"] == thread) + append_jsonl(store / "resolutions.jsonl", + [{"thread": thread, "skill": skill, "commit": commit, "eval": eval_name, "at": now_iso()}]) + rebuild(store) + print(f"claims: {thread} resolved by {commit}; a later claim in it reopens it") + return 0 + + +def cmd_report(store: Path, out: Path) -> int: + rows = threads(store) + by_id = {c["id"]: c for c in claims(store)} + cards = [] + for t in rows: + items = "".join( + f"
  • {html.escape(cid)} {html.escape(by_id[cid]['claim'])}" + f"
    {html.escape(by_id[cid]['quote'])}
    " + f"{html.escape(by_id[cid]['kind'])} / salience {by_id[cid]['salience']} / " + f"{html.escape(str(by_id[cid].get('repo')))} / {html.escape(by_id[cid]['source']['run'])}
    • " + for cid in t["claims"]) + resolved = (f"

    resolved by {html.escape(t['resolution']['commit'][:12])}

    " + if t["resolution"] else "") + cards.append( + f"" + f"{t['status']} {html.escape(t['thread'])} " + f"{html.escape(', '.join(t['skills']))} / score {t['score']} / {t['runs']} runs / " + f"{t['repos']} repos{resolved}
      {items}
    ") + page = f""" +Workflow Memory + +

    Workflow memory

    {len(rows)} threads from {len(by_id)} claims, rendered {now_iso()}. +Every claim quotes the run it came from.

    {''.join(cards) or '

    No claims yet.

    '} +""" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(page, encoding="utf-8") + print(f"claims: report written to {out}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("pending") + sub.add_parser("add").add_argument("candidates", type=Path) + sub.add_parser("mark").add_argument("runs", nargs="+") + sub.add_parser("rebuild") + sub.add_parser("check") + top = sub.add_parser("top") + top.add_argument("--limit", type=int, default=10) + top.add_argument("--all", action="store_true") + sub.add_parser("show").add_argument("target") + retract = sub.add_parser("retract") + retract.add_argument("claim") + retract.add_argument("--reason", required=True) + resolve = sub.add_parser("resolve") + resolve.add_argument("thread") + resolve.add_argument("--commit", required=True) + resolve.add_argument("--eval", dest="eval_name") + sub.add_parser("report").add_argument("out", type=Path) + args = parser.parse_args() + store = root() + if args.command == "pending": + return cmd_pending(store) + if args.command == "add": + return cmd_add(store, args.candidates) + if args.command == "mark": + return cmd_mark(store, args.runs) + if args.command == "rebuild": + rebuild(store) + print(f"claims: derived files rebuilt under {store}") + return 0 + if args.command == "check": + return cmd_check(store) + if args.command == "top": + return cmd_top(store, args.limit, args.all) + if args.command == "show": + return cmd_show(store, args.target) + if args.command == "retract": + return cmd_retract(store, args.claim, args.reason) + if args.command == "resolve": + return cmd_resolve(store, args.thread, args.commit, args.eval_name) + return cmd_report(store, args.out) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/scripts/skill-metrics.py b/dev/scripts/skill-metrics.py index ac78ef0..2301ce3 100755 --- a/dev/scripts/skill-metrics.py +++ b/dev/scripts/skill-metrics.py @@ -3,7 +3,7 @@ Usage: python3 skill-metrics.py start {skill} - python3 skill-metrics.py end {skill} [--count key=value ...] + python3 skill-metrics.py end {skill} [--count key=value ...] [--friction text ...] `start` snapshots the git state and locates the session transcript under $CLAUDE_CONFIG_DIR (default ~/.claude), anchoring on the transcript line that @@ -13,6 +13,16 @@ so later runs can be compared against earlier ones. Every value is measured; nothing here is narrated from memory. Missing transcript or git degrade to "n/a" rather than failing the run. + +`end` also appends one entry to the cross-repository run journal under +$DEV_MEMORY_DIR (default ~/.dev-workflow/memory/journal/), which the reflect +skill consolidates into cited claims about the skills themselves. The entry +holds the friction signals measured from the transcript (interrupts, denied or +failed tool calls, the user's own turns, and how often each deterministic +checker ran and failed), each with its transcript path and line, plus at most +three `--friction` lines the skill narrates, kept apart as narrated. Set +DEV_MEMORY_DIR=off, or `"memory": {"enabled": false}` in .dev/config.json, to +skip the journal. """ from __future__ import annotations @@ -31,6 +41,18 @@ TEST_FILE = re.compile(r"(^|/)(tests?|specs?|__tests__)(/|$)|[._-](test|spec)s?\.[A-Za-z]+$|^test_.*\.py$") TOKEN_KEYS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens") +# The deterministic checkers the skills loop against; a failed run of one is loop +# progress, so it is counted per checker instead of logged as a tool error. +CHECKERS = ("lint-spec.py", "check-tests.py", "pr-evidence.py", "architecture-check.py", + "check-agents-md.py", "aggregate-findings.py", "claims.py") +# A checker counts only where a command runs it, never where one greps or edits it. +CHECKER_RUN = re.compile(r"(?:^|[;&|(]\s*|\bpython3?\s+(?:-\S+\s+)*)\S*?(" + "|".join(re.escape(c) for c in CHECKERS) + r")(?=\s|$)", + re.M) +INTERRUPT = "[Request interrupted by user" +DENIED = ("doesn't want to proceed", "was rejected", "Permission denied by user") +SIGNAL_CAP = 20 +EXCERPT = 240 +MAX_FRICTION = 3 def now_iso() -> str: @@ -130,6 +152,145 @@ def subagent_files(transcript: Path) -> list[Path]: return sorted(folder.glob("*.jsonl")) if folder.is_dir() else [] +# --- friction signals -------------------------------------------------------- + +def block_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(block_text(item.get("text", item.get("content", ""))) if isinstance(item, dict) + else str(item) for item in content) + return "" + + +def human_text(obj: dict) -> str | None: + """The text a person typed, or None for tool results and injected turns.""" + if obj.get("isMeta") or obj.get("isSidechain"): + return None + content = (obj.get("message") or {}).get("content") + if isinstance(content, list): + if any(isinstance(b, dict) and b.get("type") == "tool_result" for b in content): + return None + content = block_text([b for b in content if isinstance(b, dict) and b.get("type") == "text"]) + if not isinstance(content, str): + return None + text = content.strip() + if not text or text.startswith("<") or INTERRUPT in text or text.startswith("Base directory for this skill"): + return None + return text + + +def signals(path: Path, start: int, since: float | None, state: dict) -> None: + """Add one transcript's friction signals and checker runs to `state`.""" + pending: dict[str, str] = {} + first = True + for index, obj in iter_lines(path, start): + stamp = obj.get("timestamp") + if first and stamp: + first = False + if since is not None and parse_iso(stamp) < since: + return + message = obj.get("message") or {} + if obj.get("type") == "assistant": + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + command = str((block.get("input") or {}).get("command", "")) + match = CHECKER_RUN.search(command) + checker = match.group(1) if match else None + if checker: + pending[block.get("id", "")] = checker + state["checkers"].setdefault(checker, {"runs": 0, "failed": 0})["runs"] += 1 + continue + if obj.get("type") != "user": + continue + where = {"transcript": str(path), "line": index} + content = message.get("content") + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + text = block_text(block.get("content", block.get("text", ""))) + if block.get("type") == "text" and INTERRUPT in text: + note(state, "interrupt", where, text) + elif block.get("type") == "tool_result" and block.get("is_error"): + checker = pending.get(block.get("tool_use_id", "")) + if checker: + entry = state["checkers"][checker] + entry["failed"] += 1 + entry.setdefault("first_failure", where) + else: + note(state, "denied" if any(d in text for d in DENIED) else "tool_error", where, text) + if isinstance(content, str) and INTERRUPT in content: + note(state, "interrupt", where, content) + elif path == state["main"]: + typed = human_text(obj) + if typed: + note(state, "user_turn", where, typed) + + +def note(state: dict, kind: str, where: dict, text: str) -> None: + state["counts"][kind] = state["counts"].get(kind, 0) + 1 + if state["counts"][kind] <= SIGNAL_CAP: + state["signals"].append({"type": kind, **where, "excerpt": " ".join(text.split())[:EXCERPT]}) + + +def collect_signals(transcript: Path | None, anchor: int, since: float) -> dict: + state: dict = {"main": transcript, "counts": {}, "signals": [], "checkers": {}} + if transcript and transcript.exists(): + # Skip the invocation line itself: it is the skill's own prompt, not a reaction to it. + signals(transcript, anchor + 1, None, state) + for path in subagent_files(transcript): + signals(path, 0, since, state) + state.pop("main") + return state + + +# --- run journal ------------------------------------------------------------- + +def memory_dir() -> Path | None: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + return None + config = repo_root() / ".dev" / "config.json" + try: + if json.loads(config.read_text(encoding="utf-8")).get("memory", {}).get("enabled") is False: + return None + except (OSError, ValueError, AttributeError): + pass + return Path(value).expanduser() + + +def write_journal(record: dict, ledger: Path, ledger_line: int, state: dict, + friction: list[str], found: dict) -> tuple[Path, str] | None: + root = memory_dir() + if root is None: + return None + stamp = re.sub(r"[^0-9T]", "", record["started_at"])[:15] + run_id = f"r-{stamp}-{record['skill']}" + entry = { + "id": run_id, + "skill": record["skill"], + "repo": repo_root().name, + "repo_path": str(repo_root()), + "started_at": record["started_at"], + "ended_at": record["ended_at"], + "seconds": record["seconds"], + "transcript": state.get("transcript"), + "anchor_line": state.get("anchor_line"), + "ledger": str(ledger), + "ledger_line": ledger_line, + "counts": record["counts"], + "signal_counts": found["counts"], + "checkers": found["checkers"], + "signals": found["signals"], + "friction": friction, + } + journal = root / "journal" / f"{record['ended_at'][:10]}.jsonl" + journal.parent.mkdir(parents=True, exist_ok=True) + with journal.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + return journal, run_id + + # --- git --------------------------------------------------------------------- def numstat(ref: str | None) -> tuple[int, int, set[str]]: @@ -219,7 +380,7 @@ def total(tokens: dict) -> int: return sum(tokens.get(key, 0) for key in TOKEN_KEYS) -def cmd_end(skill: str, counts: dict[str, str]) -> int: +def cmd_end(skill: str, counts: dict[str, str], friction: list[str]) -> int: state_file = state_dir() / f"{skill}.json" if not state_file.exists(): print(f"metrics: no start snapshot for {skill}; run `start {skill}` at invocation", file=sys.stderr) @@ -240,6 +401,7 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for agent in agents: tools.update(agent["tools"]) delta = git_delta(state["git"]) + found = collect_signals(transcript, state["anchor_line"], since) record = { "skill": skill, @@ -255,6 +417,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: "tokens_total": total(all_tokens), "git": delta, "counts": counts, + "signal_counts": found["counts"], + "checkers": found["checkers"], } ledger = repo_root() / ".dev" / "metrics.jsonl" previous = [] @@ -267,8 +431,10 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: if row.get("skill") == skill: previous.append(row) ledger.parent.mkdir(parents=True, exist_ok=True) + ledger_line = sum(1 for _ in ledger.open(encoding="utf-8")) + 1 if ledger.exists() else 1 with ledger.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") + journaled = write_journal(record, ledger, ledger_line, state, friction, found) top_tools = ", ".join(f"{name} {n}" for name, n in tools.most_common(4)) or "none" rows = [ @@ -290,8 +456,11 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: rows.append(("git since start", "n/a (not a git repo)")) for key, value in counts.items(): rows.append((key.replace("_", " "), value)) + measured = [f"{n} {kind.replace('_', ' ')}" for kind, n in sorted(found["counts"].items())] + measured += [f"{name} {c['runs']} runs / {c['failed']} failed" for name, c in sorted(found["checkers"].items())] + rows.append(("friction signals", ", ".join(measured) or "none measured")) if previous and main: - med_tokens = statistics.median(r["tokens_total"] for r in previous if r.get("tokens_total")) + med_tokens = statistics.median([r["tokens_total"] for r in previous if r.get("tokens_total")] or [0]) med_secs = statistics.median(r["seconds"] for r in previous) change = (total(all_tokens) - med_tokens) / med_tokens * 100 if med_tokens else 0 rows.append(( @@ -309,6 +478,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for name, value in rows: print(f"| {name.ljust(width)} | {value} |") print(f"\nledger: {ledger}") + if journaled: + print(f"journal: {journaled[0]} ({journaled[1]}, {len(friction)} narrated friction lines)") return 0 @@ -320,6 +491,8 @@ def main() -> int: end.add_argument("skill") end.add_argument("--count", action="append", default=[], metavar="KEY=VALUE", help="skill-specific measured counter to include, repeatable") + end.add_argument("--friction", action="append", default=[], metavar="TEXT", + help=f"where this run fought the skill's own instructions, at most {MAX_FRICTION}") args = parser.parse_args() if args.command == "start": return cmd_start(args.skill) @@ -329,7 +502,10 @@ def main() -> int: parser.error(f"--count expects KEY=VALUE, got {item!r}") key, value = item.split("=", 1) counts[key.strip()] = value.strip() - return cmd_end(args.skill, counts) + friction = [" ".join(line.split()) for line in args.friction if line.strip()] + if len(friction) > MAX_FRICTION: + parser.error(f"--friction takes at most {MAX_FRICTION} lines, got {len(friction)}; keep the sharpest") + return cmd_end(args.skill, counts, friction) if __name__ == "__main__": diff --git a/dev/skills/build/SKILL.md b/dev/skills/build/SKILL.md index 27b52c8..5755731 100644 --- a/dev/skills/build/SKILL.md +++ b/dev/skills/build/SKILL.md @@ -45,6 +45,8 @@ Every build run first prints the measured run metrics, pasting the table verbati python3 {build-skill-root}/../../scripts/skill-metrics.py end build --count change_sets=N --count scenarios=N --count e2e_passed=N --count e2e_failed=N ``` +Add any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md). + Then it ends with the same two lines, in this order - a green run, a blocked gate, and a run with open deviations all get both: 1. `Next step: run ship over this work, pointed at .dev/{plan-name}/implementation-notes.md and the {plan-name}-e2e-report.html.` Recommend it; never launch it yourself. diff --git a/dev/skills/reflect/SKILL.md b/dev/skills/reflect/SKILL.md new file mode 100644 index 0000000..83424ef --- /dev/null +++ b/dev/skills/reflect/SKILL.md @@ -0,0 +1,67 @@ +--- +name: reflect +description: Consolidate the run journal that scope, scope-review, build, and ship write at the end of every run into cited claims about how the skills themselves behaved, rank the recurring friction threads across repositories, and hand off one chosen thread as a scope brief with an eval case from the real run. Use periodically to decide what to improve in the dev workflow next, or with "resolve {thread} {sha}" once a fix for a thread has merged. +disable-model-invocation: true +--- + +# Reflect + +Turn what past runs measured into evidence about the skills, so the next skill change fixes something that actually recurred. +The store is `~/.dev-workflow/memory/` (or `$DEV_MEMORY_DIR`), local to this machine and shared by every repository; [references/claims.md](references/claims.md) owns its records, the claim schema, and what makes a claim worth keeping. + +`{reflect-skill-root}` is the directory holding this SKILL.md, `{claims}` is `python3 {reflect-skill-root}/../../scripts/claims.py`, and `{project-slug}` is the basename of the current repository root, or `workflow-memory` outside one. + +Limits: +- Never edit a skill, a reference, or an eval. The output is a brief a person takes to `scope`; adaptation reaches a skill only through a reviewed commit. +- Never write `claims.jsonl`, `resolutions.jsonl`, or a derived file by hand; every change goes through `{claims}`, so the checks hold. +- A claim is about how a skill behaved, never about the consuming repository's code or its people. +- Nothing is committed, pushed, or published unless the user asks. + +## Resolve mode + +When invoked as `resolve {thread} {sha} [eval-id]`, run `{claims} resolve {thread} --commit {sha} [--eval {eval-id}]`, print `{claims} top --all`, and stop. +A later claim in that thread reopens it, which is how a fix that did not hold shows up. + +## 1. Gather + +Create `/tmp/{project-slug}/reflect/`, then run `{claims} pending > /tmp/{project-slug}/reflect/pending.json` and `{claims} top --all`. +With no pending runs, skip to step 4; with no pending runs and no claims at all, say the journal is empty, name the four skills that write it, and stop. + +## 2. Extract + +Launch one read-only subagent per batch of up to five pending runs, all in one message. +Hand each the batch's journal entries, the current thread list from `{claims} top --all`, and [references/claims.md](references/claims.md), and have it write candidate claims, one JSON object per line, to `/tmp/{project-slug}/reflect/candidates-{n}.jsonl`. +Each agent reads the transcript only around the lines its signals cite, never a whole transcript, and reuses an existing thread slug whenever the claim is the same problem. +An agent may write zero candidates; a run that went as designed is not friction. + +## 3. Check loop + +Concatenate the candidate files into `/tmp/{project-slug}/reflect/candidates.jsonl` and loop until it exits 0: + +```bash +{claims} add /tmp/{project-slug}/reflect/candidates.jsonl +``` + +It adds nothing while any candidate fails, and each error names the candidate. +Fix that candidate's quote, source, or shape against the transcript, or drop it; never bend a quote toward a claim the evidence does not make. +Then run `{claims} mark` with every pending run id, including runs that produced no claims. + +## 4. Present + +Run `{claims} report /tmp/{project-slug}/reports/workflow-memory.html` and open it per [../../references/reporting.md](../../references/reporting.md). +Show `{claims} top`, then `{claims} show {thread}` for the top three open or reopened threads, condensed to each claim and its quote. +Ask one question with the host's structured user-input tool when available: which thread to pursue now, a claim that misreads its evidence and should be retracted, or none. +Recommend the thread with the most runs across the most repositories; a reopened thread outranks an open one with the same reach. + +A claim the user says misreads its evidence is retracted with `{claims} retract {id} --reason "{their words}"`, and the question is asked again. + +## 5. Brief + +For the chosen thread, write `/tmp/{project-slug}/reflect/brief-{thread}.md` per [references/brief.md](references/brief.md). +Read the implicated skill's own text at `{reflect-skill-root}/../{skill}/` to name the lines the evidence points at. +Prefer a direction that adds or tightens a deterministic check the skill loops against over one that adds prose, per the repository rule that long rule lists become guidelines. + +## 6. Close + +Print the brief's path and its one-line problem statement. +Recommend the next step: run `/dev:scope` in the workflow repository with the brief, then `/dev:build`; once the fix merges, `/dev:reflect resolve {thread} {sha} {eval-id}`. diff --git a/dev/skills/reflect/references/brief.md b/dev/skills/reflect/references/brief.md new file mode 100644 index 0000000..2f66da8 --- /dev/null +++ b/dev/skills/reflect/references/brief.md @@ -0,0 +1,40 @@ +# Brief + +The handoff from `reflect` to `scope` for one thread. +It is self-contained: a person or a fresh session reading only this file can start `scope` in the workflow repository. + +```markdown +# {thread} + +Problem: {one sentence, in terms of what the runs did, not what the skill text says} + +## Evidence + +{claims show output condensed to one bullet per claim:} +- [c-0012] {claim} - "{quote}" ({repo}, {run id}, line {line}) + +Reach: {runs} runs across {repos} repositories, status {open|reopened}; {for a reopened thread, the commit that was meant to fix it}. + +## Where the skill text is implicated + +- `dev/skills/{skill}/SKILL.md` step {n}: "{the instruction the evidence contradicts or that let it happen}" + +## Direction + +{the smallest change that would have prevented the evidence; prefer a deterministic check the skill loops against over another line of prose, and say which one} + +Not in scope: {what the evidence does not support changing} + +## Eval case + +Append to `dev/evals/{skill}.json`, built from the real run so the regression is the one that happened: + +{"id": "{skill}-{thread-suffix}", "prompt": "...", "assertions": ["..."]} + +## After the fix merges + +/dev:reflect resolve {thread} {sha} {eval id} +``` + +Keep quotes as they are in the store; trim around them, never inside them. +When the evidence points at two skills, write one brief per skill and say so in each. diff --git a/dev/skills/reflect/references/claims.md b/dev/skills/reflect/references/claims.md new file mode 100644 index 0000000..1d2fe1e --- /dev/null +++ b/dev/skills/reflect/references/claims.md @@ -0,0 +1,57 @@ +# Claims + +The memory store, the claim schema, and the judgment `claims.py` cannot make for you. + +## The store + +`$DEV_MEMORY_DIR`, default `~/.dev-workflow/memory/`: + +| path | written by | role | +| ---- | ---------- | ---- | +| `journal/YYYY-MM-DD.jsonl` | `skill-metrics.py end` | one entry per run: measured signals, checker runs, narrated friction | +| `claims.jsonl` | `claims.py add`, `retract` | every claim ever accepted, with its status | +| `consolidated.jsonl` | `claims.py mark` | runs already read | +| `resolutions.jsonl` | `claims.py resolve` | thread, fixing commit, eval id, time | +| `skills/{skill}.md`, `threads.yaml`, `progression.md` | `claims.py rebuild` | derived; every line cites a claim id | + +Records are appended or change status; they are never deleted. +A retracted claim stays so the same evidence cannot quietly bring it back. + +## A journal entry + +The fields an extracting agent reads: + +- `id`, `skill`, `repo`, `started_at`, `transcript`, `anchor_line`. +- `signals`: up to 20 per type, each `{type, transcript, line, excerpt}`. Types: `interrupt` (the user stopped the run), `denied` (the user refused a tool call), `tool_error` (a non-checker tool failed), `user_turn` (anything the user typed after invocation). +- `checkers`: per checker script, `runs`, `failed`, and `first_failure` with its line. +- `counts`: the skill's own measured counters, `signal_counts`: totals before the cap. +- `friction`: up to three lines the skill narrated about itself. Narrated, not measured: a claim resting on one alone takes salience 1 unless a measured signal in the same run agrees. + +## Candidate shape + +One JSON object per line: + +```json +{"skill": "build", "thread": "build-validation-reruns", "kind": "friction", + "claim": "Build reran the full Validation block after each change set; the user stopped it.", + "quote": "stop rerunning the whole validation block", + "source": {"run": "r-20261005T100000-build", "line": 412}, + "salience": 2} +``` + +- `skill`: the skill whose instructions the claim is about, which need not be the run's skill (a build run can expose a scope defect). +- `thread`: `{skill}-{problem}` in lowercase slug form; reuse an existing slug for the same problem. +- `kind`: `friction` (the skill made the run slower or the user intervene), `defect` (the skill's instructions were wrong or contradictory), `cost` (a measured cost far above that skill's median), `success` (a skill rule that demonstrably saved the run; keep these, they say what not to remove). +- `claim`: one sentence, at most 240 characters, no em dash. +- `quote`: verbatim text, 8 to 300 characters, found at the source. Whitespace and case are ignored, nothing else is. +- `source`: `run` plus `line` (a line of the run's transcript, or with `transcript`, of one of its subagent transcripts); omit `line` to quote the journal entry itself. +- `salience`: 1 noted once, 2 cost real time or a user intervention, 3 broke the run or produced a wrong result. +- `supersedes`: optional id of an active claim this one replaces with better evidence or a sharper reading. + +## What is not a claim + +- A `user_turn` that answers a question the skill was designed to ask, such as a scope interview answer. +- A checker that failed and passed on the next run: that is the loop working. +- A `denied` call that reflects the user's standing policy rather than a mistake the skill made. +- Anything about the consuming repository's code quality, its tests, or its people. +- A guess about why without a quote that shows it; write the observable claim instead. diff --git a/dev/skills/scope-review/SKILL.md b/dev/skills/scope-review/SKILL.md index da282ee..b75a049 100644 --- a/dev/skills/scope-review/SKILL.md +++ b/dev/skills/scope-review/SKILL.md @@ -129,7 +129,7 @@ Asked: {the question} - Answered: {the user's decision} -> {what the spec says n ## Wrap up -Open the chat summary with the table from `python3 {scope-review-skill-root}/../../scripts/skill-metrics.py end scope-review --count findings_verified=N --count findings_refuted=N --count refinements_applied=N --count escalated=N`, pasted verbatim. +Open the chat summary with the table from `python3 {scope-review-skill-root}/../../scripts/skill-metrics.py end scope-review --count findings_verified=N --count findings_refuted=N --count refinements_applied=N --count escalated=N` plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md), pasted verbatim. Then summarize in the same message: the verdict, what was refined and what the user's answers changed (so the loop's edits stay auditable after the fact), what was promoted to `docs/decisions.md` and `docs/contracts.md`, anything deferred with its open question, and a link to the report. Recommend next steps, never invoking them: diff --git a/dev/skills/scope/SKILL.md b/dev/skills/scope/SKILL.md index 3dccad7..caff8ae 100644 --- a/dev/skills/scope/SKILL.md +++ b/dev/skills/scope/SKILL.md @@ -149,7 +149,7 @@ Audit the run itself. Three checks, each reported as a typed line - silence read - **Sizing** - `held | mis-sized`. Did the small/full call survive? Name the evidence when it didn't. - **Catalog gaps** - `none | gap`. What did the reviewer or blind-spot pass find that phase 2 should have caught? A missed *category* is a proposed edit to the phase 2 list - propose it to the user, never apply it silently. -Then print the measured run metrics with `python3 {scope-skill-root}/../../scripts/skill-metrics.py end scope --count decisions=N --count change_sets=N --count scenarios=N`, pasting its table verbatim. +Then print the measured run metrics with `python3 {scope-skill-root}/../../scripts/skill-metrics.py end scope --count decisions=N --count change_sets=N --count scenarios=N`, plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md), pasting its table verbatim. Then recommend next steps, never launching them: `scope-review` first when the change is large or risky or build will run in a different session - it reviews the spec with a fresh-context panel and refines it in place before any code exists - and `build` to implement. ## Jira sync diff --git a/dev/skills/ship/SKILL.md b/dev/skills/ship/SKILL.md index 90ee2b1..1b765d7 100644 --- a/dev/skills/ship/SKILL.md +++ b/dev/skills/ship/SKILL.md @@ -135,7 +135,7 @@ Summarize whichever phases ran in one chat message, opening with the measured ru python3 {ship-skill-root}/../../scripts/skill-metrics.py end ship --count violations_found=N --count violations_fixed=N --count violations_surviving=N --count findings_verified=N --count findings_refuted=N --count remediation_rounds=N --count blockers_cleared=N --count evidence_items=N ``` -Pass only counters you tallied from tool output and the aggregate script; the table it prints (time, tokens, agents, tool calls, git delta, trend against earlier runs) is pasted verbatim, never retyped. +Pass only counters you tallied from tool output and the aggregate script, plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md); the table it prints (time, tokens, agents, tool calls, git delta, trend against earlier runs) is pasted verbatim, never retyped. For the gauntlet, per tool: violations found, fixed, and surviving (with the human call each is waiting on); name the tools acquired or built this run and where they live; state the scope honestly - "hardened the diff" is not "hardened the repo". For the review: the final verdict and top findings, linking every `review_N.md` this run wrote, the local HTML report, and the published URL when one was requested and created; per remediation round, which blockers cleared and which survived. For the pull request: its URL, draft or ready, what the Evidence section shows and where it came from, and the state of its required checks. diff --git a/docs/architecture.md b/docs/architecture.md index 4a4a254..6833c40 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,11 +1,11 @@ # Architecture Purpose: this repository is a monorepo of agent skills and the tooling that ships them. -Three plugins live here: `dev`, the hand-invoked development workflow (scope, scope-review, build, ship, commit, and two presentation skills); `factory`, whose `run` skill orchestrates copies of the same four phases unattended; and `bootstrap`, whose `agents-md` skill writes a consuming repository's root AGENTS.md from the workflow's lessons. +Three plugins live here: `dev`, the hand-invoked development workflow (scope, scope-review, build, ship, commit, reflect, and two presentation skills); `factory`, whose `run` skill orchestrates copies of the same four phases unattended; and `bootstrap`, whose `agents-md` skill writes a consuming repository's root AGENTS.md from the workflow's lessons. A person invokes each `dev` skill by hand; skills never invoke each other, and each one recommends the next step instead - except the factory `run` skill, the one sanctioned invoker, which launches its copied phase skills by path and judges their completion itself. The generated Codex distributions under `plugins/` are built from their source plugin and never edited by hand. -Captured: 2026-09-17 (full, scope) - Updated: 2026-09-25 (the bootstrap plugin writes a consuming repository's AGENTS.md) +Captured: 2026-09-17 (full, scope) - Updated: 2026-10-05 (runs journal their friction; the reflect skill consolidates it into cited claims) ## Components @@ -13,6 +13,7 @@ Captured: 2026-09-17 (full, scope) - Updated: 2026-09-25 (the bootstrap plugin w | --------- | -------------- | -------- | -------- | | dev skills | the interactive workflow skills and the references and scripts they share | `dev/skills/`, `dev/references/`, `dev/scripts/` | nothing at runtime; read by Claude, Codex, opencode, and Pi hosts | | dev evals | comprehension evals for the skills | `dev/evals/` | nothing at runtime | +| workflow memory | the run journal `dev/scripts/skill-metrics.py` appends to and the cited claims, threads, and resolutions `dev/scripts/claims.py` keeps from it; local to the machine, shared by every consuming repository, never committed | ~/.dev-workflow/memory/ (outside this repository) | read by the reflect skill | | generated plugins | the Codex distributions built from each source plugin, with invocation policy in agents/openai.yaml | `plugins/` | Codex plugin marketplace | | repo scripts | validation of the whole repository, the plugin generator, and the Pi transport self-test | `scripts/` | every other component | | harden tools | optional local analyses a ship gauntlet can draw on: added lines, coverage-weighted complexity, flaky reruns, mutation | `tools/harden/` | nothing; run by hand against a target repository | @@ -25,7 +26,12 @@ Captured: 2026-09-17 (full, scope) - Updated: 2026-09-25 (the bootstrap plugin w ### A dev skill run 1. A person invokes `/dev:scope`, `/dev:build`, `/dev:ship`, or `/dev:commit` in Claude Code (or the Codex, opencode, or Pi equivalent). 2. The skill reads its SKILL.md and references, writes plan files under .dev/{plan-name}/ in the consuming repository, and recommends the next skill; skills never invoke each other. -3. `dev/scripts/skill-metrics.py` measures each run and appends a row to the consuming repository's .dev/metrics.jsonl. +3. `dev/scripts/skill-metrics.py` measures each run, appends a row to the consuming repository's .dev/metrics.jsonl, and appends a journal entry of measured friction signals to the workflow memory. + +### Learning from runs +1. A person invokes `/dev:reflect` in any repository; `dev/scripts/claims.py pending` lists the journal entries not yet consolidated. +2. Read-only subagents propose claims about the skills, each quoting a transcript line or journal entry; `claims.py add` accepts a batch only when every quote is found at its source, then rebuilds the derived per-skill pages and threads. +3. The person picks a thread; the skill writes a scope brief with an eval case from the real run and never edits a skill. After the fix merges, `/dev:reflect resolve` records the commit, and a later claim in the thread reopens it. ### Building the Codex distribution 1. `scripts/build_codex_plugin.py` copies skills/, references/, and `scripts/` of the source plugin (and factory/phases/ for the factory) into plugins/{name}/, strips Claude-only frontmatter, and writes agents/openai.yaml for invocable skills. @@ -49,6 +55,7 @@ Captured: 2026-09-17 (full, scope) - Updated: 2026-09-25 (the bootstrap plugin w | Claude Code, Codex, opencode, Pi | host | the skills | each reads the skills in its own format; the generated tree under `plugins/` serves Codex | | git and GitHub | external | the ship skill | branch state and `gh pr` calls made during a ship run | | Jira | external HTTP | dev skills | through `acli`, only when .dev/config.json enables it | +| workflow memory | local store | skill-metrics.py, claims.py | ~/.dev-workflow/memory/ or $DEV_MEMORY_DIR; quotes from transcripts stay on the machine | | consuming repository | store | the skills | `.dev/` plan files, `docs/` ledgers, the .factory/ config and injected phase copies, and the root AGENTS.md bootstrap writes | ## Cross-cutting @@ -58,4 +65,4 @@ Rules: no em dash anywhere, SKILL.md under roughly 150 lines, every skill `disab ## Entry points -`scripts/validate.sh`, `scripts/build_codex_plugin.py`, `scripts/test_pi_runner.sh`, `dev/scripts/skill-metrics.py`, `bootstrap/skills/agents-md/scripts/check-agents-md.py`. +`scripts/validate.sh`, `scripts/build_codex_plugin.py`, `scripts/test_pi_runner.sh`, `dev/scripts/skill-metrics.py`, `dev/scripts/claims.py`, `bootstrap/skills/agents-md/scripts/check-agents-md.py`. diff --git a/docs/decisions.md b/docs/decisions.md index 1eb845d..f2f980a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -377,3 +377,21 @@ D-bootstrap-pi: Does bootstrap ship to Pi? (2026-09-25, user request) ⊘ not doing - P01 and D-pi keep the Pi package dev-only; reopen when Pi gains plugin namespaces D-bootstrap-dev-reads-agents-md: Do scope and build read AGENTS.md's Commands section? (2026-09-25, user request) ⊘ not doing yet - scope's Validation block and build's command discovery still probe manifests every run; reopen once bootstrap has run on real repositories and the Commands section has proven reliable ? verify: a functional run in `bootstrap/evals/results.md` + +## Workflow memory + +D-memory-store: Where does the record of past runs live? (2026-10-05, user request, modeled on the Muse memory system) + ✓ `~/.dev-workflow/memory/` (or `$DEV_MEMORY_DIR`), one store per machine shared by every consuming repository, never committed - a problem only shows as recurring when runs from several repositories meet, and the evidence quotes transcripts of client code (user accepted the recommendation, 2026-10-05) ⚠ runs on another machine never meet these; a person who wants that points `DEV_MEMORY_DIR` at shared storage + ✗ per repository under `.dev/` - simpler, but never sees a pattern across repositories, and `.dev/` is not committed anyway + ✗ a database with embeddings, as Muse indexes `memory.entries` - the volume is a few runs a day; `grep` over JSONL answers every query the skill asks +D-memory-capture: How does a run enter the store? (2026-10-05, user request) + ✓ automatically, by `skill-metrics.py end`, which every timed skill already calls: the friction signals are measured from the transcript (interrupts, denied and failed tool calls, the user's own turns, checker runs and failures), each with its line, and the skill adds at most three narrated `--friction` lines, kept apart from the measured ones; opt out per repository in `.dev/config.json` or per machine with `DEV_MEMORY_DIR=off` (user accepted, 2026-10-05) ⚠ `user_turn` is noisy in interactive skills, where most turns are answers the skill asked for; extraction sorts that out, the capture does not + ✗ opt-in per repository - the friction worth finding is in the runs nobody thought to record + ✗ the skill narrates its own retrospective - the model's recollection of a long run is exactly what `skill-metrics.py` exists to replace, per P-deterministic-guards-over-prose +D-memory-claims: What may the store say about a skill? (2026-10-05, user request) + ✓ only cited claims: `claims.py add` accepts a batch only when every quote is found verbatim at the transcript line or journal entry it cites, rejects credential shapes, and rebuilds the per-skill pages and threads from the records so every derived line cites a claim id; a retracted claim stays on record with its reason so the same evidence cannot return, and a resolved thread reopens on a later claim - Muse's source-cited derived memory and its forgetting workflow, enforced by a checker rather than asked for per P-deterministic-guards-over-prose ⚠ a verbatim quote proves the words occurred, not that the claim reads them rightly; the user's retraction is the guard for that + ✗ free-text lessons appended to a file - unverifiable, and a wrong lesson has no way out +D-memory-no-self-edit: May the loop change a skill on its own? (2026-10-05, user request) + ✓ never - `reflect` ends at a brief with an eval case from the real run, and the fix goes through `scope` and `build` in this repository like any change; Muse keeps its dream prose out of the prompt (`prompt_tainted: false`), and the factory dream loop that deployed its own winning revisions was removed (D-remove-factory) (user accepted, 2026-10-05) ⚠ the loop is only as frequent as a person runs `/dev:reflect` + ✗ open a pull request with the proposed skill edit - a model-authored revision reviewed only as a diff, without the scope step that argues it against alternatives + ✗ a nightly scheduled consolidation - a schedule is per host, and consolidation without someone to pick a thread only grows the store diff --git a/factory/references/run-journal.md b/factory/references/run-journal.md new file mode 100644 index 0000000..0eee3dd --- /dev/null +++ b/factory/references/run-journal.md @@ -0,0 +1,12 @@ +# Run Journal + +`skill-metrics.py end` appends one entry per run to the cross-repository journal under `~/.dev-workflow/memory/journal/` (or `$DEV_MEMORY_DIR`), which the `reflect` skill consolidates into cited claims about the skills themselves. +The script measures the friction signals on its own from the transcript: interrupts, denied and failed tool calls, the user's own turns, and how often each checker ran and failed. + +The one thing it cannot measure is where the run fought the skill's own instructions, so the skill adds that, as up to three `--friction "..."` arguments on the `end` call: + +- One line each: the step, what the instruction asked, and what the run had to do instead, quoting the moment when there is one. +- About the skill's text, never the repository's code: "step 3 assumes one Validation block; this repo has two, so the wave gate ran neither" is friction, "the tests are slow" is not. +- None when the run went as the skill describes; an empty journal entry is the common and good case. + +Set `DEV_MEMORY_DIR=off`, or `"memory": {"enabled": false}` in `.dev/config.json`, to keep a repository's runs out of the journal. diff --git a/factory/scripts/skill-metrics.py b/factory/scripts/skill-metrics.py index ac78ef0..2301ce3 100755 --- a/factory/scripts/skill-metrics.py +++ b/factory/scripts/skill-metrics.py @@ -3,7 +3,7 @@ Usage: python3 skill-metrics.py start {skill} - python3 skill-metrics.py end {skill} [--count key=value ...] + python3 skill-metrics.py end {skill} [--count key=value ...] [--friction text ...] `start` snapshots the git state and locates the session transcript under $CLAUDE_CONFIG_DIR (default ~/.claude), anchoring on the transcript line that @@ -13,6 +13,16 @@ so later runs can be compared against earlier ones. Every value is measured; nothing here is narrated from memory. Missing transcript or git degrade to "n/a" rather than failing the run. + +`end` also appends one entry to the cross-repository run journal under +$DEV_MEMORY_DIR (default ~/.dev-workflow/memory/journal/), which the reflect +skill consolidates into cited claims about the skills themselves. The entry +holds the friction signals measured from the transcript (interrupts, denied or +failed tool calls, the user's own turns, and how often each deterministic +checker ran and failed), each with its transcript path and line, plus at most +three `--friction` lines the skill narrates, kept apart as narrated. Set +DEV_MEMORY_DIR=off, or `"memory": {"enabled": false}` in .dev/config.json, to +skip the journal. """ from __future__ import annotations @@ -31,6 +41,18 @@ TEST_FILE = re.compile(r"(^|/)(tests?|specs?|__tests__)(/|$)|[._-](test|spec)s?\.[A-Za-z]+$|^test_.*\.py$") TOKEN_KEYS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens") +# The deterministic checkers the skills loop against; a failed run of one is loop +# progress, so it is counted per checker instead of logged as a tool error. +CHECKERS = ("lint-spec.py", "check-tests.py", "pr-evidence.py", "architecture-check.py", + "check-agents-md.py", "aggregate-findings.py", "claims.py") +# A checker counts only where a command runs it, never where one greps or edits it. +CHECKER_RUN = re.compile(r"(?:^|[;&|(]\s*|\bpython3?\s+(?:-\S+\s+)*)\S*?(" + "|".join(re.escape(c) for c in CHECKERS) + r")(?=\s|$)", + re.M) +INTERRUPT = "[Request interrupted by user" +DENIED = ("doesn't want to proceed", "was rejected", "Permission denied by user") +SIGNAL_CAP = 20 +EXCERPT = 240 +MAX_FRICTION = 3 def now_iso() -> str: @@ -130,6 +152,145 @@ def subagent_files(transcript: Path) -> list[Path]: return sorted(folder.glob("*.jsonl")) if folder.is_dir() else [] +# --- friction signals -------------------------------------------------------- + +def block_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(block_text(item.get("text", item.get("content", ""))) if isinstance(item, dict) + else str(item) for item in content) + return "" + + +def human_text(obj: dict) -> str | None: + """The text a person typed, or None for tool results and injected turns.""" + if obj.get("isMeta") or obj.get("isSidechain"): + return None + content = (obj.get("message") or {}).get("content") + if isinstance(content, list): + if any(isinstance(b, dict) and b.get("type") == "tool_result" for b in content): + return None + content = block_text([b for b in content if isinstance(b, dict) and b.get("type") == "text"]) + if not isinstance(content, str): + return None + text = content.strip() + if not text or text.startswith("<") or INTERRUPT in text or text.startswith("Base directory for this skill"): + return None + return text + + +def signals(path: Path, start: int, since: float | None, state: dict) -> None: + """Add one transcript's friction signals and checker runs to `state`.""" + pending: dict[str, str] = {} + first = True + for index, obj in iter_lines(path, start): + stamp = obj.get("timestamp") + if first and stamp: + first = False + if since is not None and parse_iso(stamp) < since: + return + message = obj.get("message") or {} + if obj.get("type") == "assistant": + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + command = str((block.get("input") or {}).get("command", "")) + match = CHECKER_RUN.search(command) + checker = match.group(1) if match else None + if checker: + pending[block.get("id", "")] = checker + state["checkers"].setdefault(checker, {"runs": 0, "failed": 0})["runs"] += 1 + continue + if obj.get("type") != "user": + continue + where = {"transcript": str(path), "line": index} + content = message.get("content") + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + text = block_text(block.get("content", block.get("text", ""))) + if block.get("type") == "text" and INTERRUPT in text: + note(state, "interrupt", where, text) + elif block.get("type") == "tool_result" and block.get("is_error"): + checker = pending.get(block.get("tool_use_id", "")) + if checker: + entry = state["checkers"][checker] + entry["failed"] += 1 + entry.setdefault("first_failure", where) + else: + note(state, "denied" if any(d in text for d in DENIED) else "tool_error", where, text) + if isinstance(content, str) and INTERRUPT in content: + note(state, "interrupt", where, content) + elif path == state["main"]: + typed = human_text(obj) + if typed: + note(state, "user_turn", where, typed) + + +def note(state: dict, kind: str, where: dict, text: str) -> None: + state["counts"][kind] = state["counts"].get(kind, 0) + 1 + if state["counts"][kind] <= SIGNAL_CAP: + state["signals"].append({"type": kind, **where, "excerpt": " ".join(text.split())[:EXCERPT]}) + + +def collect_signals(transcript: Path | None, anchor: int, since: float) -> dict: + state: dict = {"main": transcript, "counts": {}, "signals": [], "checkers": {}} + if transcript and transcript.exists(): + # Skip the invocation line itself: it is the skill's own prompt, not a reaction to it. + signals(transcript, anchor + 1, None, state) + for path in subagent_files(transcript): + signals(path, 0, since, state) + state.pop("main") + return state + + +# --- run journal ------------------------------------------------------------- + +def memory_dir() -> Path | None: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + return None + config = repo_root() / ".dev" / "config.json" + try: + if json.loads(config.read_text(encoding="utf-8")).get("memory", {}).get("enabled") is False: + return None + except (OSError, ValueError, AttributeError): + pass + return Path(value).expanduser() + + +def write_journal(record: dict, ledger: Path, ledger_line: int, state: dict, + friction: list[str], found: dict) -> tuple[Path, str] | None: + root = memory_dir() + if root is None: + return None + stamp = re.sub(r"[^0-9T]", "", record["started_at"])[:15] + run_id = f"r-{stamp}-{record['skill']}" + entry = { + "id": run_id, + "skill": record["skill"], + "repo": repo_root().name, + "repo_path": str(repo_root()), + "started_at": record["started_at"], + "ended_at": record["ended_at"], + "seconds": record["seconds"], + "transcript": state.get("transcript"), + "anchor_line": state.get("anchor_line"), + "ledger": str(ledger), + "ledger_line": ledger_line, + "counts": record["counts"], + "signal_counts": found["counts"], + "checkers": found["checkers"], + "signals": found["signals"], + "friction": friction, + } + journal = root / "journal" / f"{record['ended_at'][:10]}.jsonl" + journal.parent.mkdir(parents=True, exist_ok=True) + with journal.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + return journal, run_id + + # --- git --------------------------------------------------------------------- def numstat(ref: str | None) -> tuple[int, int, set[str]]: @@ -219,7 +380,7 @@ def total(tokens: dict) -> int: return sum(tokens.get(key, 0) for key in TOKEN_KEYS) -def cmd_end(skill: str, counts: dict[str, str]) -> int: +def cmd_end(skill: str, counts: dict[str, str], friction: list[str]) -> int: state_file = state_dir() / f"{skill}.json" if not state_file.exists(): print(f"metrics: no start snapshot for {skill}; run `start {skill}` at invocation", file=sys.stderr) @@ -240,6 +401,7 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for agent in agents: tools.update(agent["tools"]) delta = git_delta(state["git"]) + found = collect_signals(transcript, state["anchor_line"], since) record = { "skill": skill, @@ -255,6 +417,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: "tokens_total": total(all_tokens), "git": delta, "counts": counts, + "signal_counts": found["counts"], + "checkers": found["checkers"], } ledger = repo_root() / ".dev" / "metrics.jsonl" previous = [] @@ -267,8 +431,10 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: if row.get("skill") == skill: previous.append(row) ledger.parent.mkdir(parents=True, exist_ok=True) + ledger_line = sum(1 for _ in ledger.open(encoding="utf-8")) + 1 if ledger.exists() else 1 with ledger.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") + journaled = write_journal(record, ledger, ledger_line, state, friction, found) top_tools = ", ".join(f"{name} {n}" for name, n in tools.most_common(4)) or "none" rows = [ @@ -290,8 +456,11 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: rows.append(("git since start", "n/a (not a git repo)")) for key, value in counts.items(): rows.append((key.replace("_", " "), value)) + measured = [f"{n} {kind.replace('_', ' ')}" for kind, n in sorted(found["counts"].items())] + measured += [f"{name} {c['runs']} runs / {c['failed']} failed" for name, c in sorted(found["checkers"].items())] + rows.append(("friction signals", ", ".join(measured) or "none measured")) if previous and main: - med_tokens = statistics.median(r["tokens_total"] for r in previous if r.get("tokens_total")) + med_tokens = statistics.median([r["tokens_total"] for r in previous if r.get("tokens_total")] or [0]) med_secs = statistics.median(r["seconds"] for r in previous) change = (total(all_tokens) - med_tokens) / med_tokens * 100 if med_tokens else 0 rows.append(( @@ -309,6 +478,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for name, value in rows: print(f"| {name.ljust(width)} | {value} |") print(f"\nledger: {ledger}") + if journaled: + print(f"journal: {journaled[0]} ({journaled[1]}, {len(friction)} narrated friction lines)") return 0 @@ -320,6 +491,8 @@ def main() -> int: end.add_argument("skill") end.add_argument("--count", action="append", default=[], metavar="KEY=VALUE", help="skill-specific measured counter to include, repeatable") + end.add_argument("--friction", action="append", default=[], metavar="TEXT", + help=f"where this run fought the skill's own instructions, at most {MAX_FRICTION}") args = parser.parse_args() if args.command == "start": return cmd_start(args.skill) @@ -329,7 +502,10 @@ def main() -> int: parser.error(f"--count expects KEY=VALUE, got {item!r}") key, value = item.split("=", 1) counts[key.strip()] = value.strip() - return cmd_end(args.skill, counts) + friction = [" ".join(line.split()) for line in args.friction if line.strip()] + if len(friction) > MAX_FRICTION: + parser.error(f"--friction takes at most {MAX_FRICTION} lines, got {len(friction)}; keep the sharpest") + return cmd_end(args.skill, counts, friction) if __name__ == "__main__": diff --git a/package.json b/package.json index 34fdfb0..148b72b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tobrun/dev-workflow", - "version": "3.6.0", + "version": "3.7.0", "description": "Test-focused development workflow skills for Claude Code, Codex, and Pi.", "keywords": [ "pi-package", diff --git a/plugins/dev/.codex-plugin/plugin.json b/plugins/dev/.codex-plugin/plugin.json index 9b80da9..9a69ca0 100644 --- a/plugins/dev/.codex-plugin/plugin.json +++ b/plugins/dev/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "dev", - "version": "3.6.0", + "version": "3.7.0", "description": "Development workflow skills: scope changes with argued decisions, build across unit/integration/e2e with every scenario proven by tests, ship with a deterministic quality gauntlet and an adversarially verified review, create structured commits that feed a decision ledger, and render pitches or comprehension quizzes.", "author": { "name": "Tobrun" diff --git a/plugins/dev/references/run-journal.md b/plugins/dev/references/run-journal.md new file mode 100644 index 0000000..0eee3dd --- /dev/null +++ b/plugins/dev/references/run-journal.md @@ -0,0 +1,12 @@ +# Run Journal + +`skill-metrics.py end` appends one entry per run to the cross-repository journal under `~/.dev-workflow/memory/journal/` (or `$DEV_MEMORY_DIR`), which the `reflect` skill consolidates into cited claims about the skills themselves. +The script measures the friction signals on its own from the transcript: interrupts, denied and failed tool calls, the user's own turns, and how often each checker ran and failed. + +The one thing it cannot measure is where the run fought the skill's own instructions, so the skill adds that, as up to three `--friction "..."` arguments on the `end` call: + +- One line each: the step, what the instruction asked, and what the run had to do instead, quoting the moment when there is one. +- About the skill's text, never the repository's code: "step 3 assumes one Validation block; this repo has two, so the wave gate ran neither" is friction, "the tests are slow" is not. +- None when the run went as the skill describes; an empty journal entry is the common and good case. + +Set `DEV_MEMORY_DIR=off`, or `"memory": {"enabled": false}` in `.dev/config.json`, to keep a repository's runs out of the journal. diff --git a/plugins/dev/scripts/claims.py b/plugins/dev/scripts/claims.py new file mode 100644 index 0000000..bed1904 --- /dev/null +++ b/plugins/dev/scripts/claims.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""The dev workflow's memory of its own runs: cited claims about the skills. + +Usage: + python3 claims.py pending journal runs not yet consolidated, as JSON + python3 claims.py add CANDIDATES.jsonl validate and append claims, all or nothing + python3 claims.py mark RUN_ID ... record runs as consolidated + python3 claims.py rebuild regenerate every derived file from the records + python3 claims.py check validate the records and that derived files are current + python3 claims.py top [--limit N] [--all] rank threads, open and reopened first + python3 claims.py show TARGET a thread, skill, or claim with its evidence + python3 claims.py retract CLAIM_ID --reason R forget a claim and rebuild without it + python3 claims.py resolve THREAD --commit SHA [--eval NAME] + python3 claims.py report OUT.html render the threads as a self-contained page + +The store lives in $DEV_MEMORY_DIR (default ~/.dev-workflow/memory). The records +are journal/*.jsonl (written by skill-metrics.py end), claims.jsonl, +resolutions.jsonl, and consolidated.jsonl; skills/*.md, threads.yaml, and +progression.md are derived from them by `rebuild` and never edited by hand. + +A claim is accepted only when its quote is found verbatim (whitespace aside) at +its cited source: a transcript line of the run it names, or that run's journal +entry. A retracted claim stays in claims.jsonl so the same evidence cannot +bring it back. A thread resolved by a commit reopens when a claim observed +after the resolution joins it. Nothing here edits a skill. +""" + +from __future__ import annotations + +import argparse +import html +import json +import os +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +KINDS = ("friction", "defect", "cost", "success") +SLUG = re.compile(r"^[a-z][a-z0-9-]{1,60}$") +CLAIM_ID = re.compile(r"^c-\d{4,}$") +FIELDS = {"skill", "thread", "kind", "claim", "quote", "source", "salience", "supersedes"} +CREDENTIAL = re.compile( + r"AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{30,}|github_pat_\w{20,}|sk-[A-Za-z0-9_-]{20,}" + r"|xox[baprs]-[A-Za-z0-9-]{10,}|-----BEGIN [A-Z ]*PRIVATE KEY" + r"|(?i:(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^\s'\"]{8,})" +) +EM_DASH = "\u2014" +DERIVED_NOTE = "Derived by claims.py rebuild from the records; never edit by hand." + + +# --- store ------------------------------------------------------------------- + +def root() -> Path: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + sys.exit("claims: the memory store is turned off (DEV_MEMORY_DIR=off)") + return Path(value).expanduser() + + +def now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def read_jsonl(path: Path) -> list[dict]: + if not path.exists(): + return [] + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def write_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + tmp.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") + tmp.replace(path) + + +def append_jsonl(path: Path, rows: list[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row) + "\n") + + +def runs(store: Path) -> dict[str, dict]: + found: dict[str, dict] = {} + for path in sorted((store / "journal").glob("*.jsonl")): + for row in read_jsonl(path): + if row.get("id"): + found[row["id"]] = row + return found + + +def claims(store: Path) -> list[dict]: + return read_jsonl(store / "claims.jsonl") + + +def norm(text: str) -> str: + return " ".join(str(text).split()) + + +def strings(value) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [s for item in value.values() for s in strings(item)] + if isinstance(value, list): + return [s for item in value for s in strings(item)] + return [] + + +def transcript_line(path: Path, line: int) -> dict | None: + try: + with path.open(encoding="utf-8") as handle: + for index, raw in enumerate(handle): + if index == line: + return json.loads(raw) + except (OSError, json.JSONDecodeError): + return None + return None + + +# --- validation -------------------------------------------------------------- + +def source_text(source: dict, run: dict) -> tuple[str | None, str]: + """The text a claim's quote must appear in, and a label for messages.""" + if "line" not in source: + return "\n".join(strings(run)), f"journal entry {run['id']}" + if not isinstance(source["line"], int) or source["line"] < 0: + return None, "source.line must be a non-negative integer" + main = Path(run.get("transcript") or "") + target = Path(source.get("transcript") or main) + allowed = target == main or main.with_suffix("") in target.parents + if not run.get("transcript") or not allowed: + return None, f"source.transcript must be the run's transcript or one of its subagents ({main})" + obj = transcript_line(target, source["line"]) + if obj is None: + return None, f"{target} has no readable line {source['line']}" + return "\n".join(strings(obj.get("message", obj))), f"{target.name}:{source['line']}" + + +def evidence_key(row: dict) -> tuple: + source = row.get("source") or {} + return (source.get("run"), source.get("transcript"), source.get("line"), norm(row.get("quote", "")).lower()) + + +def shape_errors(row: dict) -> list[str]: + errors = [] + extra = set(row) - FIELDS - {"id", "status", "observed_at", "repo", "added_at", "reason"} + if extra: + errors.append(f"unknown fields {sorted(extra)}") + for key in ("skill", "thread"): + if not SLUG.match(str(row.get(key, ""))): + errors.append(f"{key} must be a lowercase slug, got {row.get(key)!r}") + if row.get("kind") not in KINDS: + errors.append(f"kind must be one of {', '.join(KINDS)}") + claim = str(row.get("claim", "")) + if not 12 <= len(claim) <= 240 or "\n" in claim: + errors.append("claim must be one line of 12 to 240 characters") + if EM_DASH in claim: + errors.append("claim contains an em dash; use two plain dashes") + quote = norm(row.get("quote", "")) + if not 8 <= len(quote) <= 300: + errors.append("quote must be 8 to 300 characters of verbatim evidence") + if row.get("salience") not in (1, 2, 3): + errors.append("salience must be 1, 2, or 3") + if not isinstance(row.get("source"), dict) or not row["source"].get("run"): + errors.append("source must be an object naming its run") + if CREDENTIAL.search(claim) or CREDENTIAL.search(quote): + errors.append("claim or quote matches a credential shape; quote around the secret") + return errors + + +def validate_candidates(rows: list[dict], store: Path) -> list[str]: + known = runs(store) + existing = claims(store) + by_id = {c["id"]: c for c in existing} + seen = {evidence_key(c): c for c in existing} + errors: list[str] = [] + batch: set[tuple] = set() + for number, row in enumerate(rows, 1): + problems = shape_errors(row) + source = row.get("source") if isinstance(row.get("source"), dict) else {} + run = known.get(source.get("run", "")) + if source.get("run") and run is None: + problems.append(f"source.run {source['run']!r} is not in the journal") + if run is not None and not problems: + text, label = source_text(source, run) + if text is None: + problems.append(label) + elif norm(row["quote"]).lower() not in norm(text).lower(): + problems.append(f"quote not found verbatim in {label}") + key = evidence_key(row) + if key in seen: + prior = seen[key] + verb = "was retracted as" if prior.get("status") == "retracted" else "duplicates" + problems.append(f"this evidence {verb} {prior['id']}") + if key in batch: + problems.append("duplicates an earlier candidate in this file") + batch.add(key) + target = row.get("supersedes") + if target is not None and (target not in by_id or by_id[target].get("status") != "active"): + problems.append(f"supersedes {target!r}, which is not an active claim") + errors += [f"candidate {number}: {p}" for p in problems] + return errors + + +# --- derived state ----------------------------------------------------------- + +def threads(store: Path) -> list[dict]: + resolutions: dict[str, dict] = {} + for row in read_jsonl(store / "resolutions.jsonl"): + resolutions[row["thread"]] = row + grouped: dict[str, list[dict]] = {} + for claim in claims(store): + if claim.get("status") == "active": + grouped.setdefault(claim["thread"], []).append(claim) + for thread in resolutions: + grouped.setdefault(thread, []) + out = [] + for name, rows in grouped.items(): + resolved = resolutions.get(name) + problems = [c for c in rows if c["kind"] != "success"] + since = [c for c in problems if resolved and c["observed_at"][:19] > resolved["at"][:19]] + if not resolved: + status = "open" if problems else "working" + else: + status = "reopened" if since else "resolved" + live = since if resolved else problems + out.append({ + "thread": name, + "skills": sorted({c["skill"] for c in rows}) or [resolved.get("skill", "?")], + "status": status, + "score": sum(c["salience"] for c in live), + "runs": len({c["source"]["run"] for c in live}), + "repos": len({c.get("repo") for c in live}), + "last_seen": max((c["observed_at"] for c in rows), default=""), + "claims": [c["id"] for c in rows], + "resolution": resolved, + "claims_before": len(problems) - len(since), + "claims_since": len(since), + }) + order = {"reopened": 0, "open": 1, "working": 2, "resolved": 3} + out.sort(key=lambda t: (order[t["status"]], -t["runs"], -t["score"], t["thread"])) + return out + + +def yaml_scalar(value) -> str: + text = str(value) + return text if re.match(r"^[A-Za-z0-9_./:@+-]+$", text) else json.dumps(text) + + +def render_threads(rows: list[dict]) -> str: + lines = [f"# {DERIVED_NOTE}"] + for t in rows: + lines.append(f"- thread: {t['thread']}") + for key in ("status", "score", "runs", "repos", "last_seen"): + lines.append(f" {key}: {yaml_scalar(t[key])}") + lines.append(f" skills: [{', '.join(t['skills'])}]") + lines.append(f" claims: [{', '.join(t['claims'])}]") + if t["resolution"]: + r = t["resolution"] + lines.append(f" resolved: {{commit: {yaml_scalar(r['commit'])}, at: {yaml_scalar(r['at'])}" + + (f", eval: {yaml_scalar(r['eval'])}" if r.get("eval") else "") + "}") + return "\n".join(lines) + "\n" + + +def render_skill(skill: str, rows: list[dict], by_id: dict[str, dict]) -> str: + out = [f"# {skill}", "", DERIVED_NOTE, ""] + sections = (("Open threads", ("reopened", "open")), ("What works", ("working",)), ("Resolved threads", ("resolved",))) + for title, statuses in sections: + picked = [t for t in rows if t["status"] in statuses and skill in t["skills"]] + if not picked: + continue + out += [f"## {title}", ""] + for t in picked: + out.append(f"### {t['thread']} ({t['status']}, score {t['score']}, {t['runs']} runs, {t['repos']} repos)") + out.append("") + for cid in t["claims"]: + c = by_id[cid] + if c["skill"] == skill: + out.append(f"- [{cid}] {c['claim']} ({c['kind']}, salience {c['salience']}, " + f"{c.get('repo', '?')}, {c['observed_at'][:10]})") + out.append("") + return "\n".join(out).rstrip() + "\n" + + +def render_progression(rows: list[dict]) -> str: + out = ["# Progression", "", DERIVED_NOTE, "", + "| thread | resolved at | commit | eval | claims before | claims since | status |", + "| ------ | ----------- | ------ | ---- | ------------- | ------------ | ------ |"] + for t in sorted((t for t in rows if t["resolution"]), key=lambda t: t["resolution"]["at"]): + r = t["resolution"] + out.append(f"| {t['thread']} | {r['at'][:10]} | {r['commit'][:12]} | {r.get('eval') or '-'} | " + f"{t['claims_before']} | {t['claims_since']} | {t['status']} |") + return "\n".join(out) + "\n" + + +def derived(store: Path) -> dict[str, str]: + rows = threads(store) + by_id = {c["id"]: c for c in claims(store)} + files = {"threads.yaml": render_threads(rows), "progression.md": render_progression(rows)} + for skill in sorted({s for t in rows for s in t["skills"] if t["claims"]}): + files[f"skills/{skill}.md"] = render_skill(skill, rows, by_id) + return files + + +def rebuild(store: Path) -> None: + skills_dir = store / "skills" + if skills_dir.is_dir(): + for stale in skills_dir.glob("*.md"): + stale.unlink() + for name, text in derived(store).items(): + path = store / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +# --- commands ---------------------------------------------------------------- + +def cmd_pending(store: Path) -> int: + done = {row["run"] for row in read_jsonl(store / "consolidated.jsonl")} + todo = [run for rid, run in sorted(runs(store).items()) if rid not in done] + print(json.dumps(todo, indent=1)) + print(f"claims: {len(todo)} pending runs", file=sys.stderr) + return 0 + + +def cmd_add(store: Path, candidates: Path) -> int: + rows = [] + for number, line in enumerate(candidates.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError as exc: + print(f"{candidates}:{number}: not JSON ({exc.msg})") + return 1 + errors = validate_candidates(rows, store) + if errors: + print("\n".join(errors)) + print(f"claims: {len(errors)} problems, nothing added; fix the candidates, not the evidence") + return 1 + existing = claims(store) + known = runs(store) + next_id = max((int(c["id"][2:]) for c in existing), default=0) + 1 + superseded = {row["supersedes"] for row in rows if row.get("supersedes")} + for claim in existing: + if claim["id"] in superseded: + claim["status"] = "superseded" + added = [] + for row in rows: + run = known[row["source"]["run"]] + row.update({"id": f"c-{next_id:04d}", "status": "active", "observed_at": run["started_at"], + "repo": run.get("repo"), "added_at": now_iso(), "quote": norm(row["quote"])}) + next_id += 1 + added.append(row) + write_jsonl(store / "claims.jsonl", existing + added) + rebuild(store) + print(f"claims: added {', '.join(r['id'] for r in added) or 'nothing'}" + + (f"; superseded {', '.join(sorted(superseded))}" if superseded else "")) + return 0 + + +def cmd_mark(store: Path, ids: list[str]) -> int: + known = runs(store) + missing = [i for i in ids if i not in known] + if missing: + print(f"claims: not in the journal: {', '.join(missing)}") + return 1 + done = {row["run"] for row in read_jsonl(store / "consolidated.jsonl")} + append_jsonl(store / "consolidated.jsonl", [{"run": i, "at": now_iso()} for i in ids if i not in done]) + print(f"claims: {len(ids)} runs marked consolidated") + return 0 + + +def cmd_check(store: Path) -> int: + rows = claims(store) + errors = [] + ids = [r.get("id") for r in rows] + if len(ids) != len(set(ids)): + errors.append("claims.jsonl: duplicate claim ids") + for row in rows: + label = row.get("id", "?") + if not CLAIM_ID.match(str(label)): + errors.append(f"{label}: malformed id") + if row.get("status") not in ("active", "superseded", "retracted"): + errors.append(f"{label}: unknown status {row.get('status')!r}") + errors += [f"{label}: {p}" for p in shape_errors(row)] + if row.get("supersedes") and row["supersedes"] not in ids: + errors.append(f"{label}: supersedes unknown {row['supersedes']}") + for name, text in derived(store).items(): + path = store / name + if not path.exists() or path.read_text(encoding="utf-8") != text: + errors.append(f"{name}: stale; run `claims.py rebuild`") + print("\n".join(errors) if errors else f"claims: {len(rows)} claims, derived files current") + return 1 if errors else 0 + + +def cmd_top(store: Path, limit: int, everything: bool) -> int: + rows = [t for t in threads(store) if everything or t["status"] in ("open", "reopened")] + if not rows: + print("claims: no open threads") + return 0 + print("| thread | skills | status | score | runs | repos | last seen |") + print("| ------ | ------ | ------ | ----- | ---- | ----- | --------- |") + for t in rows[:limit]: + print(f"| {t['thread']} | {', '.join(t['skills'])} | {t['status']} | {t['score']} | " + f"{t['runs']} | {t['repos']} | {t['last_seen'][:10]} |") + return 0 + + +def cmd_show(store: Path, target: str) -> int: + rows = claims(store) + picked = [c for c in rows if target in (c["id"], c["thread"], c["skill"])] + if not picked: + print(f"claims: nothing matches {target!r}") + return 1 + known = runs(store) + for c in picked: + source = c["source"] + print(f"## {c['id']} [{c['status']}] {c['thread']} ({c['skill']}, {c['kind']}, salience {c['salience']})") + print(f"claim: {c['claim']}") + print(f"quote: \"{c['quote']}\"") + run = known.get(source["run"]) + where = f"{source['run']}" + (f" line {source['line']}" if "line" in source else " journal entry") + print(f"source: {where} ({c.get('repo', '?')}, {c['observed_at']})") + if run is not None and "line" in source: + text, label = source_text(source, run) + if text is not None: + flat = norm(text) + at = flat.lower().find(c["quote"].lower()) + start = max(0, at - 200) if at >= 0 else 0 + print(f"context ({label}): ...{flat[start:start + len(c['quote']) + 400]}...") + else: + print(f"context: unavailable ({label})") + if c.get("reason"): + print(f"retracted because: {c['reason']}") + print() + return 0 + + +def cmd_retract(store: Path, claim_id: str, reason: str) -> int: + rows = claims(store) + hit = next((c for c in rows if c["id"] == claim_id), None) + if hit is None or hit["status"] == "retracted": + print(f"claims: {claim_id} is not a claim that can be retracted") + return 1 + hit["status"] = "retracted" + hit["reason"] = " ".join(reason.split()) + write_jsonl(store / "claims.jsonl", rows) + rebuild(store) + print(f"claims: retracted {claim_id}; derived files rebuilt without it") + return 0 + + +def cmd_resolve(store: Path, thread: str, commit: str, eval_name: str | None) -> int: + if not any(c["thread"] == thread and c["status"] == "active" for c in claims(store)): + print(f"claims: no active claims in thread {thread!r}") + return 1 + if not re.match(r"^[0-9a-f]{7,40}$", commit): + print("claims: --commit must be a commit sha") + return 1 + skill = next(c["skill"] for c in claims(store) if c["thread"] == thread) + append_jsonl(store / "resolutions.jsonl", + [{"thread": thread, "skill": skill, "commit": commit, "eval": eval_name, "at": now_iso()}]) + rebuild(store) + print(f"claims: {thread} resolved by {commit}; a later claim in it reopens it") + return 0 + + +def cmd_report(store: Path, out: Path) -> int: + rows = threads(store) + by_id = {c["id"]: c for c in claims(store)} + cards = [] + for t in rows: + items = "".join( + f"
  • {html.escape(cid)} {html.escape(by_id[cid]['claim'])}" + f"
    {html.escape(by_id[cid]['quote'])}
    " + f"{html.escape(by_id[cid]['kind'])} / salience {by_id[cid]['salience']} / " + f"{html.escape(str(by_id[cid].get('repo')))} / {html.escape(by_id[cid]['source']['run'])}
    • " + for cid in t["claims"]) + resolved = (f"

    resolved by {html.escape(t['resolution']['commit'][:12])}

    " + if t["resolution"] else "") + cards.append( + f"" + f"{t['status']} {html.escape(t['thread'])} " + f"{html.escape(', '.join(t['skills']))} / score {t['score']} / {t['runs']} runs / " + f"{t['repos']} repos{resolved}
      {items}
    ") + page = f""" +Workflow Memory + +

    Workflow memory

    {len(rows)} threads from {len(by_id)} claims, rendered {now_iso()}. +Every claim quotes the run it came from.

    {''.join(cards) or '

    No claims yet.

    '} +""" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(page, encoding="utf-8") + print(f"claims: report written to {out}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("pending") + sub.add_parser("add").add_argument("candidates", type=Path) + sub.add_parser("mark").add_argument("runs", nargs="+") + sub.add_parser("rebuild") + sub.add_parser("check") + top = sub.add_parser("top") + top.add_argument("--limit", type=int, default=10) + top.add_argument("--all", action="store_true") + sub.add_parser("show").add_argument("target") + retract = sub.add_parser("retract") + retract.add_argument("claim") + retract.add_argument("--reason", required=True) + resolve = sub.add_parser("resolve") + resolve.add_argument("thread") + resolve.add_argument("--commit", required=True) + resolve.add_argument("--eval", dest="eval_name") + sub.add_parser("report").add_argument("out", type=Path) + args = parser.parse_args() + store = root() + if args.command == "pending": + return cmd_pending(store) + if args.command == "add": + return cmd_add(store, args.candidates) + if args.command == "mark": + return cmd_mark(store, args.runs) + if args.command == "rebuild": + rebuild(store) + print(f"claims: derived files rebuilt under {store}") + return 0 + if args.command == "check": + return cmd_check(store) + if args.command == "top": + return cmd_top(store, args.limit, args.all) + if args.command == "show": + return cmd_show(store, args.target) + if args.command == "retract": + return cmd_retract(store, args.claim, args.reason) + if args.command == "resolve": + return cmd_resolve(store, args.thread, args.commit, args.eval_name) + return cmd_report(store, args.out) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/dev/scripts/skill-metrics.py b/plugins/dev/scripts/skill-metrics.py index ac78ef0..2301ce3 100755 --- a/plugins/dev/scripts/skill-metrics.py +++ b/plugins/dev/scripts/skill-metrics.py @@ -3,7 +3,7 @@ Usage: python3 skill-metrics.py start {skill} - python3 skill-metrics.py end {skill} [--count key=value ...] + python3 skill-metrics.py end {skill} [--count key=value ...] [--friction text ...] `start` snapshots the git state and locates the session transcript under $CLAUDE_CONFIG_DIR (default ~/.claude), anchoring on the transcript line that @@ -13,6 +13,16 @@ so later runs can be compared against earlier ones. Every value is measured; nothing here is narrated from memory. Missing transcript or git degrade to "n/a" rather than failing the run. + +`end` also appends one entry to the cross-repository run journal under +$DEV_MEMORY_DIR (default ~/.dev-workflow/memory/journal/), which the reflect +skill consolidates into cited claims about the skills themselves. The entry +holds the friction signals measured from the transcript (interrupts, denied or +failed tool calls, the user's own turns, and how often each deterministic +checker ran and failed), each with its transcript path and line, plus at most +three `--friction` lines the skill narrates, kept apart as narrated. Set +DEV_MEMORY_DIR=off, or `"memory": {"enabled": false}` in .dev/config.json, to +skip the journal. """ from __future__ import annotations @@ -31,6 +41,18 @@ TEST_FILE = re.compile(r"(^|/)(tests?|specs?|__tests__)(/|$)|[._-](test|spec)s?\.[A-Za-z]+$|^test_.*\.py$") TOKEN_KEYS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens") +# The deterministic checkers the skills loop against; a failed run of one is loop +# progress, so it is counted per checker instead of logged as a tool error. +CHECKERS = ("lint-spec.py", "check-tests.py", "pr-evidence.py", "architecture-check.py", + "check-agents-md.py", "aggregate-findings.py", "claims.py") +# A checker counts only where a command runs it, never where one greps or edits it. +CHECKER_RUN = re.compile(r"(?:^|[;&|(]\s*|\bpython3?\s+(?:-\S+\s+)*)\S*?(" + "|".join(re.escape(c) for c in CHECKERS) + r")(?=\s|$)", + re.M) +INTERRUPT = "[Request interrupted by user" +DENIED = ("doesn't want to proceed", "was rejected", "Permission denied by user") +SIGNAL_CAP = 20 +EXCERPT = 240 +MAX_FRICTION = 3 def now_iso() -> str: @@ -130,6 +152,145 @@ def subagent_files(transcript: Path) -> list[Path]: return sorted(folder.glob("*.jsonl")) if folder.is_dir() else [] +# --- friction signals -------------------------------------------------------- + +def block_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(block_text(item.get("text", item.get("content", ""))) if isinstance(item, dict) + else str(item) for item in content) + return "" + + +def human_text(obj: dict) -> str | None: + """The text a person typed, or None for tool results and injected turns.""" + if obj.get("isMeta") or obj.get("isSidechain"): + return None + content = (obj.get("message") or {}).get("content") + if isinstance(content, list): + if any(isinstance(b, dict) and b.get("type") == "tool_result" for b in content): + return None + content = block_text([b for b in content if isinstance(b, dict) and b.get("type") == "text"]) + if not isinstance(content, str): + return None + text = content.strip() + if not text or text.startswith("<") or INTERRUPT in text or text.startswith("Base directory for this skill"): + return None + return text + + +def signals(path: Path, start: int, since: float | None, state: dict) -> None: + """Add one transcript's friction signals and checker runs to `state`.""" + pending: dict[str, str] = {} + first = True + for index, obj in iter_lines(path, start): + stamp = obj.get("timestamp") + if first and stamp: + first = False + if since is not None and parse_iso(stamp) < since: + return + message = obj.get("message") or {} + if obj.get("type") == "assistant": + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + command = str((block.get("input") or {}).get("command", "")) + match = CHECKER_RUN.search(command) + checker = match.group(1) if match else None + if checker: + pending[block.get("id", "")] = checker + state["checkers"].setdefault(checker, {"runs": 0, "failed": 0})["runs"] += 1 + continue + if obj.get("type") != "user": + continue + where = {"transcript": str(path), "line": index} + content = message.get("content") + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + text = block_text(block.get("content", block.get("text", ""))) + if block.get("type") == "text" and INTERRUPT in text: + note(state, "interrupt", where, text) + elif block.get("type") == "tool_result" and block.get("is_error"): + checker = pending.get(block.get("tool_use_id", "")) + if checker: + entry = state["checkers"][checker] + entry["failed"] += 1 + entry.setdefault("first_failure", where) + else: + note(state, "denied" if any(d in text for d in DENIED) else "tool_error", where, text) + if isinstance(content, str) and INTERRUPT in content: + note(state, "interrupt", where, content) + elif path == state["main"]: + typed = human_text(obj) + if typed: + note(state, "user_turn", where, typed) + + +def note(state: dict, kind: str, where: dict, text: str) -> None: + state["counts"][kind] = state["counts"].get(kind, 0) + 1 + if state["counts"][kind] <= SIGNAL_CAP: + state["signals"].append({"type": kind, **where, "excerpt": " ".join(text.split())[:EXCERPT]}) + + +def collect_signals(transcript: Path | None, anchor: int, since: float) -> dict: + state: dict = {"main": transcript, "counts": {}, "signals": [], "checkers": {}} + if transcript and transcript.exists(): + # Skip the invocation line itself: it is the skill's own prompt, not a reaction to it. + signals(transcript, anchor + 1, None, state) + for path in subagent_files(transcript): + signals(path, 0, since, state) + state.pop("main") + return state + + +# --- run journal ------------------------------------------------------------- + +def memory_dir() -> Path | None: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + return None + config = repo_root() / ".dev" / "config.json" + try: + if json.loads(config.read_text(encoding="utf-8")).get("memory", {}).get("enabled") is False: + return None + except (OSError, ValueError, AttributeError): + pass + return Path(value).expanduser() + + +def write_journal(record: dict, ledger: Path, ledger_line: int, state: dict, + friction: list[str], found: dict) -> tuple[Path, str] | None: + root = memory_dir() + if root is None: + return None + stamp = re.sub(r"[^0-9T]", "", record["started_at"])[:15] + run_id = f"r-{stamp}-{record['skill']}" + entry = { + "id": run_id, + "skill": record["skill"], + "repo": repo_root().name, + "repo_path": str(repo_root()), + "started_at": record["started_at"], + "ended_at": record["ended_at"], + "seconds": record["seconds"], + "transcript": state.get("transcript"), + "anchor_line": state.get("anchor_line"), + "ledger": str(ledger), + "ledger_line": ledger_line, + "counts": record["counts"], + "signal_counts": found["counts"], + "checkers": found["checkers"], + "signals": found["signals"], + "friction": friction, + } + journal = root / "journal" / f"{record['ended_at'][:10]}.jsonl" + journal.parent.mkdir(parents=True, exist_ok=True) + with journal.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + return journal, run_id + + # --- git --------------------------------------------------------------------- def numstat(ref: str | None) -> tuple[int, int, set[str]]: @@ -219,7 +380,7 @@ def total(tokens: dict) -> int: return sum(tokens.get(key, 0) for key in TOKEN_KEYS) -def cmd_end(skill: str, counts: dict[str, str]) -> int: +def cmd_end(skill: str, counts: dict[str, str], friction: list[str]) -> int: state_file = state_dir() / f"{skill}.json" if not state_file.exists(): print(f"metrics: no start snapshot for {skill}; run `start {skill}` at invocation", file=sys.stderr) @@ -240,6 +401,7 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for agent in agents: tools.update(agent["tools"]) delta = git_delta(state["git"]) + found = collect_signals(transcript, state["anchor_line"], since) record = { "skill": skill, @@ -255,6 +417,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: "tokens_total": total(all_tokens), "git": delta, "counts": counts, + "signal_counts": found["counts"], + "checkers": found["checkers"], } ledger = repo_root() / ".dev" / "metrics.jsonl" previous = [] @@ -267,8 +431,10 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: if row.get("skill") == skill: previous.append(row) ledger.parent.mkdir(parents=True, exist_ok=True) + ledger_line = sum(1 for _ in ledger.open(encoding="utf-8")) + 1 if ledger.exists() else 1 with ledger.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") + journaled = write_journal(record, ledger, ledger_line, state, friction, found) top_tools = ", ".join(f"{name} {n}" for name, n in tools.most_common(4)) or "none" rows = [ @@ -290,8 +456,11 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: rows.append(("git since start", "n/a (not a git repo)")) for key, value in counts.items(): rows.append((key.replace("_", " "), value)) + measured = [f"{n} {kind.replace('_', ' ')}" for kind, n in sorted(found["counts"].items())] + measured += [f"{name} {c['runs']} runs / {c['failed']} failed" for name, c in sorted(found["checkers"].items())] + rows.append(("friction signals", ", ".join(measured) or "none measured")) if previous and main: - med_tokens = statistics.median(r["tokens_total"] for r in previous if r.get("tokens_total")) + med_tokens = statistics.median([r["tokens_total"] for r in previous if r.get("tokens_total")] or [0]) med_secs = statistics.median(r["seconds"] for r in previous) change = (total(all_tokens) - med_tokens) / med_tokens * 100 if med_tokens else 0 rows.append(( @@ -309,6 +478,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for name, value in rows: print(f"| {name.ljust(width)} | {value} |") print(f"\nledger: {ledger}") + if journaled: + print(f"journal: {journaled[0]} ({journaled[1]}, {len(friction)} narrated friction lines)") return 0 @@ -320,6 +491,8 @@ def main() -> int: end.add_argument("skill") end.add_argument("--count", action="append", default=[], metavar="KEY=VALUE", help="skill-specific measured counter to include, repeatable") + end.add_argument("--friction", action="append", default=[], metavar="TEXT", + help=f"where this run fought the skill's own instructions, at most {MAX_FRICTION}") args = parser.parse_args() if args.command == "start": return cmd_start(args.skill) @@ -329,7 +502,10 @@ def main() -> int: parser.error(f"--count expects KEY=VALUE, got {item!r}") key, value = item.split("=", 1) counts[key.strip()] = value.strip() - return cmd_end(args.skill, counts) + friction = [" ".join(line.split()) for line in args.friction if line.strip()] + if len(friction) > MAX_FRICTION: + parser.error(f"--friction takes at most {MAX_FRICTION} lines, got {len(friction)}; keep the sharpest") + return cmd_end(args.skill, counts, friction) if __name__ == "__main__": diff --git a/plugins/dev/skills/build/SKILL.md b/plugins/dev/skills/build/SKILL.md index b324527..1653089 100644 --- a/plugins/dev/skills/build/SKILL.md +++ b/plugins/dev/skills/build/SKILL.md @@ -44,6 +44,8 @@ Every build run first prints the measured run metrics, pasting the table verbati python3 {build-skill-root}/../../scripts/skill-metrics.py end build --count change_sets=N --count scenarios=N --count e2e_passed=N --count e2e_failed=N ``` +Add any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md). + Then it ends with the same two lines, in this order - a green run, a blocked gate, and a run with open deviations all get both: 1. `Next step: run ship over this work, pointed at .dev/{plan-name}/implementation-notes.md and the {plan-name}-e2e-report.html.` Recommend it; never launch it yourself. diff --git a/plugins/dev/skills/reflect/SKILL.md b/plugins/dev/skills/reflect/SKILL.md new file mode 100644 index 0000000..e809e4b --- /dev/null +++ b/plugins/dev/skills/reflect/SKILL.md @@ -0,0 +1,66 @@ +--- +name: reflect +description: Consolidate the run journal that scope, scope-review, build, and ship write at the end of every run into cited claims about how the skills themselves behaved, rank the recurring friction threads across repositories, and hand off one chosen thread as a scope brief with an eval case from the real run. Use periodically to decide what to improve in the dev workflow next, or with "resolve {thread} {sha}" once a fix for a thread has merged. +--- + +# Reflect + +Turn what past runs measured into evidence about the skills, so the next skill change fixes something that actually recurred. +The store is `~/.dev-workflow/memory/` (or `$DEV_MEMORY_DIR`), local to this machine and shared by every repository; [references/claims.md](references/claims.md) owns its records, the claim schema, and what makes a claim worth keeping. + +`{reflect-skill-root}` is the directory holding this SKILL.md, `{claims}` is `python3 {reflect-skill-root}/../../scripts/claims.py`, and `{project-slug}` is the basename of the current repository root, or `workflow-memory` outside one. + +Limits: +- Never edit a skill, a reference, or an eval. The output is a brief a person takes to `scope`; adaptation reaches a skill only through a reviewed commit. +- Never write `claims.jsonl`, `resolutions.jsonl`, or a derived file by hand; every change goes through `{claims}`, so the checks hold. +- A claim is about how a skill behaved, never about the consuming repository's code or its people. +- Nothing is committed, pushed, or published unless the user asks. + +## Resolve mode + +When invoked as `resolve {thread} {sha} [eval-id]`, run `{claims} resolve {thread} --commit {sha} [--eval {eval-id}]`, print `{claims} top --all`, and stop. +A later claim in that thread reopens it, which is how a fix that did not hold shows up. + +## 1. Gather + +Create `/tmp/{project-slug}/reflect/`, then run `{claims} pending > /tmp/{project-slug}/reflect/pending.json` and `{claims} top --all`. +With no pending runs, skip to step 4; with no pending runs and no claims at all, say the journal is empty, name the four skills that write it, and stop. + +## 2. Extract + +Launch one read-only subagent per batch of up to five pending runs, all in one message. +Hand each the batch's journal entries, the current thread list from `{claims} top --all`, and [references/claims.md](references/claims.md), and have it write candidate claims, one JSON object per line, to `/tmp/{project-slug}/reflect/candidates-{n}.jsonl`. +Each agent reads the transcript only around the lines its signals cite, never a whole transcript, and reuses an existing thread slug whenever the claim is the same problem. +An agent may write zero candidates; a run that went as designed is not friction. + +## 3. Check loop + +Concatenate the candidate files into `/tmp/{project-slug}/reflect/candidates.jsonl` and loop until it exits 0: + +```bash +{claims} add /tmp/{project-slug}/reflect/candidates.jsonl +``` + +It adds nothing while any candidate fails, and each error names the candidate. +Fix that candidate's quote, source, or shape against the transcript, or drop it; never bend a quote toward a claim the evidence does not make. +Then run `{claims} mark` with every pending run id, including runs that produced no claims. + +## 4. Present + +Run `{claims} report /tmp/{project-slug}/reports/workflow-memory.html` and open it per [../../references/reporting.md](../../references/reporting.md). +Show `{claims} top`, then `{claims} show {thread}` for the top three open or reopened threads, condensed to each claim and its quote. +Ask one question with the host's structured user-input tool when available: which thread to pursue now, a claim that misreads its evidence and should be retracted, or none. +Recommend the thread with the most runs across the most repositories; a reopened thread outranks an open one with the same reach. + +A claim the user says misreads its evidence is retracted with `{claims} retract {id} --reason "{their words}"`, and the question is asked again. + +## 5. Brief + +For the chosen thread, write `/tmp/{project-slug}/reflect/brief-{thread}.md` per [references/brief.md](references/brief.md). +Read the implicated skill's own text at `{reflect-skill-root}/../{skill}/` to name the lines the evidence points at. +Prefer a direction that adds or tightens a deterministic check the skill loops against over one that adds prose, per the repository rule that long rule lists become guidelines. + +## 6. Close + +Print the brief's path and its one-line problem statement. +Recommend the next step: run `/dev:scope` in the workflow repository with the brief, then `/dev:build`; once the fix merges, `/dev:reflect resolve {thread} {sha} {eval-id}`. diff --git a/plugins/dev/skills/reflect/agents/openai.yaml b/plugins/dev/skills/reflect/agents/openai.yaml new file mode 100644 index 0000000..19c3d97 --- /dev/null +++ b/plugins/dev/skills/reflect/agents/openai.yaml @@ -0,0 +1,6 @@ +interface: + display_name: "Reflect" + short_description: "Turn run friction into cited skill fixes" + default_prompt: "Use $dev:reflect to consolidate the run journal into cited claims and pick the next skill improvement." +policy: + allow_implicit_invocation: false diff --git a/plugins/dev/skills/reflect/references/brief.md b/plugins/dev/skills/reflect/references/brief.md new file mode 100644 index 0000000..2f66da8 --- /dev/null +++ b/plugins/dev/skills/reflect/references/brief.md @@ -0,0 +1,40 @@ +# Brief + +The handoff from `reflect` to `scope` for one thread. +It is self-contained: a person or a fresh session reading only this file can start `scope` in the workflow repository. + +```markdown +# {thread} + +Problem: {one sentence, in terms of what the runs did, not what the skill text says} + +## Evidence + +{claims show output condensed to one bullet per claim:} +- [c-0012] {claim} - "{quote}" ({repo}, {run id}, line {line}) + +Reach: {runs} runs across {repos} repositories, status {open|reopened}; {for a reopened thread, the commit that was meant to fix it}. + +## Where the skill text is implicated + +- `dev/skills/{skill}/SKILL.md` step {n}: "{the instruction the evidence contradicts or that let it happen}" + +## Direction + +{the smallest change that would have prevented the evidence; prefer a deterministic check the skill loops against over another line of prose, and say which one} + +Not in scope: {what the evidence does not support changing} + +## Eval case + +Append to `dev/evals/{skill}.json`, built from the real run so the regression is the one that happened: + +{"id": "{skill}-{thread-suffix}", "prompt": "...", "assertions": ["..."]} + +## After the fix merges + +/dev:reflect resolve {thread} {sha} {eval id} +``` + +Keep quotes as they are in the store; trim around them, never inside them. +When the evidence points at two skills, write one brief per skill and say so in each. diff --git a/plugins/dev/skills/reflect/references/claims.md b/plugins/dev/skills/reflect/references/claims.md new file mode 100644 index 0000000..1d2fe1e --- /dev/null +++ b/plugins/dev/skills/reflect/references/claims.md @@ -0,0 +1,57 @@ +# Claims + +The memory store, the claim schema, and the judgment `claims.py` cannot make for you. + +## The store + +`$DEV_MEMORY_DIR`, default `~/.dev-workflow/memory/`: + +| path | written by | role | +| ---- | ---------- | ---- | +| `journal/YYYY-MM-DD.jsonl` | `skill-metrics.py end` | one entry per run: measured signals, checker runs, narrated friction | +| `claims.jsonl` | `claims.py add`, `retract` | every claim ever accepted, with its status | +| `consolidated.jsonl` | `claims.py mark` | runs already read | +| `resolutions.jsonl` | `claims.py resolve` | thread, fixing commit, eval id, time | +| `skills/{skill}.md`, `threads.yaml`, `progression.md` | `claims.py rebuild` | derived; every line cites a claim id | + +Records are appended or change status; they are never deleted. +A retracted claim stays so the same evidence cannot quietly bring it back. + +## A journal entry + +The fields an extracting agent reads: + +- `id`, `skill`, `repo`, `started_at`, `transcript`, `anchor_line`. +- `signals`: up to 20 per type, each `{type, transcript, line, excerpt}`. Types: `interrupt` (the user stopped the run), `denied` (the user refused a tool call), `tool_error` (a non-checker tool failed), `user_turn` (anything the user typed after invocation). +- `checkers`: per checker script, `runs`, `failed`, and `first_failure` with its line. +- `counts`: the skill's own measured counters, `signal_counts`: totals before the cap. +- `friction`: up to three lines the skill narrated about itself. Narrated, not measured: a claim resting on one alone takes salience 1 unless a measured signal in the same run agrees. + +## Candidate shape + +One JSON object per line: + +```json +{"skill": "build", "thread": "build-validation-reruns", "kind": "friction", + "claim": "Build reran the full Validation block after each change set; the user stopped it.", + "quote": "stop rerunning the whole validation block", + "source": {"run": "r-20261005T100000-build", "line": 412}, + "salience": 2} +``` + +- `skill`: the skill whose instructions the claim is about, which need not be the run's skill (a build run can expose a scope defect). +- `thread`: `{skill}-{problem}` in lowercase slug form; reuse an existing slug for the same problem. +- `kind`: `friction` (the skill made the run slower or the user intervene), `defect` (the skill's instructions were wrong or contradictory), `cost` (a measured cost far above that skill's median), `success` (a skill rule that demonstrably saved the run; keep these, they say what not to remove). +- `claim`: one sentence, at most 240 characters, no em dash. +- `quote`: verbatim text, 8 to 300 characters, found at the source. Whitespace and case are ignored, nothing else is. +- `source`: `run` plus `line` (a line of the run's transcript, or with `transcript`, of one of its subagent transcripts); omit `line` to quote the journal entry itself. +- `salience`: 1 noted once, 2 cost real time or a user intervention, 3 broke the run or produced a wrong result. +- `supersedes`: optional id of an active claim this one replaces with better evidence or a sharper reading. + +## What is not a claim + +- A `user_turn` that answers a question the skill was designed to ask, such as a scope interview answer. +- A checker that failed and passed on the next run: that is the loop working. +- A `denied` call that reflects the user's standing policy rather than a mistake the skill made. +- Anything about the consuming repository's code quality, its tests, or its people. +- A guess about why without a quote that shows it; write the observable claim instead. diff --git a/plugins/dev/skills/scope-review/SKILL.md b/plugins/dev/skills/scope-review/SKILL.md index 55c2078..2e920e9 100644 --- a/plugins/dev/skills/scope-review/SKILL.md +++ b/plugins/dev/skills/scope-review/SKILL.md @@ -128,7 +128,7 @@ Asked: {the question} - Answered: {the user's decision} -> {what the spec says n ## Wrap up -Open the chat summary with the table from `python3 {scope-review-skill-root}/../../scripts/skill-metrics.py end scope-review --count findings_verified=N --count findings_refuted=N --count refinements_applied=N --count escalated=N`, pasted verbatim. +Open the chat summary with the table from `python3 {scope-review-skill-root}/../../scripts/skill-metrics.py end scope-review --count findings_verified=N --count findings_refuted=N --count refinements_applied=N --count escalated=N` plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md), pasted verbatim. Then summarize in the same message: the verdict, what was refined and what the user's answers changed (so the loop's edits stay auditable after the fact), what was promoted to `docs/decisions.md` and `docs/contracts.md`, anything deferred with its open question, and a link to the report. Recommend next steps, never invoking them: diff --git a/plugins/dev/skills/scope/SKILL.md b/plugins/dev/skills/scope/SKILL.md index c35f513..c32c477 100644 --- a/plugins/dev/skills/scope/SKILL.md +++ b/plugins/dev/skills/scope/SKILL.md @@ -148,7 +148,7 @@ Audit the run itself. Three checks, each reported as a typed line - silence read - **Sizing** - `held | mis-sized`. Did the small/full call survive? Name the evidence when it didn't. - **Catalog gaps** - `none | gap`. What did the reviewer or blind-spot pass find that phase 2 should have caught? A missed *category* is a proposed edit to the phase 2 list - propose it to the user, never apply it silently. -Then print the measured run metrics with `python3 {scope-skill-root}/../../scripts/skill-metrics.py end scope --count decisions=N --count change_sets=N --count scenarios=N`, pasting its table verbatim. +Then print the measured run metrics with `python3 {scope-skill-root}/../../scripts/skill-metrics.py end scope --count decisions=N --count change_sets=N --count scenarios=N`, plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md), pasting its table verbatim. Then recommend next steps, never launching them: `scope-review` first when the change is large or risky or build will run in a different session - it reviews the spec with a fresh-context panel and refines it in place before any code exists - and `build` to implement. ## Jira sync diff --git a/plugins/dev/skills/ship/SKILL.md b/plugins/dev/skills/ship/SKILL.md index df82cc2..94596d5 100644 --- a/plugins/dev/skills/ship/SKILL.md +++ b/plugins/dev/skills/ship/SKILL.md @@ -134,7 +134,7 @@ Summarize whichever phases ran in one chat message, opening with the measured ru python3 {ship-skill-root}/../../scripts/skill-metrics.py end ship --count violations_found=N --count violations_fixed=N --count violations_surviving=N --count findings_verified=N --count findings_refuted=N --count remediation_rounds=N --count blockers_cleared=N --count evidence_items=N ``` -Pass only counters you tallied from tool output and the aggregate script; the table it prints (time, tokens, agents, tool calls, git delta, trend against earlier runs) is pasted verbatim, never retyped. +Pass only counters you tallied from tool output and the aggregate script, plus any `--friction` lines per [../../references/run-journal.md](../../references/run-journal.md); the table it prints (time, tokens, agents, tool calls, git delta, trend against earlier runs) is pasted verbatim, never retyped. For the gauntlet, per tool: violations found, fixed, and surviving (with the human call each is waiting on); name the tools acquired or built this run and where they live; state the scope honestly - "hardened the diff" is not "hardened the repo". For the review: the final verdict and top findings, linking every `review_N.md` this run wrote, the local HTML report, and the published URL when one was requested and created; per remediation round, which blockers cleared and which survived. For the pull request: its URL, draft or ready, what the Evidence section shows and where it came from, and the state of its required checks. diff --git a/plugins/factory/references/run-journal.md b/plugins/factory/references/run-journal.md new file mode 100644 index 0000000..0eee3dd --- /dev/null +++ b/plugins/factory/references/run-journal.md @@ -0,0 +1,12 @@ +# Run Journal + +`skill-metrics.py end` appends one entry per run to the cross-repository journal under `~/.dev-workflow/memory/journal/` (or `$DEV_MEMORY_DIR`), which the `reflect` skill consolidates into cited claims about the skills themselves. +The script measures the friction signals on its own from the transcript: interrupts, denied and failed tool calls, the user's own turns, and how often each checker ran and failed. + +The one thing it cannot measure is where the run fought the skill's own instructions, so the skill adds that, as up to three `--friction "..."` arguments on the `end` call: + +- One line each: the step, what the instruction asked, and what the run had to do instead, quoting the moment when there is one. +- About the skill's text, never the repository's code: "step 3 assumes one Validation block; this repo has two, so the wave gate ran neither" is friction, "the tests are slow" is not. +- None when the run went as the skill describes; an empty journal entry is the common and good case. + +Set `DEV_MEMORY_DIR=off`, or `"memory": {"enabled": false}` in `.dev/config.json`, to keep a repository's runs out of the journal. diff --git a/plugins/factory/scripts/skill-metrics.py b/plugins/factory/scripts/skill-metrics.py index ac78ef0..2301ce3 100755 --- a/plugins/factory/scripts/skill-metrics.py +++ b/plugins/factory/scripts/skill-metrics.py @@ -3,7 +3,7 @@ Usage: python3 skill-metrics.py start {skill} - python3 skill-metrics.py end {skill} [--count key=value ...] + python3 skill-metrics.py end {skill} [--count key=value ...] [--friction text ...] `start` snapshots the git state and locates the session transcript under $CLAUDE_CONFIG_DIR (default ~/.claude), anchoring on the transcript line that @@ -13,6 +13,16 @@ so later runs can be compared against earlier ones. Every value is measured; nothing here is narrated from memory. Missing transcript or git degrade to "n/a" rather than failing the run. + +`end` also appends one entry to the cross-repository run journal under +$DEV_MEMORY_DIR (default ~/.dev-workflow/memory/journal/), which the reflect +skill consolidates into cited claims about the skills themselves. The entry +holds the friction signals measured from the transcript (interrupts, denied or +failed tool calls, the user's own turns, and how often each deterministic +checker ran and failed), each with its transcript path and line, plus at most +three `--friction` lines the skill narrates, kept apart as narrated. Set +DEV_MEMORY_DIR=off, or `"memory": {"enabled": false}` in .dev/config.json, to +skip the journal. """ from __future__ import annotations @@ -31,6 +41,18 @@ TEST_FILE = re.compile(r"(^|/)(tests?|specs?|__tests__)(/|$)|[._-](test|spec)s?\.[A-Za-z]+$|^test_.*\.py$") TOKEN_KEYS = ("input_tokens", "cache_read_input_tokens", "cache_creation_input_tokens", "output_tokens") +# The deterministic checkers the skills loop against; a failed run of one is loop +# progress, so it is counted per checker instead of logged as a tool error. +CHECKERS = ("lint-spec.py", "check-tests.py", "pr-evidence.py", "architecture-check.py", + "check-agents-md.py", "aggregate-findings.py", "claims.py") +# A checker counts only where a command runs it, never where one greps or edits it. +CHECKER_RUN = re.compile(r"(?:^|[;&|(]\s*|\bpython3?\s+(?:-\S+\s+)*)\S*?(" + "|".join(re.escape(c) for c in CHECKERS) + r")(?=\s|$)", + re.M) +INTERRUPT = "[Request interrupted by user" +DENIED = ("doesn't want to proceed", "was rejected", "Permission denied by user") +SIGNAL_CAP = 20 +EXCERPT = 240 +MAX_FRICTION = 3 def now_iso() -> str: @@ -130,6 +152,145 @@ def subagent_files(transcript: Path) -> list[Path]: return sorted(folder.glob("*.jsonl")) if folder.is_dir() else [] +# --- friction signals -------------------------------------------------------- + +def block_text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join(block_text(item.get("text", item.get("content", ""))) if isinstance(item, dict) + else str(item) for item in content) + return "" + + +def human_text(obj: dict) -> str | None: + """The text a person typed, or None for tool results and injected turns.""" + if obj.get("isMeta") or obj.get("isSidechain"): + return None + content = (obj.get("message") or {}).get("content") + if isinstance(content, list): + if any(isinstance(b, dict) and b.get("type") == "tool_result" for b in content): + return None + content = block_text([b for b in content if isinstance(b, dict) and b.get("type") == "text"]) + if not isinstance(content, str): + return None + text = content.strip() + if not text or text.startswith("<") or INTERRUPT in text or text.startswith("Base directory for this skill"): + return None + return text + + +def signals(path: Path, start: int, since: float | None, state: dict) -> None: + """Add one transcript's friction signals and checker runs to `state`.""" + pending: dict[str, str] = {} + first = True + for index, obj in iter_lines(path, start): + stamp = obj.get("timestamp") + if first and stamp: + first = False + if since is not None and parse_iso(stamp) < since: + return + message = obj.get("message") or {} + if obj.get("type") == "assistant": + for block in message.get("content") or []: + if isinstance(block, dict) and block.get("type") == "tool_use": + command = str((block.get("input") or {}).get("command", "")) + match = CHECKER_RUN.search(command) + checker = match.group(1) if match else None + if checker: + pending[block.get("id", "")] = checker + state["checkers"].setdefault(checker, {"runs": 0, "failed": 0})["runs"] += 1 + continue + if obj.get("type") != "user": + continue + where = {"transcript": str(path), "line": index} + content = message.get("content") + for block in content if isinstance(content, list) else []: + if not isinstance(block, dict): + continue + text = block_text(block.get("content", block.get("text", ""))) + if block.get("type") == "text" and INTERRUPT in text: + note(state, "interrupt", where, text) + elif block.get("type") == "tool_result" and block.get("is_error"): + checker = pending.get(block.get("tool_use_id", "")) + if checker: + entry = state["checkers"][checker] + entry["failed"] += 1 + entry.setdefault("first_failure", where) + else: + note(state, "denied" if any(d in text for d in DENIED) else "tool_error", where, text) + if isinstance(content, str) and INTERRUPT in content: + note(state, "interrupt", where, content) + elif path == state["main"]: + typed = human_text(obj) + if typed: + note(state, "user_turn", where, typed) + + +def note(state: dict, kind: str, where: dict, text: str) -> None: + state["counts"][kind] = state["counts"].get(kind, 0) + 1 + if state["counts"][kind] <= SIGNAL_CAP: + state["signals"].append({"type": kind, **where, "excerpt": " ".join(text.split())[:EXCERPT]}) + + +def collect_signals(transcript: Path | None, anchor: int, since: float) -> dict: + state: dict = {"main": transcript, "counts": {}, "signals": [], "checkers": {}} + if transcript and transcript.exists(): + # Skip the invocation line itself: it is the skill's own prompt, not a reaction to it. + signals(transcript, anchor + 1, None, state) + for path in subagent_files(transcript): + signals(path, 0, since, state) + state.pop("main") + return state + + +# --- run journal ------------------------------------------------------------- + +def memory_dir() -> Path | None: + value = os.environ.get("DEV_MEMORY_DIR", "~/.dev-workflow/memory") + if value.strip().lower() in ("off", "0", "false", ""): + return None + config = repo_root() / ".dev" / "config.json" + try: + if json.loads(config.read_text(encoding="utf-8")).get("memory", {}).get("enabled") is False: + return None + except (OSError, ValueError, AttributeError): + pass + return Path(value).expanduser() + + +def write_journal(record: dict, ledger: Path, ledger_line: int, state: dict, + friction: list[str], found: dict) -> tuple[Path, str] | None: + root = memory_dir() + if root is None: + return None + stamp = re.sub(r"[^0-9T]", "", record["started_at"])[:15] + run_id = f"r-{stamp}-{record['skill']}" + entry = { + "id": run_id, + "skill": record["skill"], + "repo": repo_root().name, + "repo_path": str(repo_root()), + "started_at": record["started_at"], + "ended_at": record["ended_at"], + "seconds": record["seconds"], + "transcript": state.get("transcript"), + "anchor_line": state.get("anchor_line"), + "ledger": str(ledger), + "ledger_line": ledger_line, + "counts": record["counts"], + "signal_counts": found["counts"], + "checkers": found["checkers"], + "signals": found["signals"], + "friction": friction, + } + journal = root / "journal" / f"{record['ended_at'][:10]}.jsonl" + journal.parent.mkdir(parents=True, exist_ok=True) + with journal.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry) + "\n") + return journal, run_id + + # --- git --------------------------------------------------------------------- def numstat(ref: str | None) -> tuple[int, int, set[str]]: @@ -219,7 +380,7 @@ def total(tokens: dict) -> int: return sum(tokens.get(key, 0) for key in TOKEN_KEYS) -def cmd_end(skill: str, counts: dict[str, str]) -> int: +def cmd_end(skill: str, counts: dict[str, str], friction: list[str]) -> int: state_file = state_dir() / f"{skill}.json" if not state_file.exists(): print(f"metrics: no start snapshot for {skill}; run `start {skill}` at invocation", file=sys.stderr) @@ -240,6 +401,7 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for agent in agents: tools.update(agent["tools"]) delta = git_delta(state["git"]) + found = collect_signals(transcript, state["anchor_line"], since) record = { "skill": skill, @@ -255,6 +417,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: "tokens_total": total(all_tokens), "git": delta, "counts": counts, + "signal_counts": found["counts"], + "checkers": found["checkers"], } ledger = repo_root() / ".dev" / "metrics.jsonl" previous = [] @@ -267,8 +431,10 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: if row.get("skill") == skill: previous.append(row) ledger.parent.mkdir(parents=True, exist_ok=True) + ledger_line = sum(1 for _ in ledger.open(encoding="utf-8")) + 1 if ledger.exists() else 1 with ledger.open("a", encoding="utf-8") as handle: handle.write(json.dumps(record) + "\n") + journaled = write_journal(record, ledger, ledger_line, state, friction, found) top_tools = ", ".join(f"{name} {n}" for name, n in tools.most_common(4)) or "none" rows = [ @@ -290,8 +456,11 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: rows.append(("git since start", "n/a (not a git repo)")) for key, value in counts.items(): rows.append((key.replace("_", " "), value)) + measured = [f"{n} {kind.replace('_', ' ')}" for kind, n in sorted(found["counts"].items())] + measured += [f"{name} {c['runs']} runs / {c['failed']} failed" for name, c in sorted(found["checkers"].items())] + rows.append(("friction signals", ", ".join(measured) or "none measured")) if previous and main: - med_tokens = statistics.median(r["tokens_total"] for r in previous if r.get("tokens_total")) + med_tokens = statistics.median([r["tokens_total"] for r in previous if r.get("tokens_total")] or [0]) med_secs = statistics.median(r["seconds"] for r in previous) change = (total(all_tokens) - med_tokens) / med_tokens * 100 if med_tokens else 0 rows.append(( @@ -309,6 +478,8 @@ def cmd_end(skill: str, counts: dict[str, str]) -> int: for name, value in rows: print(f"| {name.ljust(width)} | {value} |") print(f"\nledger: {ledger}") + if journaled: + print(f"journal: {journaled[0]} ({journaled[1]}, {len(friction)} narrated friction lines)") return 0 @@ -320,6 +491,8 @@ def main() -> int: end.add_argument("skill") end.add_argument("--count", action="append", default=[], metavar="KEY=VALUE", help="skill-specific measured counter to include, repeatable") + end.add_argument("--friction", action="append", default=[], metavar="TEXT", + help=f"where this run fought the skill's own instructions, at most {MAX_FRICTION}") args = parser.parse_args() if args.command == "start": return cmd_start(args.skill) @@ -329,7 +502,10 @@ def main() -> int: parser.error(f"--count expects KEY=VALUE, got {item!r}") key, value = item.split("=", 1) counts[key.strip()] = value.strip() - return cmd_end(args.skill, counts) + friction = [" ".join(line.split()) for line in args.friction if line.strip()] + if len(friction) > MAX_FRICTION: + parser.error(f"--friction takes at most {MAX_FRICTION} lines, got {len(friction)}; keep the sharpest") + return cmd_end(args.skill, counts, friction) if __name__ == "__main__": diff --git a/scripts/build_codex_plugin.py b/scripts/build_codex_plugin.py index e3560d5..ac16205 100644 --- a/scripts/build_codex_plugin.py +++ b/scripts/build_codex_plugin.py @@ -71,6 +71,11 @@ def destination(self) -> Path: "Harden, review, then open a PR with proof", "Use $dev:ship to run the quality gauntlet, the verified review, and open the pull request with evidence for this change.", ), + "reflect": ( + "Reflect", + "Turn run friction into cited skill fixes", + "Use $dev:reflect to consolidate the run journal into cited claims and pick the next skill improvement.", + ), "to-pitch": ( "To Pitch", "Turn finished work into a buy-in document", From ee3bad28f7cfe361f7b7fb8c143c86932be7e784 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:22:22 +0000 Subject: [PATCH 3/3] fix(pi): ship dev/scripts in the Pi package What: add dev/scripts to package.json's files, and extend validate.sh's P01 check so every plugin script a dev skill references as ../../scripts/X must exist and sit under a shipped path. Why: the Pi package shipped dev/skills and dev/references only, so on Pi the skills' calls to impact-scope.py, skill-metrics.py, architecture-check.py, and claims.py pointed at files that were never installed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CJNyTNFr77RZj8omURZxyA --- package.json | 1 + scripts/validate.sh | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 148b72b..c927a06 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "files": [ "dev/skills", "dev/references", + "dev/scripts", "README.md" ], "pi": { diff --git a/scripts/validate.sh b/scripts/validate.sh index 2b1dfa1..8d7484c 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -580,12 +580,26 @@ check_pi() { .name == "@tobrun/dev-workflow" and (.version | test("^[0-9]+\\.[0-9]+\\.[0-9]+([+-][0-9A-Za-z.-]+)?$")) and (.keywords | index("pi-package")) and - .files == ["dev/skills", "dev/references", "README.md"] and + .files == ["dev/skills", "dev/references", "dev/scripts", "README.md"] and .pi.skills == ["./dev/skills"] ' "$package" >/dev/null 2>&1; then fail "P01" "$package" "Invalid Pi package manifest" fi + # Skills reach the plugin's shared scripts as {skill-root}/../../scripts/X; + # each one must exist and sit under a path the package ships. + local script + while IFS= read -r script; do + if [ ! -f "dev/$script" ]; then + fail "P01" "dev/$script" "Referenced by a dev skill but missing" + elif ! jq -e --arg path "dev/$script" \ + 'any(.files[]; . as $f | $path | startswith($f + "/"))' \ + "$package" >/dev/null 2>&1; then + fail "P01" "$package" "dev/$script is referenced by a dev skill but not shipped in files" + fi + done < <(grep -rhoE '\.\./\.\./scripts/[A-Za-z0-9_.-]+' dev/skills dev/references \ + | sed 's#^\.\./\.\./##' | sort -u) + package_version=$(jq -r '.version // empty' "$package" 2>/dev/null || true) plugin_version=$(jq -r '.version // empty' \ dev/.claude-plugin/plugin.json 2>/dev/null || true)