From 91b358e8e4c4ce034ce9d3b329c9edac9d4d1a14 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:13:36 -0500 Subject: [PATCH 1/3] feat(testing): name surviving mutants as a weak-test signal and isolate parallel workers test-value now names a surviving mutant on changed code as one signal that a test checks too little, and points to /mutation-testing:audit for getting the survivors. write's determinism controls replace the one-line parallel-safety rule with worker isolation: a database or schema per worker, port 0 instead of a fixed port, a temp directory per test, and no shared file or environment variable mutated across workers, with pointers to the worker-id docs of pytest-xdist, Jest and Playwright. Adds one eval case for each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MWmASPFjTu4anSp6BowqVC --- .../tv-surviving-mutant-signal/case.yaml | 22 +++++++++++++++++++ .../samples/names-mutation-audit.json | 14 ++++++++++++ .../samples/skill-fired.json | 15 +++++++++++++ .../wr-parallel-worker-isolation/case.yaml | 22 +++++++++++++++++++ .../samples/skill-fired.json | 15 +++++++++++++ .../samples/uses-port-zero.json | 18 +++++++++++++++ plugins/testing/skills/test-value/SKILL.md | 6 +++++ plugins/testing/skills/write/context/write.md | 2 +- 8 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 plugins/testing/evals/tv-surviving-mutant-signal/case.yaml create mode 100644 plugins/testing/evals/tv-surviving-mutant-signal/samples/names-mutation-audit.json create mode 100644 plugins/testing/evals/tv-surviving-mutant-signal/samples/skill-fired.json create mode 100644 plugins/testing/evals/wr-parallel-worker-isolation/case.yaml create mode 100644 plugins/testing/evals/wr-parallel-worker-isolation/samples/skill-fired.json create mode 100644 plugins/testing/evals/wr-parallel-worker-isolation/samples/uses-port-zero.json diff --git a/plugins/testing/evals/tv-surviving-mutant-signal/case.yaml b/plugins/testing/evals/tv-surviving-mutant-signal/case.yaml new file mode 100644 index 00000000000..024536c075d --- /dev/null +++ b/plugins/testing/evals/tv-surviving-mutant-signal/case.yaml @@ -0,0 +1,22 @@ +schema_version: "1.1" +name: tv-surviving-mutant-signal +description: 'A change to applyDiscount has full line coverage, and the user asks how to tell whether its tests check enough. test-value names a surviving mutant on the changed code as a signal that a test checks too little and points to /mutation-testing:audit for getting the survivors. The prompt names the skill (test-value is not user-invocable). skill-fired reads the Skill calls, names-mutation-audit the reply; meets-expectations (llm) judges the reply. Read-only tools, graded on the reply.' +tags: [testing, test-value, mutation, no-network] +expected_outcome: 'Says coverage shows only that lines ran, names a surviving mutant on the changed code as the signal that a test checks too little, and points to /mutation-testing:audit to get the survivors.' +execution: + prompt: 'I changed applyDiscount and its tests give 100% line coverage, but I am not sure they actually check the new code. What signal tells me a test checks too little, and how do I get it? Use the `testing:test-value` skill if you have it.' + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: skill-fired + type: tool_used + tool: Skill + input_match: '"skill"\s*:\s*"(?:testing:)?test-value"' + - name: names-mutation-audit + type: regex + pattern: '/mutation-testing:audit\b' + target: last_message + - name: meets-expectations + type: llm + criteria: 'The user asked what signal shows that tests of a changed applyDiscount, at 100% line coverage, check too little, and how to get it. PASS only if the reply (1) says a surviving mutant on the changed code (a mutation of that code that every test still passes) is a signal the tests check too little, (2) points to /mutation-testing:audit as how to get the surviving mutants, and (3) does not claim the coverage figure alone shows the tests are adequate. FAIL if any point is missing or contradicted.' diff --git a/plugins/testing/evals/tv-surviving-mutant-signal/samples/names-mutation-audit.json b/plugins/testing/evals/tv-surviving-mutant-signal/samples/names-mutation-audit.json new file mode 100644 index 00000000000..9f01a928b17 --- /dev/null +++ b/plugins/testing/evals/tv-surviving-mutant-signal/samples/names-mutation-audit.json @@ -0,0 +1,14 @@ +{ + "pass": [ + { + "answer": "A surviving mutant on the changed lines means no test checks them. Run `/mutation-testing:audit` on the change to list the survivors.", + "why": "names the slash command" + } + ], + "fail": [ + { + "answer": "Run a mutation testing tool such as Stryker on the change and look for surviving mutants.", + "why": "no pointer to the audit skill" + } + ] +} diff --git a/plugins/testing/evals/tv-surviving-mutant-signal/samples/skill-fired.json b/plugins/testing/evals/tv-surviving-mutant-signal/samples/skill-fired.json new file mode 100644 index 00000000000..0d3dc7916d4 --- /dev/null +++ b/plugins/testing/evals/tv-surviving-mutant-signal/samples/skill-fired.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": [{ "tool": "Skill", "input": { "skill": "testing:test-value" } }], + "why": "the skill loaded" + } + ], + "fail": [ + { "answer": [], "why": "no Skill call" }, + { + "answer": [{ "tool": "Skill", "input": { "skill": "mutation-testing:audit" } }], + "why": "another skill" + } + ] +} diff --git a/plugins/testing/evals/wr-parallel-worker-isolation/case.yaml b/plugins/testing/evals/wr-parallel-worker-isolation/case.yaml new file mode 100644 index 00000000000..7226d99f0b0 --- /dev/null +++ b/plugins/testing/evals/wr-parallel-worker-isolation/case.yaml @@ -0,0 +1,22 @@ +schema_version: "1.1" +name: wr-parallel-worker-isolation +description: 'An integration suite moving to four pytest-xdist workers shares one database, a fixed server port, one output file and a process environment variable. testing:write gives each worker its own database or schema named from the worker id, binds the server to port 0 and reads back the assigned port, gives each test its own temp directory, and stops tests mutating the shared environment variable. skill-fired reads the Skill calls, uses-port-zero the reply; meets-expectations (llm) judges the reply. Read-only tools, graded on the reply.' +tags: [testing, write, determinism, parallel, no-network] +expected_outcome: 'Per-worker database or schema named from the xdist worker id, server bound to port 0 with the assigned port read back, a temp directory per test, and no environment variable mutated where another worker reads it.' +execution: + prompt: "We are turning on pytest-xdist with -n 4 for our integration tests. Today every test uses one Postgres database named app_test, the test API server listens on port 8000, a fixture writes /tmp/fixtures/out.json, and a few tests set os.environ['FEATURE_X'] directly. Using the testing:write skill, what do I change so these tests stay isolated when they run in parallel?" + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: skill-fired + type: tool_used + tool: Skill + input_match: '"skill"\s*:\s*"(?:testing:)?write"' + - name: uses-port-zero + type: regex + pattern: '\b[Pp]ort\s*[=:]?\s*0\b|,\s*0\s*\)' + target: last_message + - name: meets-expectations + type: llm + criteria: 'The user is moving an integration suite to four pytest-xdist workers; tests share one database app_test, a server on fixed port 8000, the file /tmp/fixtures/out.json, and set os.environ directly. PASS only if the reply (1) gives each worker its own database or schema, named from the xdist worker id, (2) binds the server to port 0 (an OS-assigned port) and reads back the actual port instead of using 8000, (3) gives each test its own temp directory instead of the fixed /tmp path, and (4) stops tests mutating os.environ in a way another test or worker reads, for example by scoping and restoring it per test. FAIL if any point is missing or contradicted.' diff --git a/plugins/testing/evals/wr-parallel-worker-isolation/samples/skill-fired.json b/plugins/testing/evals/wr-parallel-worker-isolation/samples/skill-fired.json new file mode 100644 index 00000000000..d134c77e2f1 --- /dev/null +++ b/plugins/testing/evals/wr-parallel-worker-isolation/samples/skill-fired.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": [{ "tool": "Skill", "input": { "skill": "testing:write" } }], + "why": "the skill loaded" + } + ], + "fail": [ + { "answer": [], "why": "no Skill call" }, + { + "answer": [{ "tool": "Skill", "input": { "skill": "testing:diagnose" } }], + "why": "another testing skill" + } + ] +} diff --git a/plugins/testing/evals/wr-parallel-worker-isolation/samples/uses-port-zero.json b/plugins/testing/evals/wr-parallel-worker-isolation/samples/uses-port-zero.json new file mode 100644 index 00000000000..717742b2557 --- /dev/null +++ b/plugins/testing/evals/wr-parallel-worker-isolation/samples/uses-port-zero.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Start the server with `port=0` and read `server.server_address[1]` for the port the OS gave it.", + "why": "binds port 0" + }, + { + "answer": "Bind with `sock.bind(('127.0.0.1', 0))` and pass the assigned port to the client.", + "why": "binds port 0 by address tuple" + } + ], + "fail": [ + { + "answer": "Give each worker its own port, 8000 plus the worker number.", + "why": "a computed fixed port, not an OS-assigned one" + } + ] +} diff --git a/plugins/testing/skills/test-value/SKILL.md b/plugins/testing/skills/test-value/SKILL.md index 6408f96d1f4..595f0ca6dfa 100644 --- a/plugins/testing/skills/test-value/SKILL.md +++ b/plugins/testing/skills/test-value/SKILL.md @@ -160,6 +160,12 @@ https://jestjs.io/docs/snapshot-testing. | `rule-throw-only-oracle` | `var w = new Widget(); Assert.NotNull(w);`: only a throwing constructor fails it | assert what the constructor sets; `cant-fail-ok: ` for a smoke test kept to prove wiring | | `rule-snapshot-only` | a snapshot is the only oracle | review it as code; scrub volatile values (times, ids, seeds) with the tool's matcher or scrubber; when your own change breaks it, fix the bug or confirm the diff is intended before regenerating, never re-approve blind (§4) | +A surviving mutant on changed code is one more signal of this class: the changed line was +altered and every test stayed green, so unless the mutant is equivalent (behaves exactly as the +original), no test checks what that line does. Assert the behavior +the mutant changed, with an expected value from §1. `/mutation-testing:audit` produces the +surviving mutants for a change. + **Change detectors** (fail on any edit, prove no behavior): | Rule | Shape | Fix | diff --git a/plugins/testing/skills/write/context/write.md b/plugins/testing/skills/write/context/write.md index b01046da037..90f082f1ae8 100644 --- a/plugins/testing/skills/write/context/write.md +++ b/plugins/testing/skills/write/context/write.md @@ -114,7 +114,7 @@ A test that passes or fails by chance teaches the agent running it to retry inst - **Network**: blocked by default in unit tests; an integration test opts in to the hosts it needs. External services are faked from a recorded or documented response, not a guessed one - **No sleep**: wait on a condition with a timeout, or advance the fake clock. A fixed sleep is slow when it is long enough and flaky when it is not - **Order independence**: each test builds its own state and leaves nothing another test reads. Run the suite in shuffled order (with a printed seed) to prove it -- **Parallel safety**: unique temp directories, ports, database names and file paths per test; no shared mutable singletons +- **Parallel isolation**: when the runner spreads tests across workers, give each worker its own database or schema, bind every test server to port 0 and read back the port the OS assigned instead of a fixed one, give each test its own temp directory, and never mutate a file, an environment variable or a singleton that another worker reads. Name the per-worker database from the runner's worker id where the runner documents one: [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/how-to.html#identifying-the-worker-process-during-a-test), [Jest](https://jestjs.io/docs/environment-variables#jest_worker_id), [Playwright](https://playwright.dev/docs/test-parallel#isolate-test-data-between-parallel-workers) Runner-level switches exist for several of these (a fake clock in Playwright, seeded shuffling and per-test reseeding in pytest-randomly and `go test -shuffle`, socket blocking in pytest-socket). Read the current options at the source: [Playwright clock](https://playwright.dev/docs/clock), [pytest-randomly](https://github.com/pytest-dev/pytest-randomly), [pytest-socket](https://github.com/miketheman/pytest-socket). Whether to fake a dependency at all follows `/tdd:principles` (when the `tdd` plugin is enabled). From d485f23d0cb2ffc7b233fb7254d50c72bcc2c7f4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:15:07 -0500 Subject: [PATCH 2/3] chore(testing): add the changelog fragment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MWmASPFjTu4anSp6BowqVC --- ...ng-mutant-signal-parallel-isolation-6a2728c0.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md diff --git a/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md b/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md new file mode 100644 index 00000000000..97f1c7ee5e2 --- /dev/null +++ b/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md @@ -0,0 +1,14 @@ +--- +bump: minor +--- + +### Added + +- **Surviving mutants in `testing:test-value`.** A surviving mutant on changed code is named as one + signal that a test checks too little, with `/mutation-testing:audit` as the way to get the + survivors for a change. +- **Parallel isolation in `/testing:write`.** The determinism controls give each worker its own + database or schema, bind test servers to port 0, give each test its own temp directory, and bar + mutating a file or environment variable another worker reads, with pointers to the worker-id docs + of pytest-xdist, Jest and Playwright. +- Eval cases `tv-surviving-mutant-signal` and `wr-parallel-worker-isolation`. From 097b2a68861da595d030c752e57c39f370391bcd Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:19:12 -0500 Subject: [PATCH 3/3] fix(testing): find the runner's worker id at run time instead of linking runner docs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MWmASPFjTu4anSp6BowqVC --- .../feat-testing-mutant-signal-parallel-isolation-6a2728c0.md | 4 ++-- plugins/testing/skills/write/context/write.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md b/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md index 97f1c7ee5e2..d3beefeac63 100644 --- a/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md +++ b/.changes/testing/feat-testing-mutant-signal-parallel-isolation-6a2728c0.md @@ -9,6 +9,6 @@ bump: minor survivors for a change. - **Parallel isolation in `/testing:write`.** The determinism controls give each worker its own database or schema, bind test servers to port 0, give each test its own temp directory, and bar - mutating a file or environment variable another worker reads, with pointers to the worker-id docs - of pytest-xdist, Jest and Playwright. + mutating a file or environment variable another worker reads. The agent finds its runner's + worker id from the repository's test config and the runner's current docs at run time. - Eval cases `tv-surviving-mutant-signal` and `wr-parallel-worker-isolation`. diff --git a/plugins/testing/skills/write/context/write.md b/plugins/testing/skills/write/context/write.md index 90f082f1ae8..7eff8a5eb0a 100644 --- a/plugins/testing/skills/write/context/write.md +++ b/plugins/testing/skills/write/context/write.md @@ -114,7 +114,7 @@ A test that passes or fails by chance teaches the agent running it to retry inst - **Network**: blocked by default in unit tests; an integration test opts in to the hosts it needs. External services are faked from a recorded or documented response, not a guessed one - **No sleep**: wait on a condition with a timeout, or advance the fake clock. A fixed sleep is slow when it is long enough and flaky when it is not - **Order independence**: each test builds its own state and leaves nothing another test reads. Run the suite in shuffled order (with a printed seed) to prove it -- **Parallel isolation**: when the runner spreads tests across workers, give each worker its own database or schema, bind every test server to port 0 and read back the port the OS assigned instead of a fixed one, give each test its own temp directory, and never mutate a file, an environment variable or a singleton that another worker reads. Name the per-worker database from the runner's worker id where the runner documents one: [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/how-to.html#identifying-the-worker-process-during-a-test), [Jest](https://jestjs.io/docs/environment-variables#jest_worker_id), [Playwright](https://playwright.dev/docs/test-parallel#isolate-test-data-between-parallel-workers) +- **Parallel isolation**: when the runner spreads tests across workers, give each worker its own database or schema, bind every test server to port 0 and read back the port the OS assigned instead of a fixed one, give each test its own temp directory, and never mutate a file, an environment variable or a singleton that another worker reads. Name each per-worker resource from the runner's worker id: find how this runner exposes it from the repository's test config and the runner's current docs at run time, with `/discovery:research` when the repository does not settle it Runner-level switches exist for several of these (a fake clock in Playwright, seeded shuffling and per-test reseeding in pytest-randomly and `go test -shuffle`, socket blocking in pytest-socket). Read the current options at the source: [Playwright clock](https://playwright.dev/docs/clock), [pytest-randomly](https://github.com/pytest-dev/pytest-randomly), [pytest-socket](https://github.com/miketheman/pytest-socket). Whether to fake a dependency at all follows `/tdd:principles` (when the `tdd` plugin is enabled).