From b5df2c569970f4afb699af2408dcb1ca797eb856 Mon Sep 17 00:00:00 2001 From: "melodic-automation-lanes[bot]" <337595936+melodic-automation-lanes[bot]@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:49:01 +0000 Subject: [PATCH 1/5] Resolve issue #15728 Lane-Activity: work-resolve-issue/resolve-issue --- ...ssue-issue-15728-43770035123-1-f8582146.md | 10 +++ .../case.yaml | 10 +-- .../samples/hands-off-to-implement.json | 24 ++++++ .../samples/writes-no-analyzer.json | 0 .../scaffold.sh | 0 .../samples/hands-over-the-pointer.json | 15 ---- .../case.yaml | 25 ++++++ .../names-no-other-ecosystem-tool.json | 15 ++++ .../samples/names-the-repo-tool.json | 18 +++++ .../scaffold.sh | 71 +++++++++++++++++ .../case.yaml | 6 +- .../samples/semgrep-pointer.json | 5 +- .../case.yaml | 30 +++++++ .../samples/message-names-the-fix.json | 18 +++++ .../samples/names-no-csharp-tutorial.json | 15 ++++ .../samples/rung-and-builder.json | 18 +++++ .../scaffold.sh | 58 ++++++++++++++ .../case.yaml | 25 ++++++ .../samples/custom-analyzer-rung.json | 21 +++++ .../samples/not-sent-to-dotnet.json | 15 ++++ .../scaffold.sh | 51 ++++++++++++ plugins/corrections/skills/codify/SKILL.md | 16 ++-- .../skills/codify/actions/findings.md | 2 +- .../codify/reference/finding-classes.md | 79 +++++++++++-------- scripts/docs-anchor-baseline.txt | 1 - 25 files changed, 484 insertions(+), 64 deletions(-) create mode 100644 .changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md rename plugins/corrections/evals/{codify-declines-to-implement-the-rung => codify-declines-to-implement-hands-off-the-build}/case.yaml (52%) create mode 100644 plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json rename plugins/corrections/evals/{codify-declines-to-implement-the-rung => codify-declines-to-implement-hands-off-the-build}/samples/writes-no-analyzer.json (100%) rename plugins/corrections/evals/{codify-declines-to-implement-the-rung => codify-declines-to-implement-hands-off-the-build}/scaffold.sh (100%) delete mode 100644 plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/hands-over-the-pointer.json create mode 100644 plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/case.yaml create mode 100644 plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-no-other-ecosystem-tool.json create mode 100644 plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-the-repo-tool.json create mode 100755 plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh create mode 100644 plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml create mode 100644 plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/message-names-the-fix.json create mode 100644 plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json create mode 100644 plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/rung-and-builder.json create mode 100755 plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/scaffold.sh create mode 100644 plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml create mode 100644 plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/custom-analyzer-rung.json create mode 100644 plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/not-sent-to-dotnet.json create mode 100755 plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/scaffold.sh diff --git a/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md b/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md new file mode 100644 index 00000000000..ad56848ef37 --- /dev/null +++ b/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md @@ -0,0 +1,10 @@ +--- +bump: minor +--- + +### Changed + +- `/corrections:codify` routes a finding by what its check must decide, not by language. The finding class `dotnet-invariant` is now `semantic-invariant`, an invariant that needs types or symbols in any language, and still takes `custom-analyzer`. The finding-classes reference writes the current id `syntax-pattern-rule` and no longer points at an analyzer tutorial, a pattern-matcher plugin or an architecture-test library by name. +- Each "Owner or pointer" cell is now a hand-off. A checking rung's proposal hands over the rung and the check to prove, names `/implementation:implement` as the builder in the repository's own tooling, and states that the check's message must name the fix. When that skill is not available, the proposal stops there. +- Codify's report adds the ladder's alias notice when the repository's instructions or a stub name a rung by an old id. +- Eval cases: `codify-declines-to-implement-the-rung` and its Microsoft Learn tutorial outcome are retired. `codify-declines-to-implement-hands-off-the-build` replaces them. The `semgrep-pointer` grader no longer accepts the Semgrep docs URL. New gain cases cover a pytest `testing/audit/` finding, a TypeScript `semantic-invariant`, and a hand-off naming the tool the repository already runs. diff --git a/plugins/corrections/evals/codify-declines-to-implement-the-rung/case.yaml b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml similarity index 52% rename from plugins/corrections/evals/codify-declines-to-implement-the-rung/case.yaml rename to plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml index f8f09141949..28723841ad8 100644 --- a/plugins/corrections/evals/codify-declines-to-implement-the-rung/case.yaml +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml @@ -1,9 +1,9 @@ schema_version: "1.1" -name: codify-declines-to-implement-the-rung -description: 'Asked to just write the analyzer, the skill declines and hands over the rung''s pointer.' +name: codify-declines-to-implement-hands-off-the-build +description: 'Asked to just write the analyzer, the skill declines and hands the build of the rung and its check to /implementation:implement, pointing at no language-specific tutorial. Replaces codify-declines-to-implement-the-rung, whose tutorial outcome is retired.' tags: [codify, ported] plugins: ["../.."] -expected_outcome: 'The answer declines, points at the Microsoft Learn analyzer tutorial, and writes nothing.' +expected_outcome: 'The answer declines, hands the rung and the check to /implementation:implement, links no analyzer tutorial, and writes no analyzer.' context: scaffold_script: scaffold.sh execution: @@ -12,9 +12,9 @@ execution: max_turns: 15 timeout_seconds: 420 graders: - - name: hands-over-the-pointer + - name: hands-off-to-implement type: 'regex' - pattern: 'learn\.microsoft\.com[^\s)>]*roslyn-sdk' + pattern: '^(?![\s\S]*(?:learn\.microsoft\.com|roslyn|tutorial))(?=[\s\S]*/implementation:implement)' flags: 'i' arm: 'both' - name: writes-no-analyzer diff --git a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json new file mode 100644 index 00000000000..2d7ea0738ca --- /dev/null +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json @@ -0,0 +1,24 @@ +{ + "pass": [ + { + "answer": "I won't write the analyzer: codify only proposes. Row 3 is semantic-invariant, rung custom-analyzer. Hand the rung and the check to /implementation:implement, and the analyzer's message must name the injected clock as the fix." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Here is the analyzer." + }, + { + "answer": "See https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/tutorials/how-to-write-csharp-analyzer-code-fix" + }, + { + "answer": "Hand it to /implementation:implement, following the typescript-eslint custom rule tutorial." + }, + { + "answer": "Hand it to /implementation:implement; the tutorial is at https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/tutorials/how-to-write-csharp-analyzer-code-fix" + } + ] +} diff --git a/plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/writes-no-analyzer.json b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/writes-no-analyzer.json similarity index 100% rename from plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/writes-no-analyzer.json rename to plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/writes-no-analyzer.json diff --git a/plugins/corrections/evals/codify-declines-to-implement-the-rung/scaffold.sh b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/scaffold.sh similarity index 100% rename from plugins/corrections/evals/codify-declines-to-implement-the-rung/scaffold.sh rename to plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/scaffold.sh diff --git a/plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/hands-over-the-pointer.json b/plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/hands-over-the-pointer.json deleted file mode 100644 index febe2e0dd23..00000000000 --- a/plugins/corrections/evals/codify-declines-to-implement-the-rung/samples/hands-over-the-pointer.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "pass": [ - { - "answer": "See https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/tutorials/how-to-write-csharp-analyzer-code-fix" - } - ], - "fail": [ - { - "answer": "" - }, - { - "answer": "Here is the analyzer." - } - ] -} diff --git a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/case.yaml b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/case.yaml new file mode 100644 index 00000000000..a65dccc7e03 --- /dev/null +++ b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/case.yaml @@ -0,0 +1,25 @@ +schema_version: "1.1" +name: codify-hand-off-names-the-tool-the-repo-runs +description: 'Gain case: a layer-boundary row in a Python repository that already runs import-linter takes architecture-test, and the build hand-off names that tool and /implementation:implement rather than an architecture-test library for another ecosystem.' +tags: [codify] +plugins: ["../.."] +expected_outcome: 'Row 1 is architecture-test. The hand-off names import-linter (its pyproject contract or the lint-imports command) as the tooling the check is built in, names /implementation:implement as the builder, and names no .NET or JavaScript architecture-test library.' +context: + scaffold_script: scaffold.sh +execution: + prompt: '/corrections:codify findings findings.md What rung does the row get, and what is the next step? This run has no Bash tool: skip the proof and report the check as unproven, and present the proposal in the report, writing nothing.' + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: names-the-repo-tool + type: 'regex' + pattern: '^(?=[\s\S]*architecture-test)(?=[\s\S]*(?:import-linter|importlinter|lint-imports))(?=[\s\S]*/implementation:implement)' + flags: 'i' + arm: 'both' + - name: names-no-other-ecosystem-tool + type: 'regex' + pattern: 'ArchUnit|dependency-cruiser|depcruise' + flags: 'i' + match: 'not_contains' + arm: 'both' diff --git a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-no-other-ecosystem-tool.json b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-no-other-ecosystem-tool.json new file mode 100644 index 00000000000..d3ca8c976a9 --- /dev/null +++ b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-no-other-ecosystem-tool.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Row 1: architecture-test, a new import-linter contract, built by /implementation:implement." + } + ], + "fail": [ + { + "answer": "Row 1: architecture-test. Pointer: ArchUnitNET for .NET; dependency-cruiser for JS/TS." + }, + { + "answer": "Write the boundary test with dependency-cruiser." + } + ] +} diff --git a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-the-repo-tool.json b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-the-repo-tool.json new file mode 100644 index 00000000000..d56f1000802 --- /dev/null +++ b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/samples/names-the-repo-tool.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Row 1: structure, rung architecture-test. Build hand-off: add a forbidden contract to [tool.importlinter] in pyproject.toml, built by /implementation:implement; the check is `lint-imports`." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Row 1: architecture-test, handed to /implementation:implement." + }, + { + "answer": "Row 1: architecture-test. Add an import-linter contract." + } + ] +} diff --git a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh new file mode 100755 index 00000000000..08124d4c3df --- /dev/null +++ b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# A Python repository that already runs import-linter for its layer contracts, and a findings file +# whose one row breaks a layer boundary that no contract covers yet. +set -euo pipefail +git init -q -b main . +git config user.email eval@example.invalid +git config user.name eval +git config commit.gpgsign false +mkdir -p src/app/web src/app/domain src/app/persistence +cat >pyproject.toml <<'EOF' +[project] +name = "app" +version = "0.1.0" + +[tool.importlinter] +root_package = "app" + +[[tool.importlinter.contracts]] +name = "Domain does not import web" +type = "forbidden" +source_modules = ["app.domain"] +forbidden_modules = ["app.web"] +EOF +cat >Makefile <<'EOF' +lint: + lint-imports +EOF +cat >src/app/persistence/orders.py <<'EOF' +def load(order_id): + return {"id": order_id} +EOF +cat >src/app/web/orders.py <<'EOF' +from app.persistence.orders import load + + +def show(order_id): + return load(order_id) +EOF +touch src/app/__init__.py src/app/web/__init__.py src/app/domain/__init__.py src/app/persistence/__init__.py +git add . +git commit -q -m "feat: order page" +cat >findings.md <<'EOF' +--- +type: review-findings +date: 2026-01-01T00:00:00Z +branch: fixture +tier: medium +--- + +## Findings + +| Rank | Tier | Confidence | Location | Surface(s) | Finding | Action | +|------|------|------------|----------|------------|---------|--------| +| 1 | IMPORTANT | high | src/app/web/orders.py:1 | architecture-guardian | The web layer imports the persistence package directly, skipping the domain layer. | Assert the layer boundary and route through the domain layer. | + +## By dimension + +### architecture + +| Rank | Tier | Confidence | Location | Surface(s) | Finding | Action | +|------|------|------------|----------|------------|---------|--------| +| 1 | IMPORTANT | high | src/app/web/orders.py:1 | architecture-guardian | The web layer imports the persistence package directly, skipping the domain layer. | Assert the layer boundary and route through the domain layer. | + +## Unparsed + +None. + +## Surfaces + +Ran: architecture-guardian. +EOF diff --git a/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/case.yaml b/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/case.yaml index 92525cf17e6..7f95e93baae 100644 --- a/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/case.yaml +++ b/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/case.yaml @@ -1,9 +1,9 @@ schema_version: "1.1" name: codify-invalid-state-lands-on-make-impossible -description: 'An invalid-state row goes to make-impossible with the redesign handed off; a dangerous-call row gets the syntax-pattern rung or its Semgrep pointer; nothing installs a plugin.' +description: 'An invalid-state row goes to make-impossible with the redesign handed off; a dangerous-call row gets the syntax-pattern rung under its current id or its alias; nothing installs a plugin.' tags: [codify, ported] plugins: ["../.."] -expected_outcome: 'Row 1 is make-impossible, handed to /architecture:improve or /planning:design, implementing nothing. Row 2 is the syntax-pattern rung or names Semgrep''s documentation. The report prints no plugin install command.' +expected_outcome: 'Row 1 is make-impossible, handed to /architecture:improve or /planning:design, implementing nothing. Row 2 is the syntax-pattern rung, as syntax-pattern-rule or its alias semgrep-rule. The report prints no plugin install command.' context: scaffold_script: scaffold.sh execution: @@ -19,7 +19,7 @@ graders: arm: 'both' - name: semgrep-pointer type: 'regex' - pattern: 'semgrep-rule|syntax-pattern-rule|docs\.semgrep\.dev' + pattern: 'semgrep-rule|syntax-pattern-rule' flags: 'i' arm: 'both' - name: prints-no-plugin-install diff --git a/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/samples/semgrep-pointer.json b/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/samples/semgrep-pointer.json index cc24eecf82a..4d4d03b0478 100644 --- a/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/samples/semgrep-pointer.json +++ b/plugins/corrections/evals/codify-invalid-state-lands-on-make-impossible/samples/semgrep-pointer.json @@ -1,7 +1,7 @@ { "pass": [ { - "answer": "Row 2: semgrep-rule, https://docs.semgrep.dev/writing-rules/overview" + "answer": "Row 2: semgrep-rule (alias of syntax-pattern-rule)." }, { "answer": "Row 2: syntax-pattern-rule." @@ -13,6 +13,9 @@ }, { "answer": "Row 2: instruction." + }, + { + "answer": "Row 2: see https://docs.semgrep.dev/writing-rules/overview" } ] } diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml new file mode 100644 index 00000000000..9e72beca011 --- /dev/null +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml @@ -0,0 +1,30 @@ +schema_version: "1.1" +name: codify-pytest-oracle-finding-gets-no-csharp-tutorial +description: 'Gain case: a testing:audit finding on a pytest test takes a rung and a check to prove, points at no C# analyzer tutorial, hands the build to /implementation:implement, and says the check''s message must name the fix.' +tags: [codify] +plugins: ["../.."] +expected_outcome: 'The row is semantic-invariant on custom-analyzer from its rule id. The proposal names the check to prove, links no C# or Roslyn tutorial, names /implementation:implement as the builder, and states that the check''s message names the fix.' +context: + scaffold_script: scaffold.sh +execution: + prompt: '/corrections:codify findings findings.md What rung does the row get, and what is the next step? This run has no Bash tool: skip the proof and report the check as unproven, and present the proposal in the report, writing nothing.' + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: names-no-csharp-tutorial + type: 'regex' + pattern: 'learn\.microsoft\.com|roslyn|C#[^\n]{0,40}tutorial|tutorial[^\n]{0,40}C#' + flags: 'i' + match: 'not_contains' + arm: 'both' + - name: rung-and-builder + type: 'regex' + pattern: '^(?=[\s\S]*custom-analyzer)(?=[\s\S]*/implementation:implement)' + flags: 'i' + arm: 'both' + - name: message-names-the-fix + type: 'regex' + pattern: 'message[^\n]{0,120}\bfix\b|\bfix\b[^\n]{0,120}message' + flags: 'i' + arm: 'both' diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/message-names-the-fix.json b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/message-names-the-fix.json new file mode 100644 index 00000000000..f5a6c16be27 --- /dev/null +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/message-names-the-fix.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "The check's message must name the fix: assert on the returned total." + }, + { + "answer": "Requirement: the fix is named in the check's message, so whoever trips it knows what to assert instead." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Rung: custom-analyzer. Check: unproven." + } + ] +} diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json new file mode 100644 index 00000000000..e74dc29fd78 --- /dev/null +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Row 1: semantic-invariant, rung custom-analyzer, class-basis rule-id. Build hand-off: /implementation:implement, in the linter the repository already runs." + } + ], + "fail": [ + { + "answer": "Row 1: custom-analyzer. See Microsoft Learn, https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/tutorials/how-to-write-csharp-analyzer-code-fix" + }, + { + "answer": "Row 1: custom-analyzer; follow the C# analyzer tutorial." + } + ] +} diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/rung-and-builder.json b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/rung-and-builder.json new file mode 100644 index 00000000000..b7b39877089 --- /dev/null +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/rung-and-builder.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Rung: custom-analyzer. Builder: /implementation:implement (not available here, so the proposal stops at the rung and the check)." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Rung: custom-analyzer. Write the analyzer yourself." + }, + { + "answer": "Hand it to /implementation:implement. Rung: automated-review." + } + ] +} diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/scaffold.sh b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/scaffold.sh new file mode 100755 index 00000000000..df4d20caa91 --- /dev/null +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/scaffold.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# A Python repository tested with pytest, and a findings file whose one row is a testing:audit +# finding on a pytest test. Nothing in the tree is C# or .NET. +set -euo pipefail +git init -q -b main . +git config user.email eval@example.invalid +git config user.name eval +git config commit.gpgsign false +mkdir -p src/shop tests +cat >pyproject.toml <<'EOF' +[project] +name = "shop" +version = "0.1.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] +EOF +cat >src/shop/pricing.py <<'EOF' +def total(cart, tax): + return sum(item.price for item in cart.items) * (1 + tax.rate(cart.region)) +EOF +cat >tests/test_pricing.py <<'EOF' +from unittest.mock import Mock + +from shop.pricing import total + + +def test_total_applies_tax(): + tax = Mock() + tax.rate.return_value = 0.2 + cart = Mock(items=[Mock(price=10)], region="EU") + total(cart, tax) + tax.rate.assert_called_once_with("EU") +EOF +git add . +git commit -q -m "feat: pricing total" +cat >findings.md <<'EOF' +--- +type: review-findings +date: 2026-01-01T00:00:00Z +branch: fixture +tier: medium +--- + +## Findings + +| Rank | Tier | Confidence | Location | Surface(s) | Finding | Action | +|------|------|------------|----------|------------|---------|--------| +| 1 | IMPORTANT | high | tests/test_pricing.py:11 | testing:audit | testing/audit/rule-mock-only-oracle fired=1 excerpt=tax.rate.assert_called_once_with("EU") | Assert on the returned total, not only on the mock. | + +## Unparsed + +None. + +## Surfaces + +Ran: testing:audit. +EOF diff --git a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml new file mode 100644 index 00000000000..173af9d03c0 --- /dev/null +++ b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml @@ -0,0 +1,25 @@ +schema_version: "1.1" +name: codify-typescript-invariant-takes-custom-analyzer +description: 'Gain case: a TypeScript invariant that needs a value''s type, not its syntax, takes the custom-analyzer rung; it is not sent to a syntactic matcher or to a .NET-only rung.' +tags: [codify] +plugins: ["../.."] +expected_outcome: 'Row 1 is semantic-invariant on custom-analyzer, with the build handed to /implementation:implement. The reply does not call custom-analyzer .NET-only and links no C# analyzer tutorial.' +context: + scaffold_script: scaffold.sh +execution: + prompt: '/corrections:codify findings findings.md What class and rung does row 1 get, and what is the next step? This run has no Bash tool: skip the proof and report the check as unproven, and present the proposal in the report, writing nothing.' + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: custom-analyzer-rung + type: 'regex' + pattern: '^(?=[\s\S]*semantic-invariant)(?=[\s\S]*custom-analyzer)(?=[\s\S]*/implementation:implement)' + flags: 'i' + arm: 'both' + - name: not-sent-to-dotnet + type: 'regex' + pattern: 'learn\.microsoft\.com|roslyn|\.NET-only|only for \.NET' + flags: 'i' + match: 'not_contains' + arm: 'both' diff --git a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/custom-analyzer-rung.json b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/custom-analyzer-rung.json new file mode 100644 index 00000000000..0c64c7dea96 --- /dev/null +++ b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/custom-analyzer-rung.json @@ -0,0 +1,21 @@ +{ + "pass": [ + { + "answer": "Row 1: class semantic-invariant (judgment), rung custom-analyzer. Build hand-off: /implementation:implement." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Row 1: syntactic-pattern, rung syntax-pattern-rule." + }, + { + "answer": "Row 1: semantic-invariant, rung custom-analyzer. Write the analyzer yourself." + }, + { + "answer": "Row 1: dotnet-invariant would take custom-analyzer, but this is TypeScript, so syntactic-pattern." + } + ] +} diff --git a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/not-sent-to-dotnet.json b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/not-sent-to-dotnet.json new file mode 100644 index 00000000000..29d8b491bb5 --- /dev/null +++ b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/samples/not-sent-to-dotnet.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Row 1: semantic-invariant, custom-analyzer, built in the linter the repository already runs." + } + ], + "fail": [ + { + "answer": "The custom-analyzer rung is .NET-only, so row 1 falls to syntax-pattern-rule." + }, + { + "answer": "Pointer: https://learn.microsoft.com/en-us/dotnet/csharp/roslyn-sdk/tutorials/how-to-write-csharp-analyzer-code-fix" + } + ] +} diff --git a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/scaffold.sh b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/scaffold.sh new file mode 100755 index 00000000000..5682a7a6671 --- /dev/null +++ b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/scaffold.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# A TypeScript repository and a findings file whose one row is an invariant that turns on a value's +# type, not on how the call is spelled: no syntactic match can tell a Secret from a string. +set -euo pipefail +git init -q -b main . +git config user.email eval@example.invalid +git config user.name eval +git config commit.gpgsign false +mkdir -p src/billing +cat >package.json <<'EOF' +{ "name": "billing", "version": "1.0.0", "scripts": { "lint": "eslint src", "test": "vitest" } } +EOF +cat >src/secret.ts <<'EOF' +export type Secret = T & { readonly __secret: unique symbol }; +EOF +cat >src/logger.ts <<'EOF' +export const logger = { info: (...args: unknown[]) => console.log(JSON.stringify(args)) }; +EOF +cat >src/billing/charge.ts <<'EOF' +import type { Secret } from "../secret"; +import { logger } from "../logger"; + +export const charge = (card: { token: Secret; last4: string }) => { + const t = card.token; + logger.info("charging", t); +}; +EOF +git add . +git commit -q -m "feat: charge" +cat >findings.md <<'EOF' +--- +type: review-findings +date: 2026-01-01T00:00:00Z +branch: fixture +tier: medium +--- + +## Findings + +| Rank | Tier | Confidence | Location | Surface(s) | Finding | Action | +|------|------|------------|----------|------------|---------|--------| +| 1 | CRITICAL | high | src/billing/charge.ts:6 | code-reviewer | Passes `t`, a value typed `Secret`, to `logger.info`, which writes it in plain text. The variable's name gives no hint of its type. | Fail any logger call that receives a value whose type is `Secret`, whatever the variable is called. | + +## Unparsed + +None. + +## Surfaces + +Ran: code-reviewer. +EOF diff --git a/plugins/corrections/skills/codify/SKILL.md b/plugins/corrections/skills/codify/SKILL.md index 2bcd686725e..fa8390e155b 100644 --- a/plugins/corrections/skills/codify/SKILL.md +++ b/plugins/corrections/skills/codify/SKILL.md @@ -182,7 +182,11 @@ The proposal holds: the description, the occurrences counted, the rung id and th behind it, the scope and why, the check command, the proof result (old ref, both exit codes), the missing-context answer when it applies, the existing violations with the cleanup or ratchet offer, and the install step with whether it blocks, both taken from -["Next step per rung"](reference/finding-classes.md#next-step-per-rung). `ratchet_offer` false +["Next step per rung"](reference/finding-classes.md#next-step-per-rung). For a checking rung the +install step is the build hand-off: hand over the rung and the check, naming +`/implementation:implement` as the builder, to build it in the repository's own tooling. When that +skill is not available, the proposal ends with the rung and the check. Either way the proposal states that the check's message must name the fix, so +whoever trips it knows what to do instead. `ratchet_offer` false leaves the ratchet offer out; the existing violations and any cleanup offer stay. A proposal for a codebase, static-analysis or verification rung also holds the literal `Codifies:` line for the pull request that lands it, and the task-list item that carries it when the work is filed as an issue, @@ -193,14 +197,16 @@ holds neither. Problems first: a mistake that meets the ladder's "Guardrail criterion" with no check in the repository yet, stated in those words as a "no guardrail" finding naming that section; an unproven -check; a count below threshold; an unresolved old ref. Then the count, the rung by its id, the scope, the proof line from the script, and the proposal path or that nothing was written. End -with the question for any blocking install. +check; a count below threshold; an unresolved old ref. Then the count, the rung by its id, the scope, the proof line from the script, and the proposal path or that nothing was written. When +the repository's instructions or a stub named a rung by an alias, add the notice naming the rename +that the ladder's alias line requires. End with the question for any blocking install. ## Boundary - It never installs a check unasked, and never installs an unproven one. - It never implements a rung. No analyzer, rule, architecture test or hook is written here. Asked to - "just write the analyzer", decline and hand over the rung's pointer. + "just write the analyzer", decline and hand over the rung and the check, naming + `/implementation:implement` as the builder. - It never fixes an existing violation. The cleanup it offers runs through `/implementation:implement`. - Moving a mistake up the ladder is not promotion in the autonomy sense: it changes no work class or guardrail class, and neither implies the other. @@ -214,7 +220,7 @@ with the question for any blocking install. - hook: /harness-config:audit-automation-gaps. - make-impossible: /architecture:improve. - a group at or over the threshold: /corrections:codify "". -- existing violations to clean up: /implementation:implement. +- existing violations to clean up, or a proven check to build: /implementation:implement. ## Gotchas diff --git a/plugins/corrections/skills/codify/actions/findings.md b/plugins/corrections/skills/codify/actions/findings.md index 9875dd2fbe6..77220babbaa 100644 --- a/plugins/corrections/skills/codify/actions/findings.md +++ b/plugins/corrections/skills/codify/actions/findings.md @@ -60,5 +60,5 @@ also never carries `type: review-findings`, a top-level `branch:` key or a `## F ## 5. Report Before the hub's report: the `ratchet_offer` line, any `> DEGRADED:` line, then one row per finding -with rank, class, class basis, rung, owner, proof result and proposal path, then +with rank, class, class basis, rung, hand-off, proof result and proposal path, then `N findings → N proposals in `, where N is the table's row count. diff --git a/plugins/corrections/skills/codify/reference/finding-classes.md b/plugins/corrections/skills/codify/reference/finding-classes.md index 049ae620a05..8124d2581f2 100644 --- a/plugins/corrections/skills/codify/reference/finding-classes.md +++ b/plugins/corrections/skills/codify/reference/finding-classes.md @@ -1,26 +1,34 @@ # Finding classes -For a row of a review findings file: class to rung to owner. The ladder's rungs and its guardrail -criterion stay the authority, and codify's selection rule is strongest first for every input. A -class row names the check its findings usually reach. Walk the ladder from the top: a stronger rung -that can decide the row wins, and the row's owner follows that rung ("Next step per rung" below). -An `invalid-state` row is offered `make-impossible` first; codify only proposes that redesign. +For a row of a review findings file: class to rung to hand-off. The ladder's rungs and its +guardrail criterion stay the authority, and codify's selection rule is strongest first for every +input. A class row names the check its findings usually reach. Walk the ladder from the top: a +stronger rung that can decide the row wins, and the row's hand-off follows that rung ("Next step per +rung" below). An `invalid-state` row is offered `make-impossible` first; codify only proposes that +redesign. + +A class names what the check must decide, never a language or a product. Codify picks the rung and +proves the check; the build is handed off. A **build hand-off** is the rung, the check to prove and +the requirement that the check's message names the fix, with `/implementation:implement` named as +the builder. When that skill is available, hand it over; otherwise the proposal stops there, with +the rung, the check and the builder stated. The check is built in the repository's own tooling, the +analyzer, matcher or test library it already runs; codify names no product of its own. ## Class table -| Finding class | Rung | Owner or pointer | +| Finding class | Rung | Hand-off | |---|---|---| | `invalid-state` (a value or combination the code can express but must never hold: two flags that must not both be true, a field that is always set after construction but typed as optional, two ids of different kinds that can be passed in each other's place) | `make-impossible` | `/architecture:improve` when that skill is available, otherwise `/planning:design`; the proposal names the type, data-structure or API change and implements nothing | -| `style` (formatting, whitespace, ordering, naming style) | `editorconfig-severity` | in-repo `.editorconfig` | -| `defined-diagnostic` (a diagnostic an installed analyzer pack or linter already defines: a cited `CAxxxx`/`IDExxxx`/`SAxxxx`, an ESLint, ruff, or markdownlint id, or a rule the pack documents) | `analyzer-pack-rule` | in-repo configuration: enable the rule or raise its severity in `.editorconfig`, `.globalconfig`, or the linter's own config | -| `dotnet-invariant` (a project-specific API or usage invariant in C# expressible over syntax or the semantic model: a banned API, a required attribute, a misuse pattern) | `custom-analyzer` | Microsoft Learn, "Tutorial: Write your first analyzer and code fix" () | -| `syntactic-pattern` (a code pattern expressible as a syntactic match in any language: a dangerous call, an injection sink, a secret shape, a cross-language invariant; also `dotnet-invariant` in a non-.NET ecosystem) | `semgrep-rule` | the `semgrep-rule-creator` plugin when installed; otherwise Semgrep's rule-writing documentation () | -| `structure` (dependency direction, layering, namespace-to-layer naming, forbidden references) | `architecture-test` | ArchUnitNET () for .NET; dependency-cruiser () for JS/TS | +| `style` (formatting, whitespace, ordering, naming style) | `editorconfig-severity` | build hand-off: the setting in the in-repo `.editorconfig` | +| `defined-diagnostic` (a diagnostic an installed analyzer pack or linter already defines: a cited diagnostic id, or a rule the pack documents) | `analyzer-pack-rule` | build hand-off: enable the rule or raise its severity in the configuration the repository already uses for that pack | +| `semantic-invariant` (a project-specific API or usage invariant that needs meaning, not syntax, in any language: a banned API resolved by symbol, a required attribute, a misuse pattern that depends on types or on which symbol a call reaches) | `custom-analyzer` | build hand-off: an analyzer for the compiler or linter the repository already runs | +| `syntactic-pattern` (a code pattern expressible as a syntactic match in any language: a dangerous call, an injection sink, a secret shape, a cross-language invariant) | `syntax-pattern-rule` | build hand-off: a rule for the pattern matcher the repository already runs | +| `structure` (dependency direction, layering, namespace-to-layer naming, forbidden references) | `architecture-test` | build hand-off: a test written with the repository's own architecture-test library | | `process` (commit shape, file placement, generated-file freshness, session behavior: anything observed at tool-call or commit time rather than in source) | `action-guard` | the `harness-config` plugin's automation-gaps audit when enabled; otherwise record the candidate and stop | | `design-judgment` (readability, correctness reasoning, prose quality) and `unclassified` | `automated-review` | none; the finding stays a review-time judgment | -The `custom-analyzer` rung is .NET-only. The same invariant in any other ecosystem is -`syntactic-pattern`, and falls to the `semgrep-rule` rung. +An invariant a syntactic match can decide is `syntactic-pattern`, whatever the language; one that +needs to resolve types or symbols is `semantic-invariant`. Neither class is tied to an ecosystem. ## Lookup sections @@ -34,21 +42,21 @@ rung: no stronger rung is walked. The one id form the detector-findings contract allows is `//rule-`, so a match here is byte-exact on the whole id. -Every owner cell below reading "already deterministic: keep the `:` detector" is +Every hand-off cell below reading "already deterministic: keep the `:` detector" is gated on that plugin still being enabled in the consuming repo; where it is not, the finding has no deterministic rung there and falls back to `automated-review`, a review-time judgment, until the detector is enabled again. The same gate and the same fallback bind the rule-family rows. -| Rule id | Class | Rung | Owner or pointer | Basis | +| Rule id | Class | Rung | Hand-off | Basis | |---|---|---|---|---| | `mutation-testing/audit/rule-survivor-productive` | `design-judgment` | `automated-review` | none | the remedy is a covering test whose assertion is a judgment about the code's contract, which no check can state for you | | `mutation-testing/audit/rule-survivor-unclassified` | `design-judgment` | `automated-review` | none | the row exists because the classification itself was unresolved, so no deterministic rung can be argued for it | | `mutation-testing/audit/rule-survivor-arid` | `design-judgment` | `automated-review` | none | aridity is a judgment about whether the node is worth testing at all | | `mutation-testing/audit/rule-survivor-equivalent` | `design-judgment` | `automated-review` | none | equivalence is undecidable in general, so a check that claimed it would be asserting something it cannot know | -| `testing/audit/rule-zero-assertion` | `defined-diagnostic` | `analyzer-pack-rule` | the test-framework analyzer pack already in the project | an assertion-free test body is a whole-method syntactic shape, which is what a test-framework analyzer pack is for; the work is enabling or raising a diagnostic, not writing one | -| `testing/audit/rule-recomputed-expectation` | `dotnet-invariant` | `custom-analyzer` | Microsoft Learn analyzer tutorial (see the class table) | deciding an expected value was recomputed from the code under test needs the semantic model to resolve both sides to the same symbol, which no pack diagnostic states | -| `testing/audit/rule-mock-only-oracle` | `dotnet-invariant` | `custom-analyzer` | Microsoft Learn analyzer tutorial (see the class table) | whether the oracle reaches only mocks is a property of which symbols the assertions touch, again a semantic-model question, and over the project's own mocking library | -| `testing/audit/rule-inert-assertion` | `defined-diagnostic` | `analyzer-pack-rule` | the test-framework analyzer pack already in the project | we map this rule to the pack diagnostic for the test's language, so the work is enabling or raising it, not writing one: xUnit2021, eslint-plugin-jest `valid-expect`, ShellCheck SC2314, Ruff F631. **Pointer**: when choosing the diagnostic to enable, fetch its rule page live: [xUnit2021](https://xunit.net/xunit.analyzers/rules/xUnit2021), [`valid-expect`](https://github.com/jest-community/eslint-plugin-jest/blob/main/docs/rules/valid-expect.md), [SC2314](https://www.shellcheck.net/wiki/SC2314), [F631](https://docs.astral.sh/ruff/rules/assert-tuple/). Revisit the mapping when `testing:audit` changes what `rule-inert-assertion` matches | +| `testing/audit/rule-zero-assertion` | `defined-diagnostic` | `analyzer-pack-rule` | build hand-off: the test-framework analyzer pack already in the project | an assertion-free test body is a whole-method syntactic shape, which is what a test-framework analyzer pack is for; the work is enabling or raising a diagnostic, not writing one | +| `testing/audit/rule-recomputed-expectation` | `semantic-invariant` | `custom-analyzer` | build hand-off (see the class table) | deciding an expected value was recomputed from the code under test needs the semantic model to resolve both sides to the same symbol, which no pack diagnostic states | +| `testing/audit/rule-mock-only-oracle` | `semantic-invariant` | `custom-analyzer` | build hand-off (see the class table) | whether the oracle reaches only mocks is a property of which symbols the assertions touch, again a semantic-model question, and over the project's own mocking library | +| `testing/audit/rule-inert-assertion` | `defined-diagnostic` | `analyzer-pack-rule` | build hand-off: the test-framework or linter pack already in the project | the test's language has a pack diagnostic for an assertion whose result is never checked, so the work is enabling or raising it, not writing one. When choosing it, read the pack's own rule page live. Revisit the mapping when `testing:audit` changes what `rule-inert-assertion` matches | | `testing/audit/rule-constant-restatement` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | the producing detector is itself the deterministic check, a pattern over the equality's two sides, so the cheapest rung is the one already running | | `testing/audit/rule-source-text-read` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | as above | | `testing/audit/rule-conditional-assertion` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | as above | @@ -56,7 +64,7 @@ detector is enabled again. The same gate and the same fallback bind the rule-fam | `testing/audit/rule-snapshot-only` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | as above | | `testing/audit/rule-weak-oracle` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | as above | | `testing/audit/rule-flaky-passes-suite` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `testing:audit` detector | no analyzer pack reads Playwright `retries` against `failOnFlakyTests` (judgment), so the detector's config read stays the check | -| `testing/audit/rule-only-not-forbidden` | `defined-diagnostic` | `analyzer-pack-rule` | the project's ESLint config | we map this rule to eslint-plugin-playwright `no-focused-test`, enabled in the project's ESLint config. **Pointer**: when enabling it, fetch [its rule page](https://github.com/playwright-community/eslint-plugin-playwright/blob/main/docs/rules/no-focused-test.md) live. Revisit the mapping when `testing:audit` changes what `rule-only-not-forbidden` matches | +| `testing/audit/rule-only-not-forbidden` | `defined-diagnostic` | `analyzer-pack-rule` | build hand-off: the project's own linter configuration | the test framework's linter plugin defines a diagnostic for a focused test, so the work is enabling it in the configuration the project already runs. When enabling it, read the plugin's own rule page live. Revisit the mapping when `testing:audit` changes what `rule-only-not-forbidden` matches | | `ai-slop/audit/rule-em-dash` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `ai-slop:audit` detector | the producing detector is itself the deterministic check, so the cheapest rung is the one already running; hold its severity rather than rebuild it | | `ai-slop/audit/rule-emoji-formatting` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `ai-slop:audit` detector | as above: the detector fires this rule from a pattern, not a judgment | | `ai-slop/audit/rule-curly-artifacts` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `ai-slop:audit` detector | as above | @@ -87,10 +95,10 @@ detector is enabled again. The same gate and the same fallback bind the rule-fam A lower and distinct ladder step, so an id a producer adds later still lands on its family's class instead of falling through to prose. -| Family prefix | Class | Rung | Owner or pointer | Basis | +| Family prefix | Class | Rung | Hand-off | Basis | |---|---|---|---|---| | `mutation-testing/audit/` | `design-judgment` | `automated-review` | none | every rule in this family reports a surviving mutant, whose remedy is a test whose assertion is judgment | -| `testing/audit/` | `dotnet-invariant` | `custom-analyzer` | Microsoft Learn analyzer tutorial (see the class table) | the family's rules are properties of a test's oracle; the family default takes the semantic-model rung, and an individual rule a pack already covers is listed above | +| `testing/audit/` | `semantic-invariant` | `custom-analyzer` | build hand-off (see the class table) | the family's rules are properties of a test's oracle in whatever language the test is written; the family default takes the semantic-model rung, and an individual rule a pack already covers is listed above | | `ai-slop/audit/` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `ai-slop:audit` detector | the producing detector is the deterministic check for the whole family | | `harness-config/audit-instructions/` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `harness-config:audit-instructions` detector | as above | | `docs-hygiene/audit-noise/` | `defined-diagnostic` | `analyzer-pack-rule` | already deterministic: keep the `docs-hygiene:audit-noise` detector | as above | @@ -104,7 +112,7 @@ is unambiguous, plus a default that hands everything else to the next ladder ste | Dimension | Class | Rung | Basis | |---|---|---|---| | `architecture` | `structure` | `architecture-test` | the dimension names dependency and layering shape, which is exactly what an architecture test asserts | -| `security` | `syntactic-pattern` | `semgrep-rule` | the recurring security findings a check can catch are sink and secret shapes, which a syntactic matcher expresses in any language | +| `security` | `syntactic-pattern` | `syntax-pattern-rule` | the recurring security findings a check can catch are sink and secret shapes, which a syntactic matcher expresses in any language | | `docs` | `design-judgment` | `automated-review` | prose quality is judgment; a docs finding that is mechanical carries a rule id and was already resolved a step earlier | | any other or unlisted dimension | (none) | (none) | fall through to the judgment step; the enum is open, so an unlisted heading carries no mapping and guessing one would be worse than reading the row | @@ -138,9 +146,16 @@ carried out by `/implementation:implement`. One rule chooses between cleanup and cleanup when every existing violation can be fixed in the change that lands the rule, as one reviewable pull request with no fix needing a design decision. Otherwise, or when the count is unknown, a counting rung (`editorconfig-severity`, `analyzer-pack-rule`, `custom-analyzer`, -`semgrep-rule`, `architecture-test`) with the ratchet offer on ends with it: once the rule exists, -the count goes to `/toolchain:ratchet` and the cleanup follows as its own work. A zero count lands -the rule. +`syntax-pattern-rule`, `architecture-test`) with the ratchet offer on ends with it: once the rule +exists, the count goes to `/toolchain:ratchet` and the cleanup follows as its own work. A zero count +lands the rule. + +Every checking rung below ends in the build hand-off (see the opening of this file), except +`action-guard` and a rung whose producing detector already is the check: the rung, the check to +prove, and the requirement that the check's message names the fix (the ladder's guardrail criterion +4), with `/implementation:implement` named as the builder. The check is the repository's own, built +in the tooling it already runs; when that tooling is visible in the tree (a config file, a rules +directory, a test project), name it. - **`make-impossible`**: name the type, data structure or API whose shape lets the invalid state exist, and the change that would remove it. Hand it to `/architecture:improve` when that skill is @@ -149,13 +164,11 @@ the rule. - **`editorconfig-severity`**: name the setting and the file; no plugin is involved. - **`analyzer-pack-rule`**: name the diagnostic id and the config file that raises its severity. Where the producing detector already is the deterministic check, the next step is to keep it. -- **`custom-analyzer`**: point at Microsoft Learn, "Tutorial: Write your first analyzer and code - fix" (). - This rung is .NET-only; the same invariant elsewhere belongs on the Semgrep rung. -- **`semgrep-rule`**: `/semgrep-rule-creator:semgrep-rule-creator` when that plugin is installed; - otherwise Semgrep's rule-writing documentation (). -- **`architecture-test`**: ArchUnitNET () for .NET and the - dependency-cruiser repository docs () for JS/TS. +- **`custom-analyzer`**: name the invariant the analyzer decides and the symbols or types it + resolves, in any language whose compiler or linter can run one. +- **`syntax-pattern-rule`**: name the pattern to match and the languages it covers. +- **`architecture-test`**: name the modules or layers and the reference the test forbids, read from + the boundary rules the ladder's "Where boundary rules live" lists. - **`action-guard`**: `/harness-config:audit-automation-gaps hooks` when that plugin is enabled, presenting the proposal as a candidate; otherwise the proposal records the candidate with its frequency, incident history and the enforcement already covering it. diff --git a/scripts/docs-anchor-baseline.txt b/scripts/docs-anchor-baseline.txt index b68fc985aa1..522ee4bacc7 100644 --- a/scripts/docs-anchor-baseline.txt +++ b/scripts/docs-anchor-baseline.txt @@ -27,7 +27,6 @@ plugins/code-tidying/skills/remove-comments/reference/tooling.md 1 plugins/code-tidying/skills/tidy/reference/bundled-simplify.md 1 plugins/context-budget/skills/audit/reference/native-explain-usage.md 1 plugins/context7/skills/lookup/context/cli.md 1 -plugins/corrections/skills/codify/reference/finding-classes.md 3 plugins/debugging/skills/analyze-profile/reference/formats/heapsnapshot.md 1 plugins/debugging/skills/analyze-profile/reference/formats/other-formats.md 1 plugins/debugging/skills/debug/reference/ecosystem-debugging.md 1 From dd6bd9a7a55f51f9ffd820cb6cd18a95c272811c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:11:14 -0500 Subject: [PATCH 2/5] fix(corrections): widen codify's alias notice and loosen three negative graders The Report line for the ladder's alias covered only an old rung id read from the repository's instructions or a stub. It now covers any read, a findings file included, gives one line naming the old id and the current one, and links the ladder's alias line, as the ladder asks of a skill that meets the old id. Three negative graders on the new cases failed a correct reply: hands-off-to-implement on any mention of "tutorial" or of Roslyn (the fixture's row 3 is C#, where a Roslyn analyzer is the repository's own tooling), names-no-csharp-tutorial on "no C# tutorial applies", and not-sent-to-dotnet on "custom-analyzer is not .NET-only". Each now matches the tutorial link, a tutorial the reply points to, or the claim itself, and each has those replies as pass samples. validate-cases: exit 0, FAIL 0. Refs: #15728 Co-Authored-By: Claude Fable 5.2 --- ...ork-resolve-issue-issue-15728-43770035123-1-f8582146.md | 2 +- .../case.yaml | 2 +- .../samples/hands-off-to-implement.json | 3 +++ .../case.yaml | 2 +- .../samples/names-no-csharp-tutorial.json | 3 +++ .../case.yaml | 2 +- .../samples/not-sent-to-dotnet.json | 6 ++++++ plugins/corrections/skills/codify/SKILL.md | 7 ++++--- 8 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md b/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md index ad56848ef37..e4b4651aecb 100644 --- a/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md +++ b/.changes/corrections/work-resolve-issue-issue-15728-43770035123-1-f8582146.md @@ -6,5 +6,5 @@ bump: minor - `/corrections:codify` routes a finding by what its check must decide, not by language. The finding class `dotnet-invariant` is now `semantic-invariant`, an invariant that needs types or symbols in any language, and still takes `custom-analyzer`. The finding-classes reference writes the current id `syntax-pattern-rule` and no longer points at an analyzer tutorial, a pattern-matcher plugin or an architecture-test library by name. - Each "Owner or pointer" cell is now a hand-off. A checking rung's proposal hands over the rung and the check to prove, names `/implementation:implement` as the builder in the repository's own tooling, and states that the check's message must name the fix. When that skill is not available, the proposal stops there. -- Codify's report adds the ladder's alias notice when the repository's instructions or a stub name a rung by an old id. +- Codify's report adds the ladder's alias notice, one line naming the old id and the current one, when it reads a rung under an old id from the repository's instructions, a stub or a findings file. - Eval cases: `codify-declines-to-implement-the-rung` and its Microsoft Learn tutorial outcome are retired. `codify-declines-to-implement-hands-off-the-build` replaces them. The `semgrep-pointer` grader no longer accepts the Semgrep docs URL. New gain cases cover a pytest `testing/audit/` finding, a TypeScript `semantic-invariant`, and a hand-off naming the tool the repository already runs. diff --git a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml index 28723841ad8..38c2f7823e6 100644 --- a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml @@ -14,7 +14,7 @@ execution: graders: - name: hands-off-to-implement type: 'regex' - pattern: '^(?![\s\S]*(?:learn\.microsoft\.com|roslyn|tutorial))(?=[\s\S]*/implementation:implement)' + pattern: '^(?![\s\S]*(?:learn\.microsoft\.com|roslyn-sdk|(?:follow|see|read|use|start with|work through)[^\n]{0,40}\btutorial))(?=[\s\S]*/implementation:implement)' flags: 'i' arm: 'both' - name: writes-no-analyzer diff --git a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json index 2d7ea0738ca..73645e5fab2 100644 --- a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json @@ -2,6 +2,9 @@ "pass": [ { "answer": "I won't write the analyzer: codify only proposes. Row 3 is semantic-invariant, rung custom-analyzer. Hand the rung and the check to /implementation:implement, and the analyzer's message must name the injected clock as the fix." + }, + { + "answer": "I won't write it, and there is no tutorial to point you at. Row 3 needs a Roslyn analyzer in the project's own analyzer setup; hand the rung and the check to /implementation:implement." } ], "fail": [ diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml index 9e72beca011..4ae60240e8d 100644 --- a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml @@ -14,7 +14,7 @@ execution: graders: - name: names-no-csharp-tutorial type: 'regex' - pattern: 'learn\.microsoft\.com|roslyn|C#[^\n]{0,40}tutorial|tutorial[^\n]{0,40}C#' + pattern: 'learn\.microsoft\.com|roslyn|(?:follow|see|read|use|start with|work through)[^\n]{0,40}\bC#[^\n]{0,40}tutorial' flags: 'i' match: 'not_contains' arm: 'both' diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json index e74dc29fd78..5edce086e88 100644 --- a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json @@ -2,6 +2,9 @@ "pass": [ { "answer": "Row 1: semantic-invariant, rung custom-analyzer, class-basis rule-id. Build hand-off: /implementation:implement, in the linter the repository already runs." + }, + { + "answer": "This is a pytest repository, so no C# tutorial applies. Row 1: semantic-invariant, custom-analyzer; hand the rung and the check to /implementation:implement." } ], "fail": [ diff --git a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml index 173af9d03c0..1224234aa13 100644 --- a/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml +++ b/plugins/corrections/evals/codify-typescript-invariant-takes-custom-analyzer/case.yaml @@ -19,7 +19,7 @@ graders: arm: 'both' - name: not-sent-to-dotnet type: 'regex' - pattern: 'learn\.microsoft\.com|roslyn|\.NET-only|only for \.NET' + pattern: 'learn\.microsoft\.com|roslyn|(? Date: Mon, 16 Nov 2026 13:24:22 -0500 Subject: [PATCH 3/5] test(corrections): add a loss case for codify's rule-id rows The rule-id rows no longer name the pack diagnostics. This case gives codify a pytest rule-inert-assertion finding in a repository that runs ruff with the assert-tuple diagnostic off, and checks the row still lands on analyzer-pack-rule and names ruff's F631 (assert-tuple) to enable. Its samples pass and fail as listed under validate-cases. Co-Authored-By: Claude Opus 5.5 --- .../case.yaml | 24 ++++++++ .../samples/names-the-diagnostic.json | 18 ++++++ .../samples/pack-rule-rung.json | 15 +++++ .../scaffold.sh | 57 +++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/case.yaml create mode 100644 plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/names-the-diagnostic.json create mode 100644 plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/pack-rule-rung.json create mode 100755 plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/scaffold.sh diff --git a/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/case.yaml b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/case.yaml new file mode 100644 index 00000000000..f885293eff8 --- /dev/null +++ b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/case.yaml @@ -0,0 +1,24 @@ +schema_version: "1.1" +name: codify-inert-assertion-names-the-pack-diagnostic +description: 'Loss case for the rule-id rows: a testing:audit rule-inert-assertion finding on a pytest test in a repository that runs ruff stays on analyzer-pack-rule and names the ruff diagnostic to enable, now that the row no longer lists the diagnostic names.' +tags: [codify] +plugins: ["../.."] +expected_outcome: 'Row 1 is defined-diagnostic on analyzer-pack-rule from its rule id. The next step enables ruff''s assert-tuple diagnostic (F631) in the ruff configuration the repository already runs, rather than writing a new analyzer.' +context: + scaffold_script: scaffold.sh +execution: + prompt: '/corrections:codify findings findings.md What rung does the row get, and what exactly gets enabled? This run has no Bash tool: skip the proof and report the check as unproven, and present the proposal in the report, writing nothing.' + allowed_tools: [Read, Glob, Grep, Skill] + max_turns: 15 + timeout_seconds: 420 +graders: + - name: pack-rule-rung + type: 'regex' + pattern: 'analyzer-pack-rule' + flags: 'i' + arm: 'both' + - name: names-the-diagnostic + type: 'regex' + pattern: '\bF631\b|assert-tuple' + flags: 'i' + arm: 'both' diff --git a/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/names-the-diagnostic.json b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/names-the-diagnostic.json new file mode 100644 index 00000000000..f2b3049b2c8 --- /dev/null +++ b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/names-the-diagnostic.json @@ -0,0 +1,18 @@ +{ + "pass": [ + { + "answer": "Rung: analyzer-pack-rule. Add F631 to [tool.ruff.lint] select in pyproject.toml." + }, + { + "answer": "Enable ruff's assert-tuple rule in the existing ruff configuration." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Rung: analyzer-pack-rule. Enable the test-framework pack diagnostic for inert assertions in the project's linter configuration." + } + ] +} diff --git a/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/pack-rule-rung.json b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/pack-rule-rung.json new file mode 100644 index 00000000000..3631bd220d3 --- /dev/null +++ b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/samples/pack-rule-rung.json @@ -0,0 +1,15 @@ +{ + "pass": [ + { + "answer": "Rung: analyzer-pack-rule. The rule id maps to defined-diagnostic; enable the pack diagnostic in the project's ruff configuration." + } + ], + "fail": [ + { + "answer": "" + }, + { + "answer": "Rung: custom-analyzer. Hand the analyzer build to /implementation:implement." + } + ] +} diff --git a/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/scaffold.sh b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/scaffold.sh new file mode 100755 index 00000000000..2cb95ecf1d3 --- /dev/null +++ b/plugins/corrections/evals/codify-inert-assertion-names-the-pack-diagnostic/scaffold.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# A Python repository tested with pytest and linted with ruff, whose ruff selection leaves the +# assert-on-tuple diagnostic off, and a findings file whose one row is a testing:audit +# rule-inert-assertion finding on that test. +set -euo pipefail +git init -q -b main . +git config user.email eval@example.invalid +git config user.name eval +git config commit.gpgsign false +mkdir -p src/shop tests +cat >pyproject.toml <<'EOF' +[project] +name = "shop" +version = "0.1.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff.lint] +select = ["E", "W", "I"] +EOF +cat >src/shop/pricing.py <<'EOF' +def total(prices, rate): + return sum(prices) * (1 + rate) +EOF +cat >tests/test_pricing.py <<'EOF' +from shop.pricing import total + + +def test_total_applies_tax(): + result = total([10], 0.2) + assert (result == 12, "total should include tax") +EOF +git add . +git commit -q -m "feat: pricing total" +cat >findings.md <<'EOF' +--- +type: review-findings +date: 2026-01-01T00:00:00Z +branch: fixture +tier: medium +--- + +## Findings + +| Rank | Tier | Confidence | Location | Surface(s) | Finding | Action | +|------|------|------------|----------|------------|---------|--------| +| 1 | IMPORTANT | high | tests/test_pricing.py:6 | testing:audit | testing/audit/rule-inert-assertion fired=1 excerpt=assert (result == 12, "total should include tax") | Assert the comparison itself, not a tuple that is always true. | + +## Unparsed + +None. + +## Surfaces + +Ran: testing:audit. +EOF From 064c596765fd383a4f04f23f1c1e27c839ee0e56 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:35:55 -0500 Subject: [PATCH 4/5] test(corrections): bound the tutorial graders' verbs to whole words The verb list in two codify eval graders matched inside other words ("read" in "already", "use" in "because"), so a correct reply that says no tutorial applies could fail. The verbs now match as whole words, with their inflected forms listed, and each case gains a pass sample that the old pattern failed. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01MWmASMXyUAaNx5GqGM3h6c --- .../codify-declines-to-implement-hands-off-the-build/case.yaml | 2 +- .../samples/hands-off-to-implement.json | 3 +++ .../case.yaml | 2 +- .../samples/names-no-csharp-tutorial.json | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml index 38c2f7823e6..71b7eedc1b3 100644 --- a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/case.yaml @@ -14,7 +14,7 @@ execution: graders: - name: hands-off-to-implement type: 'regex' - pattern: '^(?![\s\S]*(?:learn\.microsoft\.com|roslyn-sdk|(?:follow|see|read|use|start with|work through)[^\n]{0,40}\btutorial))(?=[\s\S]*/implementation:implement)' + pattern: '^(?![\s\S]*(?:learn\.microsoft\.com|roslyn-sdk|\b(?:follow(?:s|ed|ing)?|see|read(?:s|ing)?|us(?:e|es|ed|ing)|start(?:s|ed|ing)? with|work(?:s|ed|ing)? through)\b[^\n]{0,40}\btutorial))(?=[\s\S]*/implementation:implement)' flags: 'i' arm: 'both' - name: writes-no-analyzer diff --git a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json index 73645e5fab2..f988f1f97a7 100644 --- a/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json +++ b/plugins/corrections/evals/codify-declines-to-implement-hands-off-the-build/samples/hands-off-to-implement.json @@ -5,6 +5,9 @@ }, { "answer": "I won't write it, and there is no tutorial to point you at. Row 3 needs a Roslyn analyzer in the project's own analyzer setup; hand the rung and the check to /implementation:implement." + }, + { + "answer": "I won't write it. Hand the rung and the check to /implementation:implement, built in the analyzer setup the repository already runs; no tutorial applies." } ], "fail": [ diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml index 4ae60240e8d..2ed50581cc6 100644 --- a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/case.yaml @@ -14,7 +14,7 @@ execution: graders: - name: names-no-csharp-tutorial type: 'regex' - pattern: 'learn\.microsoft\.com|roslyn|(?:follow|see|read|use|start with|work through)[^\n]{0,40}\bC#[^\n]{0,40}tutorial' + pattern: 'learn\.microsoft\.com|roslyn|\b(?:follow(?:s|ed|ing)?|see|read(?:s|ing)?|us(?:e|es|ed|ing)|start(?:s|ed|ing)? with|work(?:s|ed|ing)? through)\b[^\n]{0,40}\bC#[^\n]{0,40}tutorial' flags: 'i' match: 'not_contains' arm: 'both' diff --git a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json index 5edce086e88..951cfc71ffd 100644 --- a/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json +++ b/plugins/corrections/evals/codify-pytest-oracle-finding-gets-no-csharp-tutorial/samples/names-no-csharp-tutorial.json @@ -5,6 +5,9 @@ }, { "answer": "This is a pytest repository, so no C# tutorial applies. Row 1: semantic-invariant, custom-analyzer; hand the rung and the check to /implementation:implement." + }, + { + "answer": "Row 1: semantic-invariant, custom-analyzer. Hand it to /implementation:implement, built in the linter the repository already runs, because a C# analyzer tutorial does not apply to pytest." } ], "fail": [ From a0e3c28864b91a3e7bafa2a6f10560c5ec51d749 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:39:53 -0500 Subject: [PATCH 5/5] test(corrections): write the eval fixture's Makefile without a tab in the script The editorconfig check failed the scaffold for the Makefile recipe's tab inside a heredoc. printf writes the same bytes. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01MWmASMXyUAaNx5GqGM3h6c --- .../codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh index 08124d4c3df..79076c1f59c 100755 --- a/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh +++ b/plugins/corrections/evals/codify-hand-off-names-the-tool-the-repo-runs/scaffold.sh @@ -21,10 +21,7 @@ type = "forbidden" source_modules = ["app.domain"] forbidden_modules = ["app.web"] EOF -cat >Makefile <<'EOF' -lint: - lint-imports -EOF +printf 'lint:\n\tlint-imports\n' >Makefile cat >src/app/persistence/orders.py <<'EOF' def load(order_id): return {"id": order_id}