diff --git a/.changes/architecture/fix-record-decision-eval-graders-8d2f61b3.md b/.changes/architecture/fix-record-decision-eval-graders-8d2f61b3.md new file mode 100644 index 00000000000..e377e4567a1 --- /dev/null +++ b/.changes/architecture/fix-record-decision-eval-graders-8d2f61b3.md @@ -0,0 +1,15 @@ +--- +bump: patch +--- + +### Fixed + +- The `lists-citation` graders of the `rd-audit-lists-gone-record` and + `rd-reversal-asks-before-retiring` eval cases no longer accept a line range or list that starts at + the citing line (`cookie_store.py:1-2`, `schema.graphql:1,2`). The sweep lists the one citing line, + and a drop proposal on a range would also edit a line that cites nothing. +- The `asks-before-retiring` grader now accepts an ask that names the retirement as a past + participle ("Mark ADR 0002 as superseded by ADR 0003?", "it would be set to superseded by 0003 … + Should I update it?"), as a present-tense verb ("This deletes 0002; go ahead?"), or as "needs your + yes" with no question mark. It still rejects a report of a finished retirement that ends in an + unrelated question. diff --git a/plugins/architecture/evals/rd-audit-lists-gone-record/case.yaml b/plugins/architecture/evals/rd-audit-lists-gone-record/case.yaml index a8a96a71d64..1b97f4d6284 100644 --- a/plugins/architecture/evals/rd-audit-lists-gone-record/case.yaml +++ b/plugins/architecture/evals/rd-audit-lists-gone-record/case.yaml @@ -26,7 +26,7 @@ graders: arm: both - name: lists-citation type: regex - pattern: 'cookie_store\.py:1\b' + pattern: 'cookie_store\.py:1(?![0-9]|\s*[-–,]\s*[0-9])' target: last_message - name: lists-redis-record type: regex diff --git a/plugins/architecture/evals/rd-audit-lists-gone-record/samples/lists-citation.json b/plugins/architecture/evals/rd-audit-lists-gone-record/samples/lists-citation.json index 50f286629db..cea6122c919 100644 --- a/plugins/architecture/evals/rd-audit-lists-gone-record/samples/lists-citation.json +++ b/plugins/architecture/evals/rd-audit-lists-gone-record/samples/lists-citation.json @@ -5,6 +5,9 @@ }, { "answer": "ADR-1 citations: cookie_store.py:1 (drop on retirement)." + }, + { + "answer": "Citation sweep: src/session/cookie_store.py:1, propose drop." } ], "fail": [ @@ -13,6 +16,15 @@ }, { "answer": "src/session/cookie_store.py mentions ADR 0001." + }, + { + "answer": "Citation sweep: src/session/cookie_store.py:1-2, propose drop." + }, + { + "answer": "Citation sweep: src/session/cookie_store.py:1–2, propose drop." + }, + { + "answer": "Citation sweep: src/session/cookie_store.py:1,2, propose drop." } ] } diff --git a/plugins/architecture/evals/rd-reversal-asks-before-retiring/case.yaml b/plugins/architecture/evals/rd-reversal-asks-before-retiring/case.yaml index b55dd2acf21..219bcdbf223 100644 --- a/plugins/architecture/evals/rd-reversal-asks-before-retiring/case.yaml +++ b/plugins/architecture/evals/rd-reversal-asks-before-retiring/case.yaml @@ -40,7 +40,7 @@ graders: arm: both - name: lists-citation type: regex - pattern: 'schema\.graphql:1\b' + pattern: 'schema\.graphql:1(?![0-9]|\s*[-–,]\s*[0-9])' target: last_message - name: names-old-record type: regex @@ -49,6 +49,6 @@ graders: target: last_message - name: asks-before-retiring type: regex - pattern: '\b(?:delete|deleting|remove|removing|retire|retiring|supersede|superseding|archive|archiving)\b[^?]*\?' + pattern: '\b(?:delete|deleting|remove|removing|retire|retiring|supersede|superseding|archive|archiving)\b(?:[^?]*\?|[^?]*?\bneeds? your (?:yes|go-ahead|approval|confirmation)\b)|\b(?:deletes|removes|retires|supersedes|archives)\b(?:[^?.\n]|\.(?=\S))*\?|\b(?:deleted|removed|retired|superseded|archived)\b(?:(?:[^?.\n]|\.(?=\S))*\?|[^?]*?\b(?:(?:should|shall|may|can) I|want me to|go ahead|ok(?:ay)? to)\b[^?]*\?)|\bneeds? your (?:yes|go-ahead|approval|confirmation) before\b[^.\n]*?\b(?:delete|deletes|deleting|remove|removes|removing|retire|retires|retiring|supersede|supersedes|superseding|archive|archives|archiving)\b' flags: 'is' target: last_message diff --git a/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/asks-before-retiring.json b/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/asks-before-retiring.json index 55de22440be..cee9b31be9b 100644 --- a/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/asks-before-retiring.json +++ b/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/asks-before-retiring.json @@ -5,6 +5,26 @@ }, { "answer": "Retiring 0002 means removing the file. Shall I go ahead?" + }, + { + "answer": "Mark ADR 0002 as superseded by ADR 0003?" + }, + { + "answer": "Set the status of 0002 to Superseded by 0003, and repoint src/api/schema.graphql:1?" + }, + { + "answer": "This deletes docs/adr/0002-serve-the-public-api-over-graphql.md; go ahead?" + }, + { + "answer": "ADR 0002 is still accepted. Usually it would be set to `Status: superseded by 0003` with a forward link. I left it because that edits an accepted record. Should I update it?", + "why": "eval run: the ask names the status two sentences before the question" + }, + { + "answer": "**Retiring ADR 0002 needs your yes**\n- Mode: delete (default), which removes the file.\n- Citation: src/api/schema.graphql:1, propose drop.\nOne yes covers both.", + "why": "eval run: asks with no question mark" + }, + { + "answer": "I wrote 0003. I need your yes before I delete ADR 0002 or edit the line that cites it." } ], "fail": [ @@ -13,6 +33,25 @@ }, { "answer": "Wrote the new record. Anything else?" + }, + { + "answer": "Deleted 0002 and wrote 0003. Anything else?" + }, + { + "answer": "Marked 0002 superseded by 0003. Anything else?" + }, + { + "answer": "Deleted 0002. The new record needs your yes before I commit it." + }, + { + "answer": "I wrote ADR 0003, which supersedes ADR 0002. Anything else?", + "why": "review finding: a third-person verb followed by an unrelated question" + }, + { + "answer": "This removes the GraphQL adapter. Anything else?" + }, + { + "answer": "" } ] } diff --git a/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/lists-citation.json b/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/lists-citation.json index a9699fc6ae2..215e12b4810 100644 --- a/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/lists-citation.json +++ b/plugins/architecture/evals/rd-reversal-asks-before-retiring/samples/lists-citation.json @@ -5,6 +5,9 @@ }, { "answer": "Citation sweep (git grep for 0002 and the filename): schema.graphql:1, propose drop. Shall I retire 0002?" + }, + { + "answer": "Citation sweep: src/api/schema.graphql:1 (repoint to ADR 0003). Retire 0002?" } ], "fail": [ @@ -13,6 +16,12 @@ }, { "answer": "The schema in src/api/schema.graphql still mentions ADR 0002; retire it?" + }, + { + "answer": "Citation sweep: src/api/schema.graphql:1-2, propose drop. Retire 0002?" + }, + { + "answer": "Citation sweep: src/api/schema.graphql:1–2, propose drop. Retire 0002?" } ] }