From fd3482ca09093a5f24b690cf494e73b9ca6f204b Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Thu, 17 Sep 2026 16:16:20 +0530 Subject: [PATCH 1/8] Fix agentic triage safe-output submission Allow jq payload preparation, document one-shot JSON submission, and validate triage summaries before safe-output writes. Preserve comment limits, label restrictions, and incomplete-result reporting. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 20 +++ .../tests/validate-triage-output.test.cjs | 149 ++++++++++++++++++ .github/scripts/validate-triage-output.cjs | 92 +++++++++++ .github/workflows/issue-triage.lock.yml | 30 +++- .github/workflows/issue-triage.md | 66 +++++++- .github/workflows/verify-aw-lock.yml | 6 + 6 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/tests/validate-triage-output.test.cjs create mode 100644 .github/scripts/validate-triage-output.cjs diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 9a4ea68906..32db891098 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -1,5 +1,25 @@ # GitHub Actions Script Tests +## Agentic triage output tests + +`validate-triage-output.test.cjs` uses the built-in Node.js test runner (Node.js +20 or newer) and `jq`, both preinstalled on the CI runner. No npm dependencies +are needed. Run it from the repository root: + +```bash +node --test .github/scripts/tests/validate-triage-output.test.cjs +``` + +These tests also run in `verify-aw-lock.yml`. They cover the validation gate +before triage comments and labels are published: complete summaries, explicit +no-op/failure results, placeholder comments, unfinished templates, quoted/fenced +examples, duplicate comments, and mixed success/incomplete outcomes. +They also exercise the workflow's exact `jq` encoding/check expressions with +quotes, backticks, backslashes, shell-like text, and newlines. Set `JQ_PATH` to +the full jq executable path if it is not on your local `PATH`. + +## Shell script tests + This directory contains tests for the shell scripts used by the [cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml), [check-milestone](./../../../.github/workflows/check-milestone.yml) and diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs new file mode 100644 index 0000000000..5475d06d6b --- /dev/null +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -0,0 +1,149 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); +const { validateTriageOutput } = require('../validate-triage-output.cjs'); + +const summary = `## \u{1F50D} Triage Summary + +| Check | Result | +|-------|--------| +| Issue type | Bug | +| Environment | Missing: SQL Server version | +| Area | Area\\Async | +| Duplicates | None found | +| Regression | Not indicated | + +### Analysis + +Cancellation leaves the operation running. Investigate the async cancellation path; P1. + +### Next Steps + +- Ask the author for their SQL Server version. +`; + +const output = (body) => ({ items: [{ type: 'add_comment', body }], errors: [] }); + +test('accepts initial, follow-up, and on-demand summaries with multiline content', () => { + for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) { + validateTriageOutput(output(summary.replace('Triage Summary', `Triage Summary${suffix}`))); + } + validateTriageOutput(output(summary.replaceAll('\n', '\r\n'))); +}); + +test('accepts a summary followed by an allowed label operation', () => { + const value = output(summary); + value.items.push({ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }); + validateTriageOutput(value); +}); + +test('preserves explicit no-op and failure reporting without a comment', () => { + for (const type of ['noop', 'report_incomplete', 'missing_tool', 'missing_data']) { + validateTriageOutput({ items: [{ type, reason: 'Required context unavailable' }] }); + } +}); + +test('rejects the placeholder bodies observed in failed runs, even with attribution', () => { + for (const body of ['-', '@-', 'test message please ignore', 'test simple body', '']) { + assert.throws(() => validateTriageOutput(output( + `${body}\n\n` + )), /Triage Summary/); + } +}); + +test('rejects quoted or fenced copies of the summary', () => { + for (const body of [ + summary.split('\n').map(line => `> ${line}`).join('\n'), + `\`\`\`markdown\n${summary}\n\`\`\``, + ]) { + assert.throws(() => validateTriageOutput(output(body)), /Triage Summary/); + } +}); + +test('requires every check row and non-placeholder values', () => { + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const row = summary.split('\n').find(line => line.startsWith(`| ${field} |`)); + for (const replacement of ['', `| ${field} | |`, `| ${field} | |`, `| ${field} | **TODO** |`]) { + assert.throws(() => validateTriageOutput(output(summary.replace(row, replacement))), /check row/); + } + } +}); + +test('requires populated Analysis and Next Steps sections', () => { + assert.throws(() => validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', '' + ))), /Analysis/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', '' + ))), /Next Steps/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '### Analysis', '### Explanation' + ))), /Analysis/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', + '\n> AI-generated summary.' + ))), /Next Steps/); +}); + +test('fenced examples cannot supply missing check rows or sections', () => { + const body = summary.replace( + '| Check | Result |', '```markdown\n| Check | Result |' + ).replace('### Next Steps', '```\n### Next Steps'); + assert.throws(() => validateTriageOutput(output(body)), /check row/); +}); + +test('rejects multiple comments and mixed success/incomplete outcomes', () => { + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], output(summary).items[0]], + }), /one comment/); + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], { type: 'report_incomplete', reason: 'Quota exceeded' }], + }), /incomplete/); + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], { type: 'noop', reason: 'No action needed' }], + }), /no-op/); +}); + +test('rejects missing output, empty results, and label-only results', () => { + for (const value of [null, {}, { items: null }, { items: [] }, { items: [null] }]) { + assert.throws(() => validateTriageOutput(value), /output|result/); + } + assert.throws(() => validateTriageOutput({ + items: [{ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }], + }), /summary/); +}); + +test('the documented jq commands preserve the complete Markdown payload', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'sqlclient-triage-')); + const markdown = path.join(directory, 'triage-summary.md'); + const payload = path.join(directory, 'triage-summary.json'); + const body = summary + + '\nPreserve "quotes", `backticks`, $HOME, $(not-a-command), and \\ paths.\n'; + const jq = process.env.JQ_PATH || 'jq'; + try { + fs.writeFileSync(markdown, body, 'utf8'); + const encode = spawnSync(jq, ['-Rs', '{body: .}', markdown], { encoding: 'utf8' }); + assert.ifError(encode.error); + assert.equal(encode.status, 0, encode.stderr); + fs.writeFileSync(payload, encode.stdout, 'utf8'); + const check = spawnSync(jq, [ + '-e', 'type == "object" and (.body | type == "string" and length > 0)', payload, + ], { encoding: 'utf8' }); + assert.ifError(check.error); + assert.equal(check.status, 0, check.stderr); + const result = JSON.parse(fs.readFileSync(payload, 'utf8')); + assert.deepEqual(result, { body }); + validateTriageOutput(output(result.body)); + } finally { + fs.rmSync(markdown, { force: true }); + fs.rmSync(payload, { force: true }); + fs.rmdirSync(directory); + } +}); diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs new file mode 100644 index 0000000000..745ab122c2 --- /dev/null +++ b/.github/scripts/validate-triage-output.cjs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +const fs = require('node:fs'); + +function proseOnly(body) { + let fence; + return body.replace(/|$)/g, '').split('\n').map(line => { + const marker = line.match(/^ {0,3}(`{3,}|~{3,})/); + if (fence) { + if (marker && marker[1][0] === fence[0] && + marker[1].length >= fence.length && line.trim() === marker[1]) { + fence = undefined; + } + return ''; + } + if (marker) { + fence = marker[1]; + return ''; + } + return /^(?: {0,3}>| {4}|\t)/.test(line) ? '' : line; + }).join('\n'); +} + +function meaningful(value) { + const text = value.replace(/[*_`]/g, '').trim(); + return /[\p{L}\p{N}]/u.test(text) && + !/^<[\s\S]*>$/.test(text) && + !/^(?:todo|tbd|test(?: message)?(?: please ignore)?|placeholder)$/i.test(text); +} + +function validateTriageOutput(output) { + if (!output || !Array.isArray(output.items) || output.items.length === 0 || + output.items.some(item => !item || typeof item.type !== 'string')) { + throw new Error('Missing or malformed triage output: expected non-empty result items.'); + } + + const comments = output.items.filter(item => item.type === 'add_comment'); + const incomplete = output.items.some(item => + ['report_incomplete', 'missing_tool', 'missing_data'].includes(item.type)); + const noop = output.items.some(item => item.type === 'noop'); + if (comments.length > 1) { + throw new Error('Triage must publish at most one comment.'); + } + if (comments.length === 0) { + if (!incomplete && !noop) { + throw new Error('Triage output has no summary or explicit no-op/incomplete result.'); + } + if (output.items.some(item => ['add_labels', 'remove_labels'].includes(item.type))) { + throw new Error('Triage cannot change labels without a validated summary.'); + } + return; + } + if (incomplete || noop) { + throw new Error('Triage cannot publish a summary alongside an incomplete or no-op result.'); + } + + const body = comments[0].body; + if (typeof body !== 'string' || + !/^## (?:\u{1F50D} )?Triage Summary(?: \(updated after author response\)| \(on-demand re-triage\))? *\r?\n/u.test(body.trim())) { + throw new Error('Comment must start with the completed Triage Summary heading.'); + } + const prose = proseOnly(body.replace(/\r\n?/g, '\n').trim()); + const table = prose.split(/^### /m)[0]; + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const row = table.split('\n').find(line => line.startsWith(`| ${field} |`)); + const match = row?.match(/^\| [^|]+ \| (.*?) \| *$/); + if (!match || !meaningful(match[1])) { + throw new Error(`Missing or unfinished triage check row: ${field}.`); + } + } + const sections = prose.split(/^### /m).slice(1); + for (const heading of ['Analysis', 'Next Steps']) { + const section = sections.find(value => value.split('\n', 1)[0].trim() === heading); + const content = section?.split('\n').slice(1).join('\n').split(/^#{1,3} /m)[0]; + if (!section || !meaningful(content)) { + throw new Error(`Missing or unfinished triage section: ${heading}.`); + } + } +} + +if (require.main === module) { + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('GH_AW_AGENT_OUTPUT is required; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + console.log('Triage output validated before safe-output publication.'); +} + +module.exports = { validateTriageOutput }; diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index d1ef3b2e67..0246bec46f 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8b81a4840372af279438b0250d6ae7168fa2e0a2f82ae8cd52d714f18a83d425","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0536852fc19f879532af2ce9eb87722351ee430b3f4043ee4dfc753103cd32ec","body_hash":"65940cca37f6ed65cd14cadae749cfc935f1edec13415434eec56c97f64057a1","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -900,6 +900,7 @@ jobs: # --allow-tool shell(github:*) # --allow-tool shell(grep) # --allow-tool shell(head) + # --allow-tool shell(jq) # --allow-tool shell(ls) # --allow-tool shell(printf) # --allow-tool shell(pwd) @@ -964,7 +965,7 @@ jobs: GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE @@ -1202,6 +1203,7 @@ jobs: environment: issue-triage permissions: actions: read + contents: read issues: write pull-requests: write concurrency: @@ -1742,6 +1744,7 @@ jobs: runs-on: ubuntu-slim environment: issue-triage permissions: + contents: read issues: write pull-requests: write timeout-minutes: 45 @@ -1820,6 +1823,29 @@ jobs: GH_HOST="${GITHUB_SERVER_URL#https://}" GH_HOST="${GH_HOST#http://}" echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Checkout trusted triage validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: triage-validator + persist-credentials: false + ref: ${{ github.sha }} + sparse-checkout: .github/scripts + - name: Validate triage output before publishing + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + with: + script: | + const path = require('node:path'); + const fs = require('node:fs'); + const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + core.info('Triage output validated before safe-output publication.'); - name: Process Safe Outputs id: process_safe_outputs uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index 100d8baf54..dc187cfbb7 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -51,11 +51,35 @@ permissions: pull-requests: read tools: - bash: [cat, find, grep] + bash: [cat, find, grep, jq] github: min-integrity: none safe-outputs: + steps: + - name: Checkout trusted triage validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: triage-validator + sparse-checkout: .github/scripts + persist-credentials: false + - name: Validate triage output before publishing + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + with: + script: | + const path = require('node:path'); + const fs = require('node:fs'); + const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + core.info('Triage output validated before safe-output publication.'); # One triage summary per run. `hide-older-comments` collapses previous # summaries so only the latest is visible. add-comment: @@ -116,6 +140,46 @@ Do NOT post intermediate findings. Do NOT post separate comments for area detection, duplicate checking, or environment validation. Everything goes into the single triage summary at the end. +## Safe-output submission + +Every successful `safeoutputs` call queues a real write; it is NOT a dry run. +Never test a write tool with a placeholder, `test` message, `-`, or `@-`. +Do not experiment with alternative comment calls after a submission error. + +Finish all analysis and compose the complete Markdown summary in +`/tmp/gh-aw/agent/triage-summary.md` using the file-writing tool. Before any +write intent, check that it starts with the Triage Summary heading, has all +five populated check rows, and contains meaningful Analysis and Next Steps +sections. Replace every template placeholder; do not wrap the summary in a +code fence or quote. + +Use the allowed `jq` command to JSON-encode the entire file without shell +interpolation of its contents: + +```bash +jq -Rs '{body: .}' /tmp/gh-aw/agent/triage-summary.md > /tmp/gh-aw/agent/triage-summary.json +jq -e 'type == "object" and (.body | type == "string" and length > 0)' /tmp/gh-aw/agent/triage-summary.json +``` + +Run these local preparation/check commands first and confirm both succeed. +Only then submit the finished payload exactly once: + +```bash +safeoutputs add_comment . < /tmp/gh-aw/agent/triage-summary.json +``` + +The standalone `.` reads a JSON object from stdin. `-` and `@-` are NOT +stdin sentinels. Do not use `--body -`, `--body @-`, hand-escaped Markdown +in shell arguments, or raw GitHub writes. The comment target defaults to +the triggering issue; do not override the target. + +If preparation or submission fails, stop without changing labels. Report +the failure through `safeoutputs report_incomplete` with the actual error; +do not call `add_comment` again, send a substitute comment, or claim success. +Only queue the permitted label action after the final comment is accepted. +The safe-output job validates the result before any comments or labels are +published and rejects placeholders or mixed success/incomplete outcomes. + --- ## Label-managed state diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml index eb3cc2e608..1b28bbba02 100644 --- a/.github/workflows/verify-aw-lock.yml +++ b/.github/workflows/verify-aw-lock.yml @@ -5,6 +5,9 @@ on: paths: - '.github/workflows/**/*.md' - '.github/workflows/**/*.lock.yml' + - '.github/workflows/verify-aw-lock.yml' + - '.github/scripts/validate-triage-output.cjs' + - '.github/scripts/tests/validate-triage-output.test.cjs' permissions: contents: read @@ -15,6 +18,9 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Test triage output validation + run: node --test .github/scripts/tests/validate-triage-output.test.cjs + - name: Install gh-aw extension uses: github/gh-aw-actions/setup-cli@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: From c761da278e3112d7faea57cb08988ae8c184fbc3 Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Mon, 28 Sep 2026 15:49:35 +0530 Subject: [PATCH 2/8] Address triage validator review feedback Reject partial safe-output batches, preserve valid C# generics while detecting embedded template placeholders, and prohibit comment resubmission after any attempt. Remove redundant payload-encoding coverage now provided by merged triage preparation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 8 +-- .../tests/validate-triage-output.test.cjs | 57 ++++++++----------- .github/scripts/validate-triage-output.cjs | 27 ++++++++- .github/workflows/issue-triage.lock.yml | 2 +- .github/workflows/issue-triage.md | 9 +-- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 32db891098..d9dafd68ca 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -3,8 +3,7 @@ ## Agentic triage output tests `validate-triage-output.test.cjs` uses the built-in Node.js test runner (Node.js -20 or newer) and `jq`, both preinstalled on the CI runner. No npm dependencies -are needed. Run it from the repository root: +20 or newer). No npm dependencies are needed. Run it from the repository root: ```bash node --test .github/scripts/tests/validate-triage-output.test.cjs @@ -14,9 +13,8 @@ These tests also run in `verify-aw-lock.yml`. They cover the validation gate before triage comments and labels are published: complete summaries, explicit no-op/failure results, placeholder comments, unfinished templates, quoted/fenced examples, duplicate comments, and mixed success/incomplete outcomes. -They also exercise the workflow's exact `jq` encoding/check expressions with -quotes, backticks, backslashes, shell-like text, and newlines. Set `JQ_PATH` to -the full jq executable path if it is not on your local `PATH`. +They also reject partial safe-output batches with recorded errors while +allowing C# generic type syntax in completed summaries. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index aef7fb93c9..98291d385c 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -3,10 +3,6 @@ // See the LICENSE file in the project root for more information. const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const { spawnSync } = require('node:child_process'); const { test } = require('node:test'); const { validateTriageOutput } = require('../validate-triage-output.cjs'); @@ -79,6 +75,7 @@ test('requires every check row and non-placeholder values', () => { test('rejects embedded template placeholders in rows and prose sections', () => { const unfinished = [ [summary.replace('Missing: SQL Server version', 'Missing: '), /check row/], + [summary.replace('Missing: SQL Server version', 'Missing: '), /check row/], [summary.replace( 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', 'Cancellation leaves the operation running. Investigate ; P1.' @@ -96,6 +93,18 @@ test('rejects embedded template placeholders in rows and prose sections', () => 'Cancellation leaves the operation running.', 'Cancellation leaves the operation running; SqlDataReader.GetFieldValue() is affected.' ))); + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running.', + 'Cancellation leaves the operation running; Dictionary maps the metadata.' + ))); + validateTriageOutput(output(summary.replace( + 'Missing: SQL Server version', + 'Missing: SQL Server version; Dictionary holds the available fields' + ))); + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running.', + 'Cancellation leaves the operation running; see for context.' + ))); }); test('requires populated Analysis and Next Steps sections', () => { @@ -133,6 +142,18 @@ test('rejects multiple comments and mixed success/incomplete outcomes', () => { }), /no-op/); }); +test('rejects output errors even when a complete summary is queued', () => { + const value = output(summary); + value.items.push({ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }); + value.errors.push({ type: 'remove_labels', message: 'Label operation exceeded quota' }); + assert.throws(() => validateTriageOutput(value), /errors/); + assert.throws(() => validateTriageOutput({ + items: [{ type: 'noop', reason: 'No new information' }], + errors: ['Malformed NDJSON line'], + }), /errors/); + assert.throws(() => validateTriageOutput({ ...output(summary), errors: null }), /errors/); +}); + test('rejects missing output, empty results, and label-only results', () => { for (const value of [null, {}, { items: null }, { items: [] }, { items: [null] }]) { assert.throws(() => validateTriageOutput(value), /output|result/); @@ -141,31 +162,3 @@ test('rejects missing output, empty results, and label-only results', () => { items: [{ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }], }), /summary/); }); - -test('the documented jq commands preserve the complete Markdown payload', () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'sqlclient-triage-')); - const markdown = path.join(directory, 'triage-summary.md'); - const payload = path.join(directory, 'triage-summary.json'); - const body = summary + - '\nPreserve "quotes", `backticks`, $HOME, $(not-a-command), and \\ paths.\n'; - const jq = process.env.JQ_PATH || 'jq'; - try { - fs.writeFileSync(markdown, body, 'utf8'); - const encode = spawnSync(jq, ['-Rs', '{body: .}', markdown], { encoding: 'utf8' }); - assert.ifError(encode.error); - assert.equal(encode.status, 0, encode.stderr); - fs.writeFileSync(payload, encode.stdout, 'utf8'); - const check = spawnSync(jq, [ - '-e', 'type == "object" and (.body | type == "string" and length > 0)', payload, - ], { encoding: 'utf8' }); - assert.ifError(check.error); - assert.equal(check.status, 0, check.stderr); - const result = JSON.parse(fs.readFileSync(payload, 'utf8')); - assert.deepEqual(result, { body }); - validateTriageOutput(output(result.body)); - } finally { - fs.rmSync(markdown, { force: true }); - fs.rmSync(payload, { force: true }); - fs.rmdirSync(directory); - } -}); diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index ea26dbcad5..5bd4e7eb57 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -4,8 +4,24 @@ const fs = require('node:fs'); -const TEMPLATE_PLACEHOLDER_PATTERN = - /<\s*(?:list|fields?|values?|authors?|types?|versions?|areas?|results?|details?|descriptions?|summaries?|items?|labels?|[^>\r\n]*\s[^>\r\n]*|[^>\r\n]*[|\/][^>\r\n]*)\s*>/i; +const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>\r\n]+)>/g; +const TEMPLATE_FIELD_PATTERN = + /^(?:list|fields?|values?|authors?|types?|versions?|areas?|results?|details?|descriptions?|summaries?|items?|labels?)$/i; +const GENERIC_ARGUMENTS_PATTERN = + /^[A-Za-z_][\w.?\[\]]*(?:\s*,\s*[A-Za-z_][\w.?\[\]]*)*$/; + +function hasTemplatePlaceholder(text) { + return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { + const contents = match[1]; + const value = contents.trim(); + if (contents !== value || /^(?:https?:\/\/|mailto:)\S+$/i.test(value)) { + return false; + } + return TEMPLATE_FIELD_PATTERN.test(value) || + !GENERIC_ARGUMENTS_PATTERN.test(value) || + !/[A-Za-z_][\w.]*$/.test(text.slice(0, match.index)); + }); +} function proseOnly(body) { let fence; @@ -29,7 +45,8 @@ function proseOnly(body) { function meaningful(value) { const text = value.replace(/[*_`]/g, '').trim(); return /[\p{L}\p{N}]/u.test(text) && - !TEMPLATE_PLACEHOLDER_PATTERN.test(text) && + !/^<[\s\S]*>$/.test(text) && + !hasTemplatePlaceholder(text) && !/^(?:todo|tbd|test(?: message)?(?: please ignore)?|placeholder)$/i.test(text); } @@ -38,6 +55,10 @@ function validateTriageOutput(output) { output.items.some(item => !item || typeof item.type !== 'string')) { throw new Error('Missing or malformed triage output: expected non-empty result items.'); } + if (output.errors !== undefined && + (!Array.isArray(output.errors) || output.errors.length > 0)) { + throw new Error('Triage output contains errors; refusing to publish a partial safe-output batch.'); + } const comments = output.items.filter(item => item.type === 'add_comment'); const incomplete = output.items.some(item => diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index d2aaaa315b..614f0b1d2b 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0536852fc19f879532af2ce9eb87722351ee430b3f4043ee4dfc753103cd32ec","body_hash":"076fe76be67ac00a0b84970cd54b7c3da40e97657c38b6cc0b6fae613b91901a","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0536852fc19f879532af2ce9eb87722351ee430b3f4043ee4dfc753103cd32ec","body_hash":"3b96e18888d3eed5db80c8f9ba8ff306377ed19256af3b2a6232f756c60e32ff","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index 9ad028718f..5682dbe674 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -154,10 +154,11 @@ For a multi-line summary saved to a temporary file, use the allowed `jq -Rs` command to construct the JSON payload with `item_number` and `body`, then submit it once through `safeoutputs add_comment .`. -If preparation or submission fails, inspect the error before retrying. -Do not repeatedly retry a denied command or probe with a write. If the -failure cannot be resolved with the available tools, call `report_incomplete` -with the actual error and stop without submitting placeholder content. +If preparation fails, inspect and correct it before attempting submission. +Once `add_comment` has been attempted, never retry it, even if its result is +ambiguous or reports an error. Report the actual error via `report_incomplete` +and stop without changing labels; do not probe with another write. Only +manage labels after the completed comment is accepted. --- From 5c0393d7424fb0740bb857146d49d61bc15b21ce Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Mon, 5 Oct 2026 14:59:29 +0530 Subject: [PATCH 3/8] Reject marker-prefixed triage drafts and document validation intent Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 4 +- .../tests/validate-triage-output.test.cjs | 38 +++++++++++++++++++ .github/scripts/validate-triage-output.cjs | 8 ++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index d9dafd68ca..c96a479c42 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -14,7 +14,9 @@ before triage comments and labels are published: complete summaries, explicit no-op/failure results, placeholder comments, unfinished templates, quoted/fenced examples, duplicate comments, and mixed success/incomplete outcomes. They also reject partial safe-output batches with recorded errors while -allowing C# generic type syntax in completed summaries. +allowing C# generic type syntax in completed summaries. Draft prefixes such as +`TODO:` and `TBD:` are rejected in required fields and list items; ordinary +mentions of those markers within completed prose remain valid. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 98291d385c..5cafa1b809 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -123,6 +123,44 @@ test('requires populated Analysis and Next Steps sections', () => { ))), /Next Steps/); }); +test('rejects marker-prefixed drafts in check rows and required sections', () => { + for (const draft of [ + 'TODO: replace this with the actual analysis.', + 'TBD: determine next steps.', + '**todo** investigate cancellation.', + '- TBD: determine next steps.', + '1. TODO: request reproduction details.', + 'Placeholder: insert findings here.', + ]) { + for (const [content, message] of [ + ['Missing: SQL Server version', /check row/], + ['Cancellation leaves the operation running. Investigate the async cancellation path; P1.', /Analysis/], + ['- Ask the author for their SQL Server version.', /Next Steps/], + ]) { + assert.throws(() => validateTriageOutput(output( + summary.replace(content, draft) + )), message); + } + } + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', + '- Ask the author for their SQL Server version.\n- TODO: decide the next action.' + ))), /Next Steps/); +}); + +test('accepts completed prose that mentions markers without using them as draft prefixes', () => { + for (const text of [ + 'The reproduction contains a TODO comment in application code.', + 'TodoList cancellation leaves the operation running.', + 'TODOs in the sample do not affect the reproduction.', + ]) { + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + text + ))); + } +}); + test('fenced examples cannot supply missing check rows or sections', () => { const body = summary.replace( '| Check | Result |', '```markdown\n| Check | Result |' diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 5bd4e7eb57..85d20d1deb 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +// Pre-publication gate for issue-triage safe outputs: reject incomplete summaries +// and partial batches before any queued comment or label change reaches GitHub. const fs = require('node:fs'); const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>\r\n]+)>/g; @@ -11,6 +13,7 @@ const GENERIC_ARGUMENTS_PATTERN = /^[A-Za-z_][\w.?\[\]]*(?:\s*,\s*[A-Za-z_][\w.?\[\]]*)*$/; function hasTemplatePlaceholder(text) { + // Distinguish unfilled template fields from legitimate C# generics and Markdown autolinks. return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { const contents = match[1]; const value = contents.trim(); @@ -24,6 +27,7 @@ function hasTemplatePlaceholder(text) { } function proseOnly(body) { + // Quoted examples, code blocks, and hidden attribution cannot satisfy required summary fields. let fence; return body.replace(/|$)/g, '').split('\n').map(line => { const marker = line.match(/^ {0,3}(`{3,}|~{3,})/); @@ -47,6 +51,7 @@ function meaningful(value) { return /[\p{L}\p{N}]/u.test(text) && !/^<[\s\S]*>$/.test(text) && !hasTemplatePlaceholder(text) && + !/^\s*(?:(?:[-+]|\d+[.)])\s+)?(?:todo|tbd|placeholder)\b/im.test(text) && !/^(?:todo|tbd|test(?: message)?(?: please ignore)?|placeholder)$/i.test(text); } @@ -60,6 +65,7 @@ function validateTriageOutput(output) { throw new Error('Triage output contains errors; refusing to publish a partial safe-output batch.'); } + // Explicit no-op/failure reports may proceed, but cannot accompany a summary or label changes. const comments = output.items.filter(item => item.type === 'add_comment'); const incomplete = output.items.some(item => ['report_incomplete', 'missing_tool', 'missing_data'].includes(item.type)); @@ -85,6 +91,7 @@ function validateTriageOutput(output) { !/^## (?:\u{1F50D} )?Triage Summary(?: \(updated after author response\)| \(on-demand re-triage\))? *\r?\n/u.test(body.trim())) { throw new Error('Comment must start with the completed Triage Summary heading.'); } + // Require the workflow's five check rows and both prose sections, not merely its heading. const prose = proseOnly(body.replace(/\r\n?/g, '\n').trim()); const table = prose.split(/^### /m)[0]; for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { @@ -105,6 +112,7 @@ function validateTriageOutput(output) { } if (require.main === module) { + // gh-aw supplies the complete queued batch here; throwing blocks safe-output publication. const filename = process.env.GH_AW_AGENT_OUTPUT; if (!filename) { throw new Error('GH_AW_AGENT_OUTPUT is required; refusing to publish unvalidated triage output.'); From 213ee332aa41073edcc4670309c6ab1f926aa77d Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Thu, 8 Oct 2026 12:40:03 +0530 Subject: [PATCH 4/8] Close triage validation ordering and incomplete-output gaps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 2 + .../tests/validate-triage-output.test.cjs | 37 +++++++++++++++++-- .github/scripts/validate-triage-output.cjs | 12 +++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index c96a479c42..159880e829 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -17,6 +17,8 @@ They also reject partial safe-output batches with recorded errors while allowing C# generic type syntax in completed summaries. Draft prefixes such as `TODO:` and `TBD:` are rejected in required fields and list items; ordinary mentions of those markers within completed prose remain valid. +Coverage also includes multiline template placeholders, label operations queued +before the summary, and contradictory no-op/incomplete results. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 5cafa1b809..527721cd2e 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -35,9 +35,19 @@ test('accepts initial, follow-up, and on-demand summaries with multiline content }); test('accepts a summary followed by an allowed label operation', () => { - const value = output(summary); - value.items.push({ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }); - validateTriageOutput(value); + for (const type of ['add_labels', 'remove_labels']) { + const value = output(summary); + value.items.push({ type, labels: ['Auto-Triage: Waiting for Author'] }); + validateTriageOutput(value); + } +}); + +test('rejects label operations before the summary', () => { + for (const type of ['add_labels', 'remove_labels']) { + const value = output(summary); + value.items.unshift({ type, labels: ['Auto-Triage: Waiting for Author'] }); + assert.throws(() => validateTriageOutput(value), /before/); + } }); test('preserves explicit no-op and failure reporting without a comment', () => { @@ -123,6 +133,22 @@ test('requires populated Analysis and Next Steps sections', () => { ))), /Next Steps/); }); +test('rejects multiline template placeholders embedded in completed-looking prose', () => { + for (const placeholder of [ + '<2-4 sentences: what the issue is about, which component is likely affected,\nand severity assessment (P0-P3)>', + '< missing\ndetails >', + ]) { + for (const [content, message] of [ + ['Cancellation leaves the operation running. Investigate the async cancellation path; P1.', /Analysis/], + ['- Ask the author for their SQL Server version.', /Next Steps/], + ]) { + const body = summary.replace(content, `Observed failure. ${placeholder}`); + assert.throws(() => validateTriageOutput(output(body)), message); + assert.throws(() => validateTriageOutput(output(body.replaceAll('\n', '\r\n'))), message); + } + } +}); + test('rejects marker-prefixed drafts in check rows and required sections', () => { for (const draft of [ 'TODO: replace this with the actual analysis.', @@ -178,6 +204,11 @@ test('rejects multiple comments and mixed success/incomplete outcomes', () => { assert.throws(() => validateTriageOutput({ items: [output(summary).items[0], { type: 'noop', reason: 'No action needed' }], }), /no-op/); + for (const type of ['report_incomplete', 'missing_tool', 'missing_data']) { + const items = [{ type: 'noop', reason: 'No action needed' }, { type, reason: 'Context unavailable' }]; + assert.throws(() => validateTriageOutput({ items }), /no-op.*incomplete/); + assert.throws(() => validateTriageOutput({ items: [...items].reverse() }), /no-op.*incomplete/); + } }); test('rejects output errors even when a complete summary is queued', () => { diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 85d20d1deb..29e7b471f7 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -6,7 +6,7 @@ // and partial batches before any queued comment or label change reaches GitHub. const fs = require('node:fs'); -const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>\r\n]+)>/g; +const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g; const TEMPLATE_FIELD_PATTERN = /^(?:list|fields?|values?|authors?|types?|versions?|areas?|results?|details?|descriptions?|summaries?|items?|labels?)$/i; const GENERIC_ARGUMENTS_PATTERN = @@ -17,7 +17,7 @@ function hasTemplatePlaceholder(text) { return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { const contents = match[1]; const value = contents.trim(); - if (contents !== value || /^(?:https?:\/\/|mailto:)\S+$/i.test(value)) { + if (/^(?:https?:\/\/|mailto:)\S+$/i.test(value)) { return false; } return TEMPLATE_FIELD_PATTERN.test(value) || @@ -73,6 +73,9 @@ function validateTriageOutput(output) { if (comments.length > 1) { throw new Error('Triage must publish at most one comment.'); } + if (incomplete && noop) { + throw new Error('Triage cannot combine no-op and incomplete results.'); + } if (comments.length === 0) { if (!incomplete && !noop) { throw new Error('Triage output has no summary or explicit no-op/incomplete result.'); @@ -85,6 +88,11 @@ function validateTriageOutput(output) { if (incomplete || noop) { throw new Error('Triage cannot publish a summary alongside an incomplete or no-op result.'); } + const commentIndex = output.items.indexOf(comments[0]); + if (output.items.slice(0, commentIndex).some(item => + ['add_labels', 'remove_labels'].includes(item.type))) { + throw new Error('Triage cannot change labels before its summary comment.'); + } const body = comments[0].body; if (typeof body !== 'string' || From 0f31cc7088ca74d418d0a367534a69f6a841ab56 Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Fri, 9 Oct 2026 14:20:16 +0530 Subject: [PATCH 5/8] Allow comparison prose in triage validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 2 ++ .../tests/validate-triage-output.test.cjs | 32 +++++++++++++++++++ .github/scripts/validate-triage-output.cjs | 18 ++++------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 159880e829..f41a73a8e7 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -19,6 +19,8 @@ allowing C# generic type syntax in completed summaries. Draft prefixes such as mentions of those markers within completed prose remain valid. Coverage also includes multiline template placeholders, label operations queued before the summary, and contradictory no-op/incomplete results. +Comparison operators in completed prose remain valid; angle-bracket detection +targets known template fields and instructions rather than arbitrary bracketed text. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 527721cd2e..5410d816a5 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -133,6 +133,38 @@ test('requires populated Analysis and Next Steps sections', () => { ))), /Next Steps/); }); +test('accepts comparison operators in completed triage prose', () => { + for (const text of [ + 'Rows with value < 10 and value > 0 fail.', + 'Rows with value<10 and value>0 fail.', + 'Rows with value < upperBound and value > lowerBound fail.', + 'Rows with value <= 10 and value >= 0 fail.', + ]) { + for (const content of [ + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + validateTriageOutput(output(summary.replace(content, text))); + } + } +}); + +test('rejects known workflow template instructions embedded in prose', () => { + for (const template of [ + '', + '', + '', + '', + '', + '', + ]) { + assert.throws(() => validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + `Observed failure. ${template}` + ))), /Analysis/); + } +}); + test('rejects multiline template placeholders embedded in completed-looking prose', () => { for (const placeholder of [ '<2-4 sentences: what the issue is about, which component is likely affected,\nand severity assessment (P0-P3)>', diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 29e7b471f7..07bfc89d2c 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -8,21 +8,15 @@ const fs = require('node:fs'); const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g; const TEMPLATE_FIELD_PATTERN = - /^(?:list|fields?|values?|authors?|types?|versions?|areas?|results?|details?|descriptions?|summaries?|items?|labels?)$/i; -const GENERIC_ARGUMENTS_PATTERN = - /^[A-Za-z_][\w.?\[\]]*(?:\s*,\s*[A-Za-z_][\w.?\[\]]*)*$/; + /^(?:(?:the|missing|affected)\s+)*(?:list|fields?|values?|authors?|types?|versions?|areas?|components?|results?|details?|descriptions?|summaries?|items?|labels?)$/i; +const TEMPLATE_INSTRUCTION_PATTERN = + /^(?:fill in\b|2-4 sentences:|Actionable items\.|Bug\s*\/|All required environment details\b|Best matching area\b|None found\s*\/|Not indicated\s*\/)/i; function hasTemplatePlaceholder(text) { - // Distinguish unfilled template fields from legitimate C# generics and Markdown autolinks. + // Match known template forms, not arbitrary angle brackets used by comparisons or C# generics. return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { - const contents = match[1]; - const value = contents.trim(); - if (/^(?:https?:\/\/|mailto:)\S+$/i.test(value)) { - return false; - } - return TEMPLATE_FIELD_PATTERN.test(value) || - !GENERIC_ARGUMENTS_PATTERN.test(value) || - !/[A-Za-z_][\w.]*$/.test(text.slice(0, match.index)); + const value = match[1].trim(); + return TEMPLATE_FIELD_PATTERN.test(value) || TEMPLATE_INSTRUCTION_PATTERN.test(value); }); } From fd1393134bf9a9d1d18f2e20983a536b4e4c6af6 Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Fri, 9 Oct 2026 23:23:16 +0530 Subject: [PATCH 6/8] Preserve generic and XML syntax in triage summaries Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 2 ++ .../tests/validate-triage-output.test.cjs | 19 +++++++++++++++++++ .github/scripts/validate-triage-output.cjs | 6 ++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index f41a73a8e7..03860f3599 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -21,6 +21,8 @@ Coverage also includes multiline template placeholders, label operations queued before the summary, and contradictory no-op/incomplete results. Comparison operators in completed prose remain valid; angle-bracket detection targets known template fields and instructions rather than arbitrary bracketed text. +Generic arguments such as `Task` and `List`, including nested +types, and XML documentation tags are covered by acceptance regressions. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 5410d816a5..879ec0acb3 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -149,6 +149,25 @@ test('accepts comparison operators in completed triage prose', () => { } }); +test('accepts generic types and XML documentation in completed prose', () => { + for (const text of [ + 'The failing method returns Task.', + 'The reproduction uses List.', + 'The reproduction uses Task>.', + 'The metadata is stored in Dictionary.', + 'The XML documentation contains a description tag.', + 'The XML payload contains 1.', + 'The component is wrapped in Task.', + ]) { + for (const content of [ + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + validateTriageOutput(output(summary.replace(content, text))); + } + } +}); + test('rejects known workflow template instructions embedded in prose', () => { for (const template of [ '', diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 07bfc89d2c..bbdf5135ad 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -8,7 +8,7 @@ const fs = require('node:fs'); const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g; const TEMPLATE_FIELD_PATTERN = - /^(?:(?:the|missing|affected)\s+)*(?:list|fields?|values?|authors?|types?|versions?|areas?|components?|results?|details?|descriptions?|summaries?|items?|labels?)$/i; + /^(?:list|component|(?:the\s+)?affected\s+component|missing\s+details)$/i; const TEMPLATE_INSTRUCTION_PATTERN = /^(?:fill in\b|2-4 sentences:|Actionable items\.|Bug\s*\/|All required environment details\b|Best matching area\b|None found\s*\/|Not indicated\s*\/)/i; @@ -16,7 +16,9 @@ function hasTemplatePlaceholder(text) { // Match known template forms, not arbitrary angle brackets used by comparisons or C# generics. return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { const value = match[1].trim(); - return TEMPLATE_FIELD_PATTERN.test(value) || TEMPLATE_INSTRUCTION_PATTERN.test(value); + const followsTypeName = /[A-Za-z_][\w.]*$/.test(text.slice(0, match.index)); + return TEMPLATE_INSTRUCTION_PATTERN.test(value) || + (!followsTypeName && TEMPLATE_FIELD_PATTERN.test(value)); }); } From af74dd4ec2c39f3764b51d1ed6c5ee63d446a8f8 Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Mon, 12 Oct 2026 11:59:45 +0530 Subject: [PATCH 7/8] Validate triage heading without stripping indentation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 2 ++ .github/scripts/tests/validate-triage-output.test.cjs | 6 ++++++ .github/scripts/validate-triage-output.cjs | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 03860f3599..f8092b6282 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -23,6 +23,8 @@ Comparison operators in completed prose remain valid; angle-bracket detection targets known template fields and instructions rather than arbitrary bracketed text. Generic arguments such as `Task` and `List`, including nested types, and XML documentation tags are covered by acceptance regressions. +The summary heading must start at the beginning of the body; leading indentation +cannot turn a code-block heading into an accepted Markdown heading. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 879ec0acb3..5bd3215d18 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -64,6 +64,12 @@ test('rejects the placeholder bodies observed in failed runs, even with attribut } }); +test('rejects indented headings even when the remaining summary is valid prose', () => { + for (const indentation of [' ', '\t', '\n ', '\r\n\t']) { + assert.throws(() => validateTriageOutput(output(indentation + summary)), /Triage Summary/); + } +}); + test('rejects quoted or fenced copies of the summary', () => { for (const body of [ summary.split('\n').map(line => `> ${line}`).join('\n'), diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index bbdf5135ad..65c8cff58a 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -92,7 +92,7 @@ function validateTriageOutput(output) { const body = comments[0].body; if (typeof body !== 'string' || - !/^## (?:\u{1F50D} )?Triage Summary(?: \(updated after author response\)| \(on-demand re-triage\))? *\r?\n/u.test(body.trim())) { + !/^## (?:\u{1F50D} )?Triage Summary(?: \(updated after author response\)| \(on-demand re-triage\))? *\r?\n/u.test(body)) { throw new Error('Comment must start with the completed Triage Summary heading.'); } // Require the workflow's five check rows and both prose sections, not merely its heading. From 42d95c20cf9665c5abfdf47d4663d88a538a0dae Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Mon, 12 Oct 2026 14:57:22 +0530 Subject: [PATCH 8/8] Enforce triage label state and reject remaining unfinished output Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 4 + .../tests/validate-triage-output.test.cjs | 90 ++++++++++++++++++- .github/scripts/validate-triage-output.cjs | 39 +++++++- 3 files changed, 127 insertions(+), 6 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index f8092b6282..ffbae37106 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -25,6 +25,10 @@ Generic arguments such as `Task` and `List`, including nested types, and XML documentation tags are covered by acceptance regressions. The summary heading must start at the beginning of the body; leading indentation cannot turn a code-block heading into an accepted Markdown heading. +Initial and follow-up summaries must queue exactly one Waiting for Author label +operation matching the Environment row; on-demand summaries must leave labels +unchanged. Tests also reject diagnostic placeholders within required fields, +duplicate required rows/sections, and unfinished content in additional prose. ## Shell script tests diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index 5bd3215d18..b2d593e6b7 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -25,7 +25,17 @@ Cancellation leaves the operation running. Investigate the async cancellation pa - Ask the author for their SQL Server version. `; -const output = (body) => ({ items: [{ type: 'add_comment', body }], errors: [] }); +const waitingLabel = 'Auto-Triage: Waiting for Author'; +const output = (body) => ({ + items: [ + { type: 'add_comment', body }, + ...(!body.includes('Triage Summary (on-demand re-triage)') ? [{ + type: /\| Environment \| (?:Missing:|.*Partial:)/.test(body) ? 'add_labels' : 'remove_labels', + labels: [waitingLabel], + }] : []), + ], + errors: [], +}); test('accepts initial, follow-up, and on-demand summaries with multiline content', () => { for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) { @@ -36,8 +46,9 @@ test('accepts initial, follow-up, and on-demand summaries with multiline content test('accepts a summary followed by an allowed label operation', () => { for (const type of ['add_labels', 'remove_labels']) { - const value = output(summary); - value.items.push({ type, labels: ['Auto-Triage: Waiting for Author'] }); + const body = type === 'add_labels' ? summary : + summary.replace('Missing: SQL Server version', 'All required environment details provided'); + const value = output(body); validateTriageOutput(value); } }); @@ -56,6 +67,79 @@ test('preserves explicit no-op and failure reporting without a comment', () => { } }); +test('enforces label state for initial, follow-up, and on-demand summaries', () => { + for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) { + for (const environment of [ + 'Missing: SQL Server version', + '\u26A0\uFE0F Missing: SQL Server version', + '\u26A0\uFE0F Partial: received OS; still missing SQL Server version', + 'All required environment details provided', + ]) { + const body = summary.replace('Triage Summary', `Triage Summary${suffix}`) + .replace('Missing: SQL Server version', environment); + const onDemand = suffix.includes('on-demand'); + const expected = /Missing:|Partial:/.test(environment) ? 'add_labels' : 'remove_labels'; + const comment = { type: 'add_comment', body }; + const label = { type: expected, labels: [waitingLabel] }; + validateTriageOutput({ items: onDemand ? [comment] : [comment, label] }); + const invalid = onDemand ? [ + [comment, label], + [comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }], + ] : [ + [comment], + [comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }], + [comment, label, label], + [comment, { ...label, labels: [] }], + [comment, { ...label, labels: ['Unrelated label'] }], + [comment, { ...label, labels: [waitingLabel, 'Unrelated label'] }], + ]; + for (const items of invalid) { + assert.throws(() => validateTriageOutput({ items }), /label/i); + } + } + } +}); + +test('rejects diagnostic placeholders inside required rows and sections', () => { + for (const content of [ + 'Bug', + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + assert.throws(() => validateTriageOutput(output(summary.replace( + content, 'test simple body' + ))), /unfinished/); + } +}); + +test('rejects duplicate required rows and sections', () => { + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const row = summary.split('\n').find(line => line.startsWith(`| ${field} |`)); + for (const duplicate of [row, `| ${field} | |`, `| ${field} | |`]) { + assert.throws(() => validateTriageOutput(output(summary.replace(row, `${row}\n${duplicate}`))), /duplicate/i); + assert.throws(() => validateTriageOutput(output( + summary + `\n### Additional Context\n\n${duplicate}\n` + )), /duplicate/i); + } + } + for (const heading of ['Analysis', 'Next Steps']) { + assert.throws(() => validateTriageOutput(output( + summary + `\n### ${heading}\n\nCompleted prose.\n` + )), /duplicate/i); + } +}); + +test('rejects unfinished content outside required fields without rejecting quoted examples', () => { + for (const draft of ['Additional context: ', 'TODO: finish the report', + '2-4 sentences: what the issue is about', 'Actionable items. Be specific, not vague.']) { + assert.throws(() => validateTriageOutput(output( + summary + `\n### Additional Context\n\n${draft}\n` + )), /unfinished/i); + validateTriageOutput(output(summary + `\n### Additional Context\n\n> ${draft}\n`)); + validateTriageOutput(output(summary + `\n### Additional Context\n\n\`\`\`text\n${draft}\n\`\`\`\n`)); + } +}); + test('rejects the placeholder bodies observed in failed runs, even with attribution', () => { for (const body of ['-', '@-', 'test message please ignore', 'test simple body', '']) { assert.throws(() => validateTriageOutput(output( diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 65c8cff58a..4cc0fb1049 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -48,7 +48,7 @@ function meaningful(value) { !/^<[\s\S]*>$/.test(text) && !hasTemplatePlaceholder(text) && !/^\s*(?:(?:[-+]|\d+[.)])\s+)?(?:todo|tbd|placeholder)\b/im.test(text) && - !/^(?:todo|tbd|test(?: message)?(?: please ignore)?|placeholder)$/i.test(text); + !/^(?:todo|tbd|test(?: message| simple body)?(?: please ignore)?|placeholder)$/i.test(text); } function validateTriageOutput(output) { @@ -98,21 +98,54 @@ function validateTriageOutput(output) { // Require the workflow's five check rows and both prose sections, not merely its heading. const prose = proseOnly(body.replace(/\r\n?/g, '\n').trim()); const table = prose.split(/^### /m)[0]; + let environment; for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { - const row = table.split('\n').find(line => line.startsWith(`| ${field} |`)); + const rows = table.split('\n').filter(line => line.startsWith(`| ${field} |`)); + if (prose.split('\n').filter(line => line.startsWith(`| ${field} |`)).length > 1) { + throw new Error(`Duplicate triage check row: ${field}.`); + } + const row = rows[0]; const match = row?.match(/^\| [^|]+ \| (.*?) \| *$/); if (!match || !meaningful(match[1])) { throw new Error(`Missing or unfinished triage check row: ${field}.`); } + if (field === 'Environment') { + environment = match[1].replace(/[*_`]/g, '').trim(); + } } const sections = prose.split(/^### /m).slice(1); for (const heading of ['Analysis', 'Next Steps']) { - const section = sections.find(value => value.split('\n', 1)[0].trim() === heading); + const matchingSections = sections.filter(value => value.split('\n', 1)[0].trim() === heading); + if (matchingSections.length > 1) { + throw new Error(`Duplicate triage section: ${heading}.`); + } + const section = matchingSections[0]; const content = section?.split('\n').slice(1).join('\n').split(/^#{1,3} /m)[0]; if (!section || !meaningful(content)) { throw new Error(`Missing or unfinished triage section: ${heading}.`); } } + // Extra prose must not reintroduce unfinished content outside the required fields. + if (!meaningful(prose) || + /^(?:\s*(?:[-+]|\d+[.)])\s+)?\s*(?:test simple body|2-4 sentences:|Actionable items\.)/im.test(prose)) { + throw new Error('Unfinished triage content outside the required fields.'); + } + + const labels = output.items.filter(item => ['add_labels', 'remove_labels'].includes(item.type)); + const onDemand = body.split('\n', 1)[0].includes('(on-demand re-triage)'); + if (onDemand) { + if (labels.length > 0) { + throw new Error('On-demand triage must preserve label state.'); + } + } else { + const missingEnvironment = /^(?:\u26A0\uFE0F?\s*)?(?:Missing|Partial):/iu.test(environment); + const expectedType = missingEnvironment ? 'add_labels' : 'remove_labels'; + if (labels.length !== 1 || labels[0].type !== expectedType || + !Array.isArray(labels[0].labels) || labels[0].labels.length !== 1 || + labels[0].labels[0] !== 'Auto-Triage: Waiting for Author') { + throw new Error(`Triage requires exactly one ${expectedType} operation for the Waiting for Author label.`); + } + } } if (require.main === module) {