diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 9a4ea68906..ffbae37106 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -1,5 +1,37 @@ # GitHub Actions Script Tests +## Agentic triage output tests + +`validate-triage-output.test.cjs` uses the built-in Node.js test runner (Node.js +20 or newer). No npm dependencies are needed. Run it from the repository root: + +```bash +node --test .github/scripts/tests/validate-triage-output.test.cjs +``` + +These tests also run in `verify-aw-lock.yml`. They cover the validation gate +before triage comments and labels are published: complete summaries, explicit +no-op/failure results, placeholder comments, unfinished templates, quoted/fenced +examples, duplicate comments, and mixed success/incomplete outcomes. +They also reject partial safe-output batches with recorded errors while +allowing C# generic type syntax in completed summaries. Draft prefixes such as +`TODO:` and `TBD:` are rejected in required fields and list items; ordinary +mentions of those markers within completed prose remain valid. +Coverage also includes multiline template placeholders, label operations queued +before the summary, and contradictory no-op/incomplete results. +Comparison operators in completed prose remain valid; angle-bracket detection +targets known template fields and instructions rather than arbitrary bracketed text. +Generic arguments such as `Task` and `List`, including nested +types, and XML documentation tags are covered by acceptance regressions. +The summary heading must start at the beginning of the body; leading indentation +cannot turn a code-block heading into an accepted Markdown heading. +Initial and follow-up summaries must queue exactly one Waiting for Author label +operation matching the Environment row; on-demand summaries must leave labels +unchanged. Tests also reject diagnostic placeholders within required fields, +duplicate required rows/sections, and unfinished content in additional prose. + +## Shell script tests + This directory contains tests for the shell scripts used by the [cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml), [check-milestone](./../../../.github/workflows/check-milestone.yml) and diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs new file mode 100644 index 0000000000..b2d593e6b7 --- /dev/null +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -0,0 +1,374 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { validateTriageOutput } = require('../validate-triage-output.cjs'); + +const summary = `## \u{1F50D} Triage Summary + +| Check | Result | +|-------|--------| +| Issue type | Bug | +| Environment | Missing: SQL Server version | +| Area | Area\\Async | +| Duplicates | None found | +| Regression | Not indicated | + +### Analysis + +Cancellation leaves the operation running. Investigate the async cancellation path; P1. + +### Next Steps + +- Ask the author for their SQL Server version. +`; + +const waitingLabel = 'Auto-Triage: Waiting for Author'; +const output = (body) => ({ + items: [ + { type: 'add_comment', body }, + ...(!body.includes('Triage Summary (on-demand re-triage)') ? [{ + type: /\| Environment \| (?:Missing:|.*Partial:)/.test(body) ? 'add_labels' : 'remove_labels', + labels: [waitingLabel], + }] : []), + ], + errors: [], +}); + +test('accepts initial, follow-up, and on-demand summaries with multiline content', () => { + for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) { + validateTriageOutput(output(summary.replace('Triage Summary', `Triage Summary${suffix}`))); + } + validateTriageOutput(output(summary.replaceAll('\n', '\r\n'))); +}); + +test('accepts a summary followed by an allowed label operation', () => { + for (const type of ['add_labels', 'remove_labels']) { + const body = type === 'add_labels' ? summary : + summary.replace('Missing: SQL Server version', 'All required environment details provided'); + const value = output(body); + validateTriageOutput(value); + } +}); + +test('rejects label operations before the summary', () => { + for (const type of ['add_labels', 'remove_labels']) { + const value = output(summary); + value.items.unshift({ type, labels: ['Auto-Triage: Waiting for Author'] }); + assert.throws(() => validateTriageOutput(value), /before/); + } +}); + +test('preserves explicit no-op and failure reporting without a comment', () => { + for (const type of ['noop', 'report_incomplete', 'missing_tool', 'missing_data']) { + validateTriageOutput({ items: [{ type, reason: 'Required context unavailable' }] }); + } +}); + +test('enforces label state for initial, follow-up, and on-demand summaries', () => { + for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) { + for (const environment of [ + 'Missing: SQL Server version', + '\u26A0\uFE0F Missing: SQL Server version', + '\u26A0\uFE0F Partial: received OS; still missing SQL Server version', + 'All required environment details provided', + ]) { + const body = summary.replace('Triage Summary', `Triage Summary${suffix}`) + .replace('Missing: SQL Server version', environment); + const onDemand = suffix.includes('on-demand'); + const expected = /Missing:|Partial:/.test(environment) ? 'add_labels' : 'remove_labels'; + const comment = { type: 'add_comment', body }; + const label = { type: expected, labels: [waitingLabel] }; + validateTriageOutput({ items: onDemand ? [comment] : [comment, label] }); + const invalid = onDemand ? [ + [comment, label], + [comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }], + ] : [ + [comment], + [comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }], + [comment, label, label], + [comment, { ...label, labels: [] }], + [comment, { ...label, labels: ['Unrelated label'] }], + [comment, { ...label, labels: [waitingLabel, 'Unrelated label'] }], + ]; + for (const items of invalid) { + assert.throws(() => validateTriageOutput({ items }), /label/i); + } + } + } +}); + +test('rejects diagnostic placeholders inside required rows and sections', () => { + for (const content of [ + 'Bug', + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + assert.throws(() => validateTriageOutput(output(summary.replace( + content, 'test simple body' + ))), /unfinished/); + } +}); + +test('rejects duplicate required rows and sections', () => { + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const row = summary.split('\n').find(line => line.startsWith(`| ${field} |`)); + for (const duplicate of [row, `| ${field} | |`, `| ${field} | |`]) { + assert.throws(() => validateTriageOutput(output(summary.replace(row, `${row}\n${duplicate}`))), /duplicate/i); + assert.throws(() => validateTriageOutput(output( + summary + `\n### Additional Context\n\n${duplicate}\n` + )), /duplicate/i); + } + } + for (const heading of ['Analysis', 'Next Steps']) { + assert.throws(() => validateTriageOutput(output( + summary + `\n### ${heading}\n\nCompleted prose.\n` + )), /duplicate/i); + } +}); + +test('rejects unfinished content outside required fields without rejecting quoted examples', () => { + for (const draft of ['Additional context: ', 'TODO: finish the report', + '2-4 sentences: what the issue is about', 'Actionable items. Be specific, not vague.']) { + assert.throws(() => validateTriageOutput(output( + summary + `\n### Additional Context\n\n${draft}\n` + )), /unfinished/i); + validateTriageOutput(output(summary + `\n### Additional Context\n\n> ${draft}\n`)); + validateTriageOutput(output(summary + `\n### Additional Context\n\n\`\`\`text\n${draft}\n\`\`\`\n`)); + } +}); + +test('rejects the placeholder bodies observed in failed runs, even with attribution', () => { + for (const body of ['-', '@-', 'test message please ignore', 'test simple body', '']) { + assert.throws(() => validateTriageOutput(output( + `${body}\n\n` + )), /Triage Summary/); + } +}); + +test('rejects indented headings even when the remaining summary is valid prose', () => { + for (const indentation of [' ', '\t', '\n ', '\r\n\t']) { + assert.throws(() => validateTriageOutput(output(indentation + summary)), /Triage Summary/); + } +}); + +test('rejects quoted or fenced copies of the summary', () => { + for (const body of [ + summary.split('\n').map(line => `> ${line}`).join('\n'), + `\`\`\`markdown\n${summary}\n\`\`\``, + ]) { + assert.throws(() => validateTriageOutput(output(body)), /Triage Summary/); + } +}); + +test('requires every check row and non-placeholder values', () => { + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const row = summary.split('\n').find(line => line.startsWith(`| ${field} |`)); + for (const replacement of ['', `| ${field} | |`, `| ${field} | |`, `| ${field} | **TODO** |`]) { + assert.throws(() => validateTriageOutput(output(summary.replace(row, replacement))), /check row/); + } + } +}); + +test('rejects embedded template placeholders in rows and prose sections', () => { + const unfinished = [ + [summary.replace('Missing: SQL Server version', 'Missing: '), /check row/], + [summary.replace('Missing: SQL Server version', 'Missing: '), /check row/], + [summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + 'Cancellation leaves the operation running. Investigate ; P1.' + ), /Analysis/], + [summary.replace( + '- Ask the author for their SQL Server version.', + '- Ask the author for their .' + ), /Next Steps/], + ]; + for (const [body, message] of unfinished) { + assert.throws(() => validateTriageOutput(output(body)), message); + } + + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running.', + 'Cancellation leaves the operation running; SqlDataReader.GetFieldValue() is affected.' + ))); + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running.', + 'Cancellation leaves the operation running; Dictionary maps the metadata.' + ))); + validateTriageOutput(output(summary.replace( + 'Missing: SQL Server version', + 'Missing: SQL Server version; Dictionary holds the available fields' + ))); + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running.', + 'Cancellation leaves the operation running; see for context.' + ))); +}); + +test('requires populated Analysis and Next Steps sections', () => { + assert.throws(() => validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', '' + ))), /Analysis/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', '' + ))), /Next Steps/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '### Analysis', '### Explanation' + ))), /Analysis/); + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', + '\n> AI-generated summary.' + ))), /Next Steps/); +}); + +test('accepts comparison operators in completed triage prose', () => { + for (const text of [ + 'Rows with value < 10 and value > 0 fail.', + 'Rows with value<10 and value>0 fail.', + 'Rows with value < upperBound and value > lowerBound fail.', + 'Rows with value <= 10 and value >= 0 fail.', + ]) { + for (const content of [ + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + validateTriageOutput(output(summary.replace(content, text))); + } + } +}); + +test('accepts generic types and XML documentation in completed prose', () => { + for (const text of [ + 'The failing method returns Task.', + 'The reproduction uses List.', + 'The reproduction uses Task>.', + 'The metadata is stored in Dictionary.', + 'The XML documentation contains a description tag.', + 'The XML payload contains 1.', + 'The component is wrapped in Task.', + ]) { + for (const content of [ + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + '- Ask the author for their SQL Server version.', + ]) { + validateTriageOutput(output(summary.replace(content, text))); + } + } +}); + +test('rejects known workflow template instructions embedded in prose', () => { + for (const template of [ + '', + '', + '', + '', + '', + '', + ]) { + assert.throws(() => validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + `Observed failure. ${template}` + ))), /Analysis/); + } +}); + +test('rejects multiline template placeholders embedded in completed-looking prose', () => { + for (const placeholder of [ + '<2-4 sentences: what the issue is about, which component is likely affected,\nand severity assessment (P0-P3)>', + '< missing\ndetails >', + ]) { + for (const [content, message] of [ + ['Cancellation leaves the operation running. Investigate the async cancellation path; P1.', /Analysis/], + ['- Ask the author for their SQL Server version.', /Next Steps/], + ]) { + const body = summary.replace(content, `Observed failure. ${placeholder}`); + assert.throws(() => validateTriageOutput(output(body)), message); + assert.throws(() => validateTriageOutput(output(body.replaceAll('\n', '\r\n'))), message); + } + } +}); + +test('rejects marker-prefixed drafts in check rows and required sections', () => { + for (const draft of [ + 'TODO: replace this with the actual analysis.', + 'TBD: determine next steps.', + '**todo** investigate cancellation.', + '- TBD: determine next steps.', + '1. TODO: request reproduction details.', + 'Placeholder: insert findings here.', + ]) { + for (const [content, message] of [ + ['Missing: SQL Server version', /check row/], + ['Cancellation leaves the operation running. Investigate the async cancellation path; P1.', /Analysis/], + ['- Ask the author for their SQL Server version.', /Next Steps/], + ]) { + assert.throws(() => validateTriageOutput(output( + summary.replace(content, draft) + )), message); + } + } + assert.throws(() => validateTriageOutput(output(summary.replace( + '- Ask the author for their SQL Server version.', + '- Ask the author for their SQL Server version.\n- TODO: decide the next action.' + ))), /Next Steps/); +}); + +test('accepts completed prose that mentions markers without using them as draft prefixes', () => { + for (const text of [ + 'The reproduction contains a TODO comment in application code.', + 'TodoList cancellation leaves the operation running.', + 'TODOs in the sample do not affect the reproduction.', + ]) { + validateTriageOutput(output(summary.replace( + 'Cancellation leaves the operation running. Investigate the async cancellation path; P1.', + text + ))); + } +}); + +test('fenced examples cannot supply missing check rows or sections', () => { + const body = summary.replace( + '| Check | Result |', '```markdown\n| Check | Result |' + ).replace('### Next Steps', '```\n### Next Steps'); + assert.throws(() => validateTriageOutput(output(body)), /check row/); +}); + +test('rejects multiple comments and mixed success/incomplete outcomes', () => { + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], output(summary).items[0]], + }), /one comment/); + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], { type: 'report_incomplete', reason: 'Quota exceeded' }], + }), /incomplete/); + assert.throws(() => validateTriageOutput({ + items: [output(summary).items[0], { type: 'noop', reason: 'No action needed' }], + }), /no-op/); + for (const type of ['report_incomplete', 'missing_tool', 'missing_data']) { + const items = [{ type: 'noop', reason: 'No action needed' }, { type, reason: 'Context unavailable' }]; + assert.throws(() => validateTriageOutput({ items }), /no-op.*incomplete/); + assert.throws(() => validateTriageOutput({ items: [...items].reverse() }), /no-op.*incomplete/); + } +}); + +test('rejects output errors even when a complete summary is queued', () => { + const value = output(summary); + value.items.push({ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }); + value.errors.push({ type: 'remove_labels', message: 'Label operation exceeded quota' }); + assert.throws(() => validateTriageOutput(value), /errors/); + assert.throws(() => validateTriageOutput({ + items: [{ type: 'noop', reason: 'No new information' }], + errors: ['Malformed NDJSON line'], + }), /errors/); + assert.throws(() => validateTriageOutput({ ...output(summary), errors: null }), /errors/); +}); + +test('rejects missing output, empty results, and label-only results', () => { + for (const value of [null, {}, { items: null }, { items: [] }, { items: [null] }]) { + assert.throws(() => validateTriageOutput(value), /output|result/); + } + assert.throws(() => validateTriageOutput({ + items: [{ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }], + }), /summary/); +}); diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs new file mode 100644 index 0000000000..4cc0fb1049 --- /dev/null +++ b/.github/scripts/validate-triage-output.cjs @@ -0,0 +1,161 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// Pre-publication gate for issue-triage safe outputs: reject incomplete summaries +// and partial batches before any queued comment or label change reaches GitHub. +const fs = require('node:fs'); + +const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g; +const TEMPLATE_FIELD_PATTERN = + /^(?:list|component|(?:the\s+)?affected\s+component|missing\s+details)$/i; +const TEMPLATE_INSTRUCTION_PATTERN = + /^(?:fill in\b|2-4 sentences:|Actionable items\.|Bug\s*\/|All required environment details\b|Best matching area\b|None found\s*\/|Not indicated\s*\/)/i; + +function hasTemplatePlaceholder(text) { + // Match known template forms, not arbitrary angle brackets used by comparisons or C# generics. + return [...text.matchAll(TEMPLATE_PLACEHOLDER_PATTERN)].some((match) => { + const value = match[1].trim(); + const followsTypeName = /[A-Za-z_][\w.]*$/.test(text.slice(0, match.index)); + return TEMPLATE_INSTRUCTION_PATTERN.test(value) || + (!followsTypeName && TEMPLATE_FIELD_PATTERN.test(value)); + }); +} + +function proseOnly(body) { + // Quoted examples, code blocks, and hidden attribution cannot satisfy required summary fields. + let fence; + return body.replace(/|$)/g, '').split('\n').map(line => { + const marker = line.match(/^ {0,3}(`{3,}|~{3,})/); + if (fence) { + if (marker && marker[1][0] === fence[0] && + marker[1].length >= fence.length && line.trim() === marker[1]) { + fence = undefined; + } + return ''; + } + if (marker) { + fence = marker[1]; + return ''; + } + return /^(?: {0,3}>| {4}|\t)/.test(line) ? '' : line; + }).join('\n'); +} + +function meaningful(value) { + const text = value.replace(/[*_`]/g, '').trim(); + return /[\p{L}\p{N}]/u.test(text) && + !/^<[\s\S]*>$/.test(text) && + !hasTemplatePlaceholder(text) && + !/^\s*(?:(?:[-+]|\d+[.)])\s+)?(?:todo|tbd|placeholder)\b/im.test(text) && + !/^(?:todo|tbd|test(?: message| simple body)?(?: please ignore)?|placeholder)$/i.test(text); +} + +function validateTriageOutput(output) { + if (!output || !Array.isArray(output.items) || output.items.length === 0 || + output.items.some(item => !item || typeof item.type !== 'string')) { + throw new Error('Missing or malformed triage output: expected non-empty result items.'); + } + if (output.errors !== undefined && + (!Array.isArray(output.errors) || output.errors.length > 0)) { + throw new Error('Triage output contains errors; refusing to publish a partial safe-output batch.'); + } + + // Explicit no-op/failure reports may proceed, but cannot accompany a summary or label changes. + const comments = output.items.filter(item => item.type === 'add_comment'); + const incomplete = output.items.some(item => + ['report_incomplete', 'missing_tool', 'missing_data'].includes(item.type)); + const noop = output.items.some(item => item.type === 'noop'); + if (comments.length > 1) { + throw new Error('Triage must publish at most one comment.'); + } + if (incomplete && noop) { + throw new Error('Triage cannot combine no-op and incomplete results.'); + } + if (comments.length === 0) { + if (!incomplete && !noop) { + throw new Error('Triage output has no summary or explicit no-op/incomplete result.'); + } + if (output.items.some(item => ['add_labels', 'remove_labels'].includes(item.type))) { + throw new Error('Triage cannot change labels without a validated summary.'); + } + return; + } + if (incomplete || noop) { + throw new Error('Triage cannot publish a summary alongside an incomplete or no-op result.'); + } + const commentIndex = output.items.indexOf(comments[0]); + if (output.items.slice(0, commentIndex).some(item => + ['add_labels', 'remove_labels'].includes(item.type))) { + throw new Error('Triage cannot change labels before its summary comment.'); + } + + const body = comments[0].body; + if (typeof body !== 'string' || + !/^## (?:\u{1F50D} )?Triage Summary(?: \(updated after author response\)| \(on-demand re-triage\))? *\r?\n/u.test(body)) { + throw new Error('Comment must start with the completed Triage Summary heading.'); + } + // Require the workflow's five check rows and both prose sections, not merely its heading. + const prose = proseOnly(body.replace(/\r\n?/g, '\n').trim()); + const table = prose.split(/^### /m)[0]; + let environment; + for (const field of ['Issue type', 'Environment', 'Area', 'Duplicates', 'Regression']) { + const rows = table.split('\n').filter(line => line.startsWith(`| ${field} |`)); + if (prose.split('\n').filter(line => line.startsWith(`| ${field} |`)).length > 1) { + throw new Error(`Duplicate triage check row: ${field}.`); + } + const row = rows[0]; + const match = row?.match(/^\| [^|]+ \| (.*?) \| *$/); + if (!match || !meaningful(match[1])) { + throw new Error(`Missing or unfinished triage check row: ${field}.`); + } + if (field === 'Environment') { + environment = match[1].replace(/[*_`]/g, '').trim(); + } + } + const sections = prose.split(/^### /m).slice(1); + for (const heading of ['Analysis', 'Next Steps']) { + const matchingSections = sections.filter(value => value.split('\n', 1)[0].trim() === heading); + if (matchingSections.length > 1) { + throw new Error(`Duplicate triage section: ${heading}.`); + } + const section = matchingSections[0]; + const content = section?.split('\n').slice(1).join('\n').split(/^#{1,3} /m)[0]; + if (!section || !meaningful(content)) { + throw new Error(`Missing or unfinished triage section: ${heading}.`); + } + } + // Extra prose must not reintroduce unfinished content outside the required fields. + if (!meaningful(prose) || + /^(?:\s*(?:[-+]|\d+[.)])\s+)?\s*(?:test simple body|2-4 sentences:|Actionable items\.)/im.test(prose)) { + throw new Error('Unfinished triage content outside the required fields.'); + } + + const labels = output.items.filter(item => ['add_labels', 'remove_labels'].includes(item.type)); + const onDemand = body.split('\n', 1)[0].includes('(on-demand re-triage)'); + if (onDemand) { + if (labels.length > 0) { + throw new Error('On-demand triage must preserve label state.'); + } + } else { + const missingEnvironment = /^(?:\u26A0\uFE0F?\s*)?(?:Missing|Partial):/iu.test(environment); + const expectedType = missingEnvironment ? 'add_labels' : 'remove_labels'; + if (labels.length !== 1 || labels[0].type !== expectedType || + !Array.isArray(labels[0].labels) || labels[0].labels.length !== 1 || + labels[0].labels[0] !== 'Auto-Triage: Waiting for Author') { + throw new Error(`Triage requires exactly one ${expectedType} operation for the Waiting for Author label.`); + } + } +} + +if (require.main === module) { + // gh-aw supplies the complete queued batch here; throwing blocks safe-output publication. + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('GH_AW_AGENT_OUTPUT is required; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + console.log('Triage output validated before safe-output publication.'); +} + +module.exports = { validateTriageOutput }; diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index b747ae5480..614f0b1d2b 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"15a5f04fd64fafbae521cc59a430429c7c6a18ae0eaf32087998a7c23a84a572","body_hash":"076fe76be67ac00a0b84970cd54b7c3da40e97657c38b6cc0b6fae613b91901a","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0536852fc19f879532af2ce9eb87722351ee430b3f4043ee4dfc753103cd32ec","body_hash":"3b96e18888d3eed5db80c8f9ba8ff306377ed19256af3b2a6232f756c60e32ff","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1203,6 +1203,7 @@ jobs: environment: issue-triage permissions: actions: read + contents: read issues: write pull-requests: write concurrency: @@ -1743,6 +1744,7 @@ jobs: runs-on: ubuntu-slim environment: issue-triage permissions: + contents: read issues: write pull-requests: write timeout-minutes: 45 @@ -1821,6 +1823,29 @@ jobs: GH_HOST="${GITHUB_SERVER_URL#https://}" GH_HOST="${GH_HOST#http://}" echo "GH_HOST=${GH_HOST}" >> "$GITHUB_ENV" + - name: Checkout trusted triage validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: triage-validator + persist-credentials: false + ref: ${{ github.sha }} + sparse-checkout: .github/scripts + - name: Validate triage output before publishing + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + with: + script: | + const path = require('node:path'); + const fs = require('node:fs'); + const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + core.info('Triage output validated before safe-output publication.'); - name: Process Safe Outputs id: process_safe_outputs uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index 2e1c42ac30..5682dbe674 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -56,6 +56,30 @@ tools: min-integrity: none safe-outputs: + steps: + - name: Checkout trusted triage validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: triage-validator + sparse-checkout: .github/scripts + persist-credentials: false + - name: Validate triage output before publishing + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + with: + script: | + const path = require('node:path'); + const fs = require('node:fs'); + const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); + const filename = process.env.GH_AW_AGENT_OUTPUT; + if (!filename) { + throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); + } + validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + core.info('Triage output validated before safe-output publication.'); # One triage summary per run. `hide-older-comments` collapses previous # summaries so only the latest is visible. add-comment: @@ -130,10 +154,11 @@ For a multi-line summary saved to a temporary file, use the allowed `jq -Rs` command to construct the JSON payload with `item_number` and `body`, then submit it once through `safeoutputs add_comment .`. -If preparation or submission fails, inspect the error before retrying. -Do not repeatedly retry a denied command or probe with a write. If the -failure cannot be resolved with the available tools, call `report_incomplete` -with the actual error and stop without submitting placeholder content. +If preparation fails, inspect and correct it before attempting submission. +Once `add_comment` has been attempted, never retry it, even if its result is +ambiguous or reports an error. Report the actual error via `report_incomplete` +and stop without changing labels; do not probe with another write. Only +manage labels after the completed comment is accepted. --- diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml index eb3cc2e608..1b28bbba02 100644 --- a/.github/workflows/verify-aw-lock.yml +++ b/.github/workflows/verify-aw-lock.yml @@ -5,6 +5,9 @@ on: paths: - '.github/workflows/**/*.md' - '.github/workflows/**/*.lock.yml' + - '.github/workflows/verify-aw-lock.yml' + - '.github/scripts/validate-triage-output.cjs' + - '.github/scripts/tests/validate-triage-output.test.cjs' permissions: contents: read @@ -15,6 +18,9 @@ jobs: steps: - uses: actions/checkout@v6 + - name: Test triage output validation + run: node --test .github/scripts/tests/validate-triage-output.test.cjs + - name: Install gh-aw extension uses: github/gh-aw-actions/setup-cli@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: