From 8c634a9c0ba3bce8415e4fca85f22a271c619417 Mon Sep 17 00:00:00 2001 From: Jason Kirtland Date: Mon, 5 Oct 2026 07:57:46 -0700 Subject: [PATCH] Lead the instructions question with the agent check count --- .../src/cli/services/security-check.test.ts | 12 +++++-- .../app/src/cli/services/security-check.ts | 35 ++++++++++++------- .../src/cli/services/security-output.test.ts | 8 ++--- .../app/src/cli/services/security-output.ts | 7 ++-- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/packages/app/src/cli/services/security-check.test.ts b/packages/app/src/cli/services/security-check.test.ts index a4011ad2f8b..a3ec745120c 100644 --- a/packages/app/src/cli/services/security-check.test.ts +++ b/packages/app/src/cli/services/security-check.test.ts @@ -522,8 +522,9 @@ describe('securityCheck', () => { await securityCheck(testOptions(), dependencies) - expect(appSecurityInstructionsPrompt).toEqual({ - message: 'How would you like to hand the results to your coding agent?', + expect(appSecurityInstructionsPrompt(31)).toEqual({ + message: + '31 recommended agent checks available to complete your scan. How do you want to pass that prompt to your agent?', choices: [ {label: 'Copy instructions to the clipboard', value: 'copy'}, {label: 'Print instructions to the terminal', value: 'print'}, @@ -532,6 +533,7 @@ describe('securityCheck', () => { defaultValue: 'copy', }) expect(dependencies.selectInstructionsDestination).toHaveBeenCalledOnce() + expect(dependencies.selectInstructionsDestination).toHaveBeenCalledWith(31) expect(dependencies.deliverInstructions).toHaveBeenCalledWith({ appDirectory, resultsKey: 'shopify.app', @@ -541,6 +543,12 @@ describe('securityCheck', () => { }) }) + test('names a single recommended agent check in the singular', () => { + expect(appSecurityInstructionsPrompt(1).message).toBe( + '1 recommended agent check available to complete your scan. How do you want to pass that prompt to your agent?', + ) + }) + test('gives the instructions the exact scope of the run, as typed', async () => { await inTemporaryDirectory(async (directory) => { await mkdir(joinPath(directory, 'backend')) diff --git a/packages/app/src/cli/services/security-check.ts b/packages/app/src/cli/services/security-check.ts index 017b398ad7e..e40e550fe04 100644 --- a/packages/app/src/cli/services/security-check.ts +++ b/packages/app/src/cli/services/security-check.ts @@ -76,7 +76,7 @@ interface SecurityDependencies { artifacts: {deterministicFindings: DeterministicFindingsDocument; agentChecks: AgentChecks}, ): Promise canPrompt(): boolean - selectInstructionsDestination(): Promise + selectInstructionsDestination(agentCheckCount: number): Promise deliverInstructions(options: { appDirectory: string resultsKey: string @@ -91,14 +91,18 @@ interface SecurityDependencies { setExitCode(exitCode: number): void } -export const appSecurityInstructionsPrompt: RenderSelectPromptOptions = { - message: 'How would you like to hand the results to your coding agent?', - choices: [ - {label: 'Copy instructions to the clipboard', value: 'copy'}, - {label: 'Print instructions to the terminal', value: 'print'}, - {label: 'Nothing', value: 'nothing'}, - ], - defaultValue: 'copy', +export function appSecurityInstructionsPrompt( + agentCheckCount: number, +): RenderSelectPromptOptions { + return { + message: `${agentCheckCount} recommended agent ${agentCheckCount === 1 ? 'check' : 'checks'} available to complete your scan. How do you want to pass that prompt to your agent?`, + choices: [ + {label: 'Copy instructions to the clipboard', value: 'copy'}, + {label: 'Print instructions to the terminal', value: 'print'}, + {label: 'Nothing', value: 'nothing'}, + ], + defaultValue: 'copy', + } } const defaultDependencies: SecurityDependencies = { @@ -107,7 +111,8 @@ const defaultDependencies: SecurityDependencies = { listFiles: listAppSecurityFiles, writeArtifacts: writeCheckArtifacts, canPrompt: terminalSupportsPrompting, - selectInstructionsDestination: () => renderSelectPrompt(appSecurityInstructionsPrompt), + selectInstructionsDestination: (agentCheckCount) => + renderSelectPrompt(appSecurityInstructionsPrompt(agentCheckCount)), deliverInstructions: deliverAppSecurityInstructions, output: outputResult, renderInfo, @@ -122,11 +127,12 @@ async function instructionsDestination( options: SecurityOptions, dependencies: SecurityDependencies, canPrompt: boolean, + agentCheckCount: number, ): Promise { if (options.json || options.skipInstructions) return 'nothing' if (options.yes) return 'print' if (!canPrompt) return 'nothing' - return dependencies.selectInstructionsDestination() + return dependencies.selectInstructionsDestination(agentCheckCount) } function securityReportInput( @@ -237,7 +243,12 @@ export default async function securityCheck( ) } - const destination = await instructionsDestination(options, dependencies, canPrompt) + const destination = await instructionsDestination( + options, + dependencies, + canPrompt, + execution.agentChecks.checks.length, + ) if (destination !== 'nothing') { await dependencies.deliverInstructions({ appDirectory, diff --git a/packages/app/src/cli/services/security-output.test.ts b/packages/app/src/cli/services/security-output.test.ts index 9a4ca87a205..1ac998c1562 100644 --- a/packages/app/src/cli/services/security-output.test.ts +++ b/packages/app/src/cli/services/security-output.test.ts @@ -192,8 +192,8 @@ describe('buildSecurityAlert', () => { expect(customSections[artifactsIndex]?.body).toEqual({ list: { items: [ - ['Deterministic findings:', {filePath: '/tmp/app/.shopify/app-security/deterministic-findings.json'}], - ['Agent checks:', {filePath: '/tmp/app/.shopify/app-security/agent-checks.json'}], + ['Deterministic findings:', {filePath: '.shopify/app-security/deterministic-findings.json'}], + ['Agent security check instructions:', {filePath: '.shopify/app-security/agent-checks.json'}], ], }, }) @@ -202,7 +202,7 @@ describe('buildSecurityAlert', () => { body: { list: { items: [ - ['Have your coding agent read', {filePath: '/tmp/app/.shopify/app-security/agent-checks.json'}], + ['Have your coding agent run the agent checks'], ['Record the agent results with', {command: formatAppSecurityCommand(commands.record)}], ['Review the results with', {command: formatAppSecurityCommand(commands.review)}], ], @@ -302,7 +302,7 @@ describe('buildSecurityAlert', () => { expect(section(reportInput({commands}), 'Next steps')?.body).toEqual({ list: { items: [ - ['Have your coding agent read', {filePath: '/tmp/app/.shopify/app-security/agent-checks.json'}], + ['Have your coding agent run the agent checks'], ['Record the agent results with', {command: recordCommand}], ['Review the results with', {command: formatAppSecurityCommand(commands.review)}], ], diff --git a/packages/app/src/cli/services/security-output.ts b/packages/app/src/cli/services/security-output.ts index 6f9de3c42af..46eb11fb5d2 100644 --- a/packages/app/src/cli/services/security-output.ts +++ b/packages/app/src/cli/services/security-output.ts @@ -129,7 +129,7 @@ function securityBody(input: SecurityReportInput): TokenItem { function securityNextSteps(input: SecurityReportInput): TokenItem[] { return [ - ['Have your coding agent read', {filePath: input.agentChecksPath}], + ['Have your coding agent run the agent checks'], ['Record the agent results with', {command: formatAppSecurityCommand(input.commands.record)}], ['Review the results with', {command: formatAppSecurityCommand(input.commands.review)}], ] @@ -187,13 +187,14 @@ function securityCustomSections(input: SecurityReportInput, groups: IssueGroup[] sections.push({title: 'Coverage gaps', body: {list: {items}}}) } + const {appDirectory} = input.selection sections.push({ title: 'Artifacts', body: { list: { items: [ - ['Deterministic findings:', {filePath: input.deterministicFindingsPath}], - ['Agent checks:', {filePath: input.agentChecksPath}], + ['Deterministic findings:', {filePath: relativePath(appDirectory, input.deterministicFindingsPath)}], + ['Agent security check instructions:', {filePath: relativePath(appDirectory, input.agentChecksPath)}], ], }, },