diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index 19ddea8417f..8e1a11c9a25 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -30,6 +30,10 @@ inputs: required: false default: '' description: The path used to persist per-flow test results between retries + exclude-tags: + required: false + default: '' + description: Comma-separated flow tags to exclude from the test run runs: using: composite @@ -78,7 +82,8 @@ runs: "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ "${{ inputs.device-os }}" \ - "${{ inputs.test-state-path }}" + "${{ inputs.test-state-path }}" \ + "${{ inputs.exclude-tags }}" - name: Store Logs if: always() uses: ./.github/actions/upload-artifact @@ -86,5 +91,5 @@ runs: name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch overwrite: true if-no-files-found: ignore - # Contains the recording, report and debug output of failed attempts only + # Only failed runs leave their report, debug output and recording here path: /tmp/MaestroLogs diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f01ae945119..63627894016 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -21,11 +21,27 @@ const path = require('path'); const { collectFlows, executeFlowSuite, + filterFlowsByTags, findAvailableSimulator, loadState, + parseJUnitReport, runMaestroFlow, } = require('../maestro-ios'); +// Simulates a single Maestro run in which the flows matching `failing` fail. +function suiteResults(failing = () => false) { + return jest.fn(async (appId, udid, flows) => { + return new Map( + flows.map(flow => [ + path.relative(process.cwd(), flow), + failing(flow) + ? {passed: false, error: 'failed assertion'} + : {passed: true}, + ]), + ); + }); +} + describe('Maestro iOS runner', () => { let temporaryDirectory; @@ -71,14 +87,54 @@ describe('Maestro iOS runner', () => { ]); }); - it('retries a failing flow once on a rebooted simulator and keeps going', async () => { + it('excludes flows with matching tags', () => { + const androidOnlyFlow = path.join(temporaryDirectory, 'android-only.yml'); + const regularFlow = path.join(temporaryDirectory, 'regular.yml'); + fs.writeFileSync( + androidOnlyFlow, + 'appId: x\ntags:\n - android-only\n---\n- launchApp\n', + ); + fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n'); + + expect( + filterFlowsByTags([androidOnlyFlow, regularFlow], ['android-only']), + ).toEqual([regularFlow]); + }); + + it('parses flow results from a Maestro JUnit report', () => { + const report = ` + + + + + Assertion is false: "Cheesecake" is visible + + + + +`; + + expect(parseJUnitReport(report)).toEqual( + new Map([ + ['flows/first.yml', {passed: true, error: 'SUCCESS'}], + [ + 'flows/second.yml', + {passed: false, error: 'Assertion is false: "Cheesecake" is visible'}, + ], + ['flows/third.yml', {passed: true, error: 'WARNING'}], + ]), + ); + }); + + it('runs all flows together, then retries each failure once on a rebooted simulator', async () => { const flows = ['first.yml', 'second.yml', 'third.yml'].map(file => path.join(temporaryDirectory, file), ); const statePath = path.join(temporaryDirectory, 'state', 'results.json'); + const executeSuite = suiteResults(flow => !flow.endsWith('first.yml')); const executeFlow = jest.fn(async (appId, udid, flow) => { if (flow.endsWith('second.yml')) { - throw new Error('failed assertion'); + throw new Error('failed again'); } }); const resetSimulator = jest.fn(); @@ -90,33 +146,32 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, + executeSuite, executeFlow, resetSimulator, }), ).rejects.toThrow('1 Maestro flow(s) failed'); + expect(executeSuite).toHaveBeenCalledTimes(1); + expect(executeSuite.mock.calls[0][2]).toEqual(flows); expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([ - [flows[0], 1], - [flows[1], 1], [flows[1], 2], - [flows[2], 1], + [flows[2], 2], ]); - expect(resetSimulator).toHaveBeenCalledTimes(1); + expect(resetSimulator).toHaveBeenCalledTimes(2); expect(resetSimulator).toHaveBeenCalledWith('device-id'); expect(Object.values(loadState(statePath).flows)).toEqual([ {status: 'passed', attempts: 1}, - {status: 'failed', attempts: 2, error: 'failed assertion'}, - {status: 'passed', attempts: 1}, + {status: 'failed', attempts: 2, error: 'failed again'}, + {status: 'passed', attempts: 2}, ]); }); - it('passes a flow that succeeds on its retry', async () => { - const flow = path.join(temporaryDirectory, 'flaky.yml'); + it('does not retry when every flow passes', async () => { + const flow = path.join(temporaryDirectory, 'first.yml'); const statePath = path.join(temporaryDirectory, 'results.json'); - const executeFlow = jest - .fn() - .mockRejectedValueOnce(new Error('flaky')) - .mockResolvedValueOnce(); + const executeFlow = jest.fn(); + const resetSimulator = jest.fn(); await executeFlowSuite({ flows: [flow], @@ -124,12 +179,15 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, + executeSuite: suiteResults(), executeFlow, - resetSimulator: jest.fn(), + resetSimulator, }); + expect(executeFlow).not.toHaveBeenCalled(); + expect(resetSimulator).not.toHaveBeenCalled(); expect(Object.values(loadState(statePath).flows)).toEqual([ - {status: 'passed', attempts: 2}, + {status: 'passed', attempts: 1}, ]); }); @@ -145,28 +203,26 @@ describe('Maestro iOS runner', () => { udid: 'device-id', state: loadState(statePath), statePath, - executeFlow: jest.fn(async (appId, udid, flow) => { - if (flow.endsWith('second.yml')) { - throw new Error('failed assertion'); - } - }), + executeSuite: suiteResults(flow => flow.endsWith('second.yml')), + executeFlow: jest.fn().mockRejectedValue(new Error('failed again')), resetSimulator: jest.fn(), }), ).rejects.toThrow(); - const retry = jest.fn(); + const executeSuite = suiteResults(); await executeFlowSuite({ flows, appId: 'com.example', udid: 'device-id', state: loadState(statePath), statePath, - executeFlow: retry, + executeSuite, + executeFlow: jest.fn(), resetSimulator: jest.fn(), }); - expect(retry).toHaveBeenCalledTimes(1); - expect(retry.mock.calls[0][2]).toBe(flows[1]); + expect(executeSuite).toHaveBeenCalledTimes(1); + expect(executeSuite.mock.calls[0][2]).toEqual([flows[1]]); }); it('waits for the recorder to exit before returning', async () => { diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index b39051f6527..fb2c26836d1 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -13,7 +13,7 @@ const path = require('path'); const usage = ` === Usage === -node maestro-ios.js [device_model] [device_os] [test_state_path] +node maestro-ios.js [device_model] [device_os] [test_state_path] [exclude_tags] @param {string} appPath - Path to the app APK @param {string} appId - App ID that needs to be launched @@ -24,6 +24,7 @@ node maestro-ios.js { + const tags = getFlowTags(flow); + const excludedTag = tags.find(tag => excludeTags.includes(tag)); + if (excludedTag != null) { + console.info(`Skipping flow tagged ${excludedTag}: ${flow}`); + return false; + } + return true; + }); +} + +function decodeXmlEntities(value) { + return value + .replaceAll('"', '"') + .replaceAll(''', "'") + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('&', '&'); +} + +// Maps each flow key in a Maestro JUnit report to whether it passed, plus the +// failure message for flows that did not. +function parseJUnitReport(contents) { + const results = new Map(); + const testCasePattern = /]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g; + for (const [, attributes, body = ''] of contents.matchAll(testCasePattern)) { + const attribute = name => + attributes.match(new RegExp(`\\b${name}="([^"]*)"`))?.[1]; + const file = attribute('file'); + if (file == null) { + continue; + } + const status = attribute('status'); + const failure = body.match(/]*>([\s\S]*?)<\/failure>/)?.[1]; + results.set(getFlowKey(decodeXmlEntities(file)), { + passed: status === 'SUCCESS' || status === 'WARNING', + error: failure != null ? decodeXmlEntities(failure.trim()) : status, + }); + } + return results; +} + +function maestroCommand(udid, flows, appId, outputPath) { + const flowArguments = flows.map(flow => `"${flow}"`).join(' '); + return `MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 $HOME/.maestro/bin/maestro --udid="${udid}" test ${flowArguments} --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; +} + +// Runs every flow in a single Maestro invocation, so the CLI and the XCTest +// driver start once instead of once per flow. Flows missing from the report +// (e.g. because Maestro crashed or timed out) count as failed. +function runMaestroSuite(appId, udid, flows) { + const outputPath = path.join(MAESTRO_LOG_DIRECTORY, 'suite'); + fs.mkdirSync(outputPath, {recursive: true}); + + const command = maestroCommand(udid, flows, appId, outputPath); + console.info(`Executing ${flows.length} flow(s) in a single Maestro run`); + console.log(command); + let runError = null; + try { + childProcess.execSync(command, { + stdio: 'inherit', + timeout: flows.length * FLOW_TIMEOUT_MS, + }); + } catch (error) { + runError = error; + } + + const reportPath = path.join(outputPath, 'report.xml'); + const report = fs.existsSync(reportPath) + ? parseJUnitReport(fs.readFileSync(reportPath, 'utf8')) + : new Map(); + const missingError = + runError instanceof Error + ? runError.message + : 'Flow did not report a result'; + const results = new Map( + flows.map(flow => [ + getFlowKey(flow), + report.get(getFlowKey(flow)) ?? {passed: false, error: missingError}, + ]), + ); + if ([...results.values()].every(result => result.passed)) { + fs.rmSync(outputPath, {recursive: true, force: true}); + } + return results; +} + async function runMaestroFlow(appId, udid, flow, attempt) { const outputName = `${path.basename(flow, path.extname(flow))}-attempt-${attempt}`; const outputPath = path.join(MAESTRO_LOG_DIRECTORY, outputName); @@ -293,13 +404,12 @@ async function runMaestroFlow(appId, udid, flow, attempt) { const videoPath = path.join(outputPath, 'recording.mov'); const recProcess = startVideoRecording(udid, videoPath); try { - const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; + const command = maestroCommand(udid, [flow], appId, outputPath); console.info(`Executing flow: ${flow} (attempt ${attempt})`); console.log(command); - childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { + childProcess.execSync(command, { stdio: 'inherit', - timeout, + timeout: FLOW_TIMEOUT_MS, }); } catch (error) { await stopVideoRecording(recProcess); @@ -317,6 +427,7 @@ async function executeFlowSuite({ udid, state, statePath, + executeSuite = runMaestroSuite, executeFlow = runMaestroFlow, resetSimulator = rebootSimulator, }) { @@ -327,30 +438,50 @@ async function executeFlowSuite({ } saveState(statePath, state); - const failedFlows = []; - for (let index = 0; index < flows.length; index++) { - const flow = flows[index]; - const flowKey = flowKeys[index]; - const result = state.flows[flowKey]; - - if (result.status === 'passed') { + const flowsToRun = flows.filter((flow, index) => { + if (state.flows[flowKeys[index]].status === 'passed') { console.info(`Skipping previously passed flow: ${flow}`); - continue; + return false; } + return true; + }); - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { - if (attempt > 1) { - console.info(`Retrying flow: ${flow}`); - resetSimulator(udid); + // First attempt: every flow in one Maestro run. + if (flowsToRun.length > 0) { + const results = await executeSuite(appId, udid, flowsToRun); + for (const flow of flowsToRun) { + const result = state.flows[getFlowKey(flow)]; + const {passed, error} = results.get(getFlowKey(flow)); + result.attempts += 1; + result.status = passed ? 'passed' : 'failed'; + if (passed) { + delete result.error; + } else { + result.error = error; } + } + saveState(statePath, state); + } + + // Retries: each failed flow on its own, on a rebooted simulator. + const failedFlows = []; + for (const flow of flowsToRun) { + const flowKey = getFlowKey(flow); + const result = state.flows[flowKey]; + + for ( + let attempt = 2; + attempt <= MAX_ATTEMPTS && result.status === 'failed'; + attempt++ + ) { + console.info(`Retrying flow: ${flow}`); + resetSimulator(udid); result.attempts += 1; try { await executeFlow(appId, udid, flow, attempt); result.status = 'passed'; delete result.error; - break; } catch (error) { - result.status = 'failed'; result.error = error instanceof Error ? error.message : String(error); } finally { saveState(statePath, state); @@ -375,7 +506,7 @@ async function executeFlowSuite({ } async function main(args = process.argv.slice(2)) { - if (args.length < 6 || args.length > 9) { + if (args.length < 6 || args.length > 10) { throw new Error(`Invalid number of arguments.\n${usage}`); } @@ -388,6 +519,10 @@ async function main(args = process.argv.slice(2)) { const deviceModel = args[6] || null; const deviceOS = args[7] || null; const statePath = args[8] || DEFAULT_STATE_PATH; + const excludeTags = (args[9] ?? '') + .split(',') + .map(tag => tag.trim()) + .filter(Boolean); console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -400,9 +535,10 @@ async function main(args = process.argv.slice(2)) { console.info(`DEVICE_MODEL: ${deviceModel ?? ''}`); console.info(`DEVICE_OS: ${deviceOS ?? ''}`); console.info(`TEST_STATE_PATH: ${statePath}`); + console.info(`EXCLUDE_TAGS: ${excludeTags.join(',') || ''}`); console.info('==============================\n'); - const flows = collectFlows(maestroFlow); + const flows = filterFlowsByTags(collectFlows(maestroFlow), excludeTags); const state = loadState(statePath); const simulator = findAvailableSimulator(deviceModel, deviceOS); @@ -431,8 +567,9 @@ if (require.main === module) { module.exports = { collectFlows, executeFlowSuite, + filterFlowsByTags, findAvailableSimulator, - formatResults, loadState, + parseJUnitReport, runMaestroFlow, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index f5c7c0c1bba..4fba45819de 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -77,6 +77,7 @@ jobs: device-model: iPhone-17-Pro device-os: iOS-26-2 test-state-path: /tmp/maestro-ios-state/results.json + exclude-tags: android-only - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact