From 89680f80df4e2254583dbe0843a5a6aa6744cc43 Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Thu, 29 Oct 2026 17:09:32 +0000 Subject: [PATCH] Retry failing iOS E2E flows once and stop rerunning real failures Each RNTester iOS flow had 5 attempts inside the job, and the whole job (both flavors, every flow, all setup) was retried up to 3 times on top, so a flow could run 15 times. The runner also stopped at the first flow that exhausted its attempts, hiding later failures. - Give each flow 2 attempts, rebooting the simulator before the retry. - Keep running the remaining flows after a failure and report every failed flow at the end, with a step summary. - Persist per-flow results between CI attempts (same format as the Android runner) so retries skip flows that already passed. - Only retry the workflow when a job stopped before running every flow. A flow that failed both attempts fails the run immediately. - Upload the recording, JUnit report and Maestro debug output for failed attempts only. --- .github/actions/maestro-ios/action.yml | 24 +- .../__tests__/maestro-ios-test.js | 221 +++++++++----- .github/workflow-scripts/maestro-ios.js | 275 ++++++++++++++---- .github/workflows/e2e-ios-rntester.yml | 85 +++++- .github/workflows/test-all.yml | 10 +- 5 files changed, 454 insertions(+), 161 deletions(-) diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index b5e4036d6280..19ddea8417f6 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -26,6 +26,10 @@ inputs: required: false default: '' description: Maestro device OS name, such as iOS-26-2 + test-state-path: + required: false + default: '' + description: The path used to persist per-flow test results between retries runs: using: composite @@ -73,24 +77,14 @@ runs: "${{ inputs.flavor }}" \ "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ - "${{ inputs.device-os }}" - - name: Store video record - if: always() - uses: ./.github/actions/upload-artifact - with: - name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch - overwrite: true - path: | - video_record_1.mov - video_record_2.mov - video_record_3.mov - video_record_4.mov - video_record_5.mov - report.xml + "${{ inputs.device-os }}" \ + "${{ inputs.test-state-path }}" - name: Store Logs - if: failure() && steps.run-tests.outcome == 'failure' + if: always() uses: ./.github/actions/upload-artifact with: name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch overwrite: true + if-no-files-found: ignore + # Contains the recording, report and debug output of failed attempts only path: /tmp/MaestroLogs diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f5f9686e4e09..f01ae9451195 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -11,21 +11,32 @@ jest.mock('child_process', () => ({ execSync: jest.fn(), spawn: jest.fn(), })); -jest.mock('fs', () => ({ - existsSync: jest.fn(), - lstatSync: jest.fn(), - readdirSync: jest.fn(), -})); const childProcess = require('child_process'); const {EventEmitter} = require('events'); const fs = require('fs'); +const os = require('os'); +const path = require('path'); -const {executeFlows, findAvailableSimulator} = require('../maestro-ios'); +const { + collectFlows, + executeFlowSuite, + findAvailableSimulator, + loadState, + runMaestroFlow, +} = require('../maestro-ios'); describe('Maestro iOS runner', () => { + let temporaryDirectory; + beforeEach(() => { jest.resetAllMocks(); + temporaryDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'maestro-ios-test-'), + ); + jest.spyOn(console, 'error').mockImplementation(() => {}); + jest.spyOn(console, 'info').mockImplementation(() => {}); + jest.spyOn(console, 'log').mockImplementation(() => {}); childProcess.spawn.mockImplementation(() => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; @@ -37,112 +48,170 @@ describe('Maestro iOS runner', () => { }); }); - it('executes each YAML flow separately and skips other files', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['second.yaml', 'image.png', 'first.yml']); + afterEach(() => { + jest.restoreAllMocks(); + fs.rmSync(temporaryDirectory, {recursive: true, force: true}); + }); - await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); + it('collects YAML flows in a stable order and skips helpers and other files', () => { + const helpersDirectory = path.join(temporaryDirectory, 'helpers'); + const nestedDirectory = path.join(temporaryDirectory, 'nested'); + fs.mkdirSync(helpersDirectory); + fs.mkdirSync(nestedDirectory); + fs.writeFileSync(path.join(temporaryDirectory, 'second.yaml'), 'appId: x'); + fs.writeFileSync(path.join(temporaryDirectory, 'image.png'), 'not a flow'); + fs.writeFileSync(path.join(temporaryDirectory, 'first.yml'), 'appId: x'); + fs.writeFileSync(path.join(helpersDirectory, 'search.yml'), 'appId: x'); + fs.writeFileSync(path.join(nestedDirectory, 'third.yml'), 'appId: x'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/first.yml"', - ); - expect(childProcess.execSync.mock.calls[1][0]).toContain( - 'test "flows/second.yaml"', - ); + expect(collectFlows(temporaryDirectory)).toEqual([ + path.join(temporaryDirectory, 'first.yml'), + path.join(nestedDirectory, 'third.yml'), + path.join(temporaryDirectory, 'second.yaml'), + ]); }); - it('retries only the failing flow', async () => { - fs.existsSync.mockReturnValue(false); - childProcess.execSync.mockImplementationOnce(() => { - throw new Error('Maestro driver failed'); + it('retries a failing flow once on a rebooted simulator and keeps going', async () => { + const flows = ['first.yml', 'second.yml', 'third.yml'].map(file => + path.join(temporaryDirectory, file), + ); + const statePath = path.join(temporaryDirectory, 'state', 'results.json'); + const executeFlow = jest.fn(async (appId, udid, flow) => { + if (flow.endsWith('second.yml')) { + throw new Error('failed assertion'); + } }); + const resetSimulator = jest.fn(); - await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + await expect( + executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow, + resetSimulator, + }), + ).rejects.toThrow('1 Maestro flow(s) failed'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - for (const call of childProcess.execSync.mock.calls) { - expect(call[0]).toContain('test "flow.yml"'); - } + expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([ + [flows[0], 1], + [flows[1], 1], + [flows[1], 2], + [flows[2], 1], + ]); + expect(resetSimulator).toHaveBeenCalledTimes(1); + expect(resetSimulator).toHaveBeenCalledWith('device-id'); + expect(Object.values(loadState(statePath).flows)).toEqual([ + {status: 'passed', attempts: 1}, + {status: 'failed', attempts: 2, error: 'failed assertion'}, + {status: 'passed', attempts: 1}, + ]); }); - it('waits for the recorder to exit before starting the next flow', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['first.yml', 'second.yml']); + it('passes a flow that succeeds on its retry', async () => { + const flow = path.join(temporaryDirectory, 'flaky.yml'); + const statePath = path.join(temporaryDirectory, 'results.json'); + const executeFlow = jest + .fn() + .mockRejectedValueOnce(new Error('flaky')) + .mockResolvedValueOnce(); + + await executeFlowSuite({ + flows: [flow], + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow, + resetSimulator: jest.fn(), + }); + + expect(Object.values(loadState(statePath).flows)).toEqual([ + {status: 'passed', attempts: 2}, + ]); + }); + + it('only runs flows that have not passed in a previous CI attempt', async () => { + const flows = ['first.yml', 'second.yml'].map(file => + path.join(temporaryDirectory, file), + ); + const statePath = path.join(temporaryDirectory, 'results.json'); + await expect( + executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow: jest.fn(async (appId, udid, flow) => { + if (flow.endsWith('second.yml')) { + throw new Error('failed assertion'); + } + }), + resetSimulator: jest.fn(), + }), + ).rejects.toThrow(); + + const retry = jest.fn(); + await executeFlowSuite({ + flows, + appId: 'com.example', + udid: 'device-id', + state: loadState(statePath), + statePath, + executeFlow: retry, + resetSimulator: jest.fn(), + }); + expect(retry).toHaveBeenCalledTimes(1); + expect(retry.mock.calls[0][2]).toBe(flows[1]); + }); + + it('waits for the recorder to exit before returning', async () => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; recordingProcess.kill = jest.fn(() => true); childProcess.spawn.mockReturnValueOnce(recordingProcess); - const execution = executeFlows( + let finished = false; + const execution = runMaestroFlow( 'com.example', 'device-id', - 'flows/', - 'Hermes', - ); + 'flows/first.yml', + 1, + ).then(() => { + finished = true; + }); await new Promise(resolve => jest.requireActual('timers').setImmediate(resolve), ); expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.spawn).toHaveBeenCalledTimes(1); + expect(childProcess.execSync.mock.calls[0][0]).toContain( + 'test "flows/first.yml"', + ); + expect(finished).toBe(false); recordingProcess.emit('exit', 0, null); await execution; - - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.spawn).toHaveBeenCalledTimes(2); + expect(finished).toBe(true); }); - it('skips helper directories while recursing into flow directories', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => !path.endsWith('.yml'), - })); - fs.readdirSync.mockImplementation(path => - path === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], - ); - - await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - - expect(fs.readdirSync).not.toHaveBeenCalledWith('flows/helpers'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/nested/flow.yml"', - ); - }); - - it('rejects after exhausting retries and stops every recorder', async () => { - const consoleError = jest - .spyOn(console, 'error') - .mockImplementation(() => {}); - fs.existsSync.mockReturnValue(false); + it('stops the recorder when Maestro fails', async () => { const error = new Error('Maestro driver failed'); childProcess.execSync.mockImplementation(() => { throw error; }); await expect( - executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'), + runMaestroFlow('com.example', 'device-id', 'flow.yml', 2), ).rejects.toBe(error); - expect(childProcess.execSync).toHaveBeenCalledTimes(5); - expect(childProcess.spawn).toHaveBeenCalledTimes(5); - for (const {value: recordingProcess} of childProcess.spawn.mock.results) { - expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - } - expect(consoleError).toHaveBeenCalledWith( - 'Failed to execute flow flow.yml after 5 attempts.', - ); + const {value: recordingProcess} = childProcess.spawn.mock.results[0]; + expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); }); it('selects an iPhone Pro simulator from the latest runtime', () => { diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index a9782ec6506f..b39051f65272 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -9,10 +9,11 @@ const childProcess = require('child_process'); const fs = require('fs'); +const path = require('path'); const usage = ` === Usage === -node maestro-ios.js [device_model] [device_os] +node maestro-ios.js [device_model] [device_os] [test_state_path] @param {string} appPath - Path to the app APK @param {string} appId - App ID that needs to be launched @@ -22,10 +23,16 @@ node maestro-ios.js ({flow, ...state.flows[flow]})); + const counts = results.reduce( + (result, flow) => { + result[flow.status] += 1; + return result; + }, + {passed: 0, failed: 0, pending: 0}, + ); + const rows = results + .map( + result => + `| ${result.status} | \`${result.flow.replaceAll('|', '\\|')}\` | ${result.attempts} |`, + ) + .join('\n'); + + return `### iOS Maestro E2E results + +Passed: ${counts.passed} · Failed: ${counts.failed} · Pending: ${counts.pending} + +| Status | Flow | Attempts | +| --- | --- | ---: | +${rows} +`; +} + +function writeResultsSummary(flowKeys, state) { + const summary = formatResults(flowKeys, state); + console.info(`\n${summary}`); + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); + } +} + +async function runMaestroFlow(appId, udid, flow, attempt) { + const outputName = `${path.basename(flow, path.extname(flow))}-attempt-${attempt}`; + const outputPath = path.join(MAESTRO_LOG_DIRECTORY, outputName); + fs.mkdirSync(outputPath, {recursive: true}); + + const videoPath = path.join(outputPath, 'recording.mov'); + const recProcess = startVideoRecording(udid, videoPath); try { const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit -e APP_ID="${appId}"`; - console.info(`Executing flow: ${flow} (attempt ${currentAttempt})`); + const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`; + console.info(`Executing flow: ${flow} (attempt ${attempt})`); console.log(command); childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { stdio: 'inherit', timeout, }); - - await stopVideoRecording(recProcess); } catch (error) { await stopVideoRecording(recProcess); + throw error; + } - if (currentAttempt < MAX_ATTEMPTS) { - console.info(`Retrying flow: ${flow}`); - await executeFlowWithRetries( - appId, - udid, - flow, - jsengine, - currentAttempt + 1, - ); - } else { + await stopVideoRecording(recProcess); + // Only failed attempts are worth keeping in the uploaded logs. + fs.rmSync(outputPath, {recursive: true, force: true}); +} + +async function executeFlowSuite({ + flows, + appId, + udid, + state, + statePath, + executeFlow = runMaestroFlow, + resetSimulator = rebootSimulator, +}) { + const flowKeys = flows.map(getFlowKey); + + for (const flow of flowKeys) { + state.flows[flow] ??= {status: 'pending', attempts: 0}; + } + saveState(statePath, state); + + const failedFlows = []; + for (let index = 0; index < flows.length; index++) { + const flow = flows[index]; + const flowKey = flowKeys[index]; + const result = state.flows[flowKey]; + + if (result.status === 'passed') { + console.info(`Skipping previously passed flow: ${flow}`); + continue; + } + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + if (attempt > 1) { + console.info(`Retrying flow: ${flow}`); + resetSimulator(udid); + } + result.attempts += 1; + try { + await executeFlow(appId, udid, flow, attempt); + result.status = 'passed'; + delete result.error; + break; + } catch (error) { + result.status = 'failed'; + result.error = error instanceof Error ? error.message : String(error); + } finally { + saveState(statePath, state); + } + } + + if (result.status === 'failed') { + failedFlows.push(flowKey); console.error( `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, ); - throw error; } } -} -async function executeFlows(appId, udid, maestroFlow, jsengine) { - if (!fs.existsSync(maestroFlow) || !fs.lstatSync(maestroFlow).isDirectory()) { - await executeFlowWithRetries(appId, udid, maestroFlow, jsengine, 1); - return; - } + writeResultsSummary(flowKeys, state); - for (const file of fs.readdirSync(maestroFlow).sort()) { - const filePath = `${maestroFlow.replace(/\/$/, '')}/${file}`; - if (fs.lstatSync(filePath).isDirectory()) { - // Fragments pulled in via `runFlow`; they have no `launchApp` of their - // own and fail when run standalone. - if (file === 'helpers') { - continue; - } - await executeFlows(appId, udid, filePath, jsengine); - } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { - await executeFlowWithRetries(appId, udid, filePath, jsengine, 1); - } + if (failedFlows.length > 0) { + throw new Error( + `${failedFlows.length} Maestro flow(s) failed:\n${failedFlows.join('\n')}`, + ); } } async function main(args = process.argv.slice(2)) { - if (args.length < 6 || args.length > 8) { + if (args.length < 6 || args.length > 9) { throw new Error(`Invalid number of arguments.\n${usage}`); } @@ -249,6 +387,7 @@ async function main(args = process.argv.slice(2)) { const workingDirectory = args[5]; const deviceModel = args[6] || null; const deviceOS = args[7] || null; + const statePath = args[8] || DEFAULT_STATE_PATH; console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -260,22 +399,40 @@ async function main(args = process.argv.slice(2)) { console.info(`WORKING_DIRECTORY: ${workingDirectory}`); console.info(`DEVICE_MODEL: ${deviceModel ?? ''}`); console.info(`DEVICE_OS: ${deviceOS ?? ''}`); + console.info(`TEST_STATE_PATH: ${statePath}`); console.info('==============================\n'); + const flows = collectFlows(maestroFlow); + const state = loadState(statePath); + const simulator = findAvailableSimulator(deviceModel, deviceOS); launchSimulator(simulator); installAppOnSimulator(appPath, simulator.udid); bringSimulatorInForeground(); await launchAppOnSimulator(appId, simulator.udid, isDebug); - await executeFlows(appId, simulator.udid, maestroFlow, jsengine); + console.info(`Start testing ${flows.length} flow(s)`); + await executeFlowSuite({ + flows, + appId, + udid: simulator.udid, + state, + statePath, + }); console.log('Test finished'); } if (require.main === module) { - main(); + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); } module.exports = { - executeFlows, + collectFlows, + executeFlowSuite, findAvailableSimulator, + formatResults, + loadState, + runMaestroFlow, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index c841c409b9ea..f5c7c0c1bbad 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -9,16 +9,25 @@ on: fail-on-error: type: boolean default: false + retry-attempt: + type: number + default: 0 outputs: status: description: 'The result of the E2E tests (success or failure)' value: ${{ jobs.report.outputs.status }} + retryable: + description: 'Whether the failure came from a job that did not finish running every flow' + value: ${{ jobs.report.outputs.retryable }} jobs: test: runs-on: macos-26-large outputs: - status: ${{ steps.report-status.outputs.status }} + # Matrix legs only keep the last value written to an output name, so each + # flavor writes its own. + debug-result: ${{ steps.classify.outputs.debug-result }} + release-result: ${{ steps.classify.outputs.release-result }} strategy: fail-fast: false matrix: @@ -26,17 +35,38 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Download previous per-flow test state + if: ${{ inputs.retry-attempt > 0 }} + continue-on-error: true + uses: ./.github/actions/download-artifact + with: + name: e2e_ios_rntester_state_${{ matrix.flavor }}_NewArch + path: /tmp/maestro-ios-state + - name: Check for unfinished E2E flows + id: test-state + shell: bash + run: | + SHOULD_RUN=true + if [[ -f /tmp/maestro-ios-state/results.json ]] && \ + jq -e '(.flows | length > 0) and all(.flows[]; .status == "passed")' /tmp/maestro-ios-state/results.json > /dev/null; then + SHOULD_RUN=false + fi + echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Setup Node.js + if: steps.test-state.outputs.should-run == 'true' uses: ./.github/actions/setup-node - name: Download App + if: steps.test-state.outputs.should-run == 'true' uses: ./.github/actions/download-artifact with: name: RNTesterApp-NewArch-${{ matrix.flavor }} path: /tmp/RNTesterBuild/RNTester.app - name: Check downloaded folder content + if: steps.test-state.outputs.should-run == 'true' run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests + if: steps.test-state.outputs.should-run == 'true' continue-on-error: true uses: ./.github/actions/maestro-ios with: @@ -46,10 +76,36 @@ jobs: flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 - - name: Report status - id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT + test-state-path: /tmp/maestro-ios-state/results.json + - name: Store per-flow test state + if: always() + uses: ./.github/actions/upload-artifact + with: + name: e2e_ios_rntester_state_${{ matrix.flavor }}_NewArch + overwrite: true + if-no-files-found: warn + path: /tmp/maestro-ios-state/results.json + - name: Classify result + id: classify + if: always() + shell: bash + # passed: every flow passed. + # failed: a flow failed all of its attempts; retrying the job would only + # repeat the same failure. + # incomplete: the job stopped before running every flow (runner or + # setup failure); a retry can pick up the remaining flows. + run: | + STATE=/tmp/maestro-ios-state/results.json + if [[ "${{ steps.test-state.outputs.should-run }}" == "false" || "${{ steps.run-tests.outcome }}" == "success" ]]; then + RESULT=passed + elif [[ -f "$STATE" ]] && jq -e 'any(.flows[]; .status == "failed")' "$STATE" > /dev/null; then + RESULT=failed + else + RESULT=incomplete + fi + FLAVOR=$(echo "${{ matrix.flavor }}" | tr '[:upper:]' '[:lower:]') + echo "Result: $RESULT" + echo "$FLAVOR-result=$RESULT" >> "$GITHUB_OUTPUT" report: needs: test @@ -57,15 +113,26 @@ jobs: runs-on: ubuntu-latest outputs: status: ${{ steps.check.outputs.status }} + retryable: ${{ steps.check.outputs.retryable }} steps: - id: check + env: + DEBUG_RESULT: ${{ needs.test.outputs.debug-result }} + RELEASE_RESULT: ${{ needs.test.outputs.release-result }} + # A missing result means the leg did not get as far as classifying + # itself, which is treated like an incomplete run. run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then + echo "Debug: ${DEBUG_RESULT:-}, Release: ${RELEASE_RESULT:-}" + if [[ "$DEBUG_RESULT" == "passed" && "$RELEASE_RESULT" == "passed" ]]; then + echo "status=success" >> $GITHUB_OUTPUT + echo "retryable=false" >> $GITHUB_OUTPUT + elif [[ "$DEBUG_RESULT" == "failed" || "$RELEASE_RESULT" == "failed" ]]; then echo "status=failure" >> $GITHUB_OUTPUT + echo "retryable=false" >> $GITHUB_OUTPUT else - echo "status=success" >> $GITHUB_OUTPUT + echo "status=failure" >> $GITHUB_OUTPUT + echo "retryable=true" >> $GITHUB_OUTPUT fi - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} + if: ${{ steps.check.outputs.status == 'failure' && (inputs.fail-on-error || steps.check.outputs.retryable == 'false') }} run: exit 1 diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index a9a3fc90350c..22cd50681ff0 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -197,18 +197,24 @@ jobs: uses: ./.github/workflows/e2e-ios-rntester.yml secrets: inherit + # Each flow already gets a retry inside the job, so a flow that failed is + # reported straight away. These retries only re-run jobs that stopped before + # running every flow, and they skip flows that already passed. test_e2e_ios_rntester_retry_1: needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} + if: ${{ always() && needs.test_e2e_ios_rntester.outputs.retryable == 'true' }} uses: ./.github/workflows/e2e-ios-rntester.yml + with: + retry-attempt: 1 secrets: inherit test_e2e_ios_rntester_retry_2: needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.retryable == 'true' }} uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true + retry-attempt: 2 secrets: inherit test_visual_ios_rntester: