diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index b5e4036d6280..b85b483a2f1a 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -26,6 +26,18 @@ inputs: required: false default: '' description: Maestro device OS name, such as iOS-26-2 + shard-index: + required: false + default: '0' + description: Zero-based index of the flow shard to run + shard-count: + required: false + default: '1' + description: Number of flow shards + reset-app: + required: false + default: 'false' + description: Whether to reset the app before each flow attempt runs: using: composite @@ -60,6 +72,10 @@ runs: - name: Run tests id: run-tests shell: bash + env: + MAESTRO_SHARD_INDEX: ${{ inputs.shard-index }} + MAESTRO_SHARD_COUNT: ${{ inputs.shard-count }} + MAESTRO_RESET_APP: ${{ inputs.reset-app }} run: | # Avoid exit from the job if one of the command returns an error. # Maestro can fail in case of flakyness, we have some retry logic. @@ -74,23 +90,10 @@ runs: "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ "${{ inputs.device-os }}" - - name: Store video record + - name: Store Maestro reports if: always() uses: ./.github/actions/upload-artifact with: - name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch - overwrite: true - path: | - video_record_1.mov - video_record_2.mov - video_record_3.mov - video_record_4.mov - video_record_5.mov - report.xml - - name: Store Logs - if: failure() && steps.run-tests.outcome == 'failure' - uses: ./.github/actions/upload-artifact - with: - name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch + name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch_shard_${{ inputs.shard-index }}_attempt_${{ github.run_attempt }} overwrite: true - path: /tmp/MaestroLogs + path: maestro-reports diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f5f9686e4e09..7fdf2ecc1fa1 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -9,23 +9,44 @@ jest.mock('child_process', () => ({ execSync: jest.fn(), + execFileSync: jest.fn(), spawn: jest.fn(), })); jest.mock('fs', () => ({ existsSync: jest.fn(), lstatSync: jest.fn(), readdirSync: jest.fn(), + mkdirSync: jest.fn(), + writeFileSync: jest.fn(), })); const childProcess = require('child_process'); const {EventEmitter} = require('events'); const fs = require('fs'); +const path = require('path'); -const {executeFlows, findAvailableSimulator} = require('../maestro-ios'); +const {executeFlows, findAvailableSimulator, main} = require('../maestro-ios'); describe('Maestro iOS runner', () => { + function flowCalls() { + return childProcess.execFileSync.mock.calls.filter( + ([file]) => file !== 'xcrun', + ); + } + + function mockFlows(files) { + fs.existsSync.mockReturnValue(true); + fs.lstatSync.mockImplementation(file => ({ + isDirectory: () => file === 'flows/', + })); + fs.readdirSync.mockReturnValue(files); + } + beforeEach(() => { jest.resetAllMocks(); + jest.spyOn(console, 'log').mockImplementation(() => {}); + jest.spyOn(console, 'info').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); childProcess.spawn.mockImplementation(() => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; @@ -37,45 +58,139 @@ describe('Maestro iOS runner', () => { }); }); + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('waits for cold boot before installing, launching, and running flows', async () => { + jest.replaceProperty(process, 'env', { + ...process.env, + MAESTRO_RESET_APP: 'true', + MAESTRO_SHARD_INDEX: '0', + MAESTRO_SHARD_COUNT: '1', + }); + childProcess.execSync.mockReturnValueOnce( + JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-26-2': [ + {name: 'iPhone 17 Pro', udid: 'device-id'}, + ], + }, + }), + ); + childProcess.execFileSync.mockImplementation((file, args, options) => { + if (args[1] === 'bootstatus' && options.timeout < 3 * 60 * 1000) { + throw new Error('Cold simulator data migration exceeded the deadline'); + } + }); + + await main([ + '/tmp/RNTester.app', + 'com.example', + 'flow.yml', + 'Hermes', + 'Release', + '/tmp/workspace', + 'iPhone-17-Pro', + 'iOS-26-2', + ]); + + const calls = childProcess.execFileSync.mock.calls; + expect(calls[0]).toEqual([ + 'xcrun', + ['simctl', 'bootstatus', 'device-id', '-b'], + {stdio: 'inherit', timeout: 10 * 60 * 1000, killSignal: 'SIGKILL'}, + ]); + expect(calls.slice(1, -1).map(([, args]) => args)).toEqual([ + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'launch', 'device-id', 'com.example'], + ['simctl', 'uninstall', 'device-id', 'com.example'], + ['simctl', 'keychain', 'device-id', 'reset'], + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'privacy', 'device-id', 'reset', 'all', 'com.example'], + ['simctl', 'ui', 'device-id', 'appearance', 'light'], + ]); + expect(childProcess.execSync).toHaveBeenLastCalledWith('open -a simulator'); + expect(flowCalls()).toHaveLength(1); + expect(flowCalls()[0][1]).toContain('APP_ID=com.example'); + expect(console.log).toHaveBeenLastCalledWith('Test finished'); + }); + + it('hard-kills a hung boot without installing the app or running flows', async () => { + const actualChildProcess = jest.requireActual('child_process'); + childProcess.execSync.mockReturnValueOnce( + JSON.stringify({ + devices: { + 'iOS 26.2': [{name: 'iPhone 17 Pro', udid: 'device-id'}], + }, + }), + ); + let timeoutError; + childProcess.execFileSync.mockImplementationOnce((file, args, options) => { + expect(args).toEqual(['simctl', 'bootstatus', 'device-id', '-b']); + expect(options.timeout).toBe(10 * 60 * 1000); + expect(options.killSignal).toBe('SIGKILL'); + try { + actualChildProcess.execFileSync( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stderr.write("ready"); setInterval(() => {}, 1000);', + ], + {...options, timeout: 500, stdio: 'pipe'}, + ); + } catch (error) { + timeoutError = error; + throw error; + } + }); + + await expect( + main([ + '/tmp/RNTester.app', + 'com.example', + 'flow.yml', + 'Hermes', + 'Release', + '/tmp/workspace', + ]), + ).rejects.toMatchObject({code: 'ETIMEDOUT', signal: 'SIGKILL'}); + + expect(String(timeoutError.stderr)).toContain('ready'); + expect(childProcess.execFileSync).toHaveBeenCalledTimes(1); + expect(childProcess.execSync).toHaveBeenCalledTimes(1); + expect(childProcess.spawn).not.toHaveBeenCalled(); + expect(fs.writeFileSync).not.toHaveBeenCalled(); + }); + it('executes each YAML flow separately and skips other files', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['second.yaml', 'image.png', 'first.yml']); + mockFlows(['second.yaml', 'image.png', 'first.yml']); await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/first.yml"', - ); - expect(childProcess.execSync.mock.calls[1][0]).toContain( - 'test "flows/second.yaml"', - ); + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + 'flows/first.yml', + 'flows/second.yaml', + ]); }); it('retries only the failing flow', async () => { - fs.existsSync.mockReturnValue(false); - childProcess.execSync.mockImplementationOnce(() => { + mockFlows(['first.yml', 'second.yml']); + childProcess.execFileSync.mockImplementationOnce(() => { throw new Error('Maestro driver failed'); }); - await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - for (const call of childProcess.execSync.mock.calls) { - expect(call[0]).toContain('test "flow.yml"'); - } + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + 'flows/first.yml', + 'flows/first.yml', + 'flows/second.yml', + ]); }); it('waits for the recorder to exit before starting the next flow', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['first.yml', 'second.yml']); - + mockFlows(['first.yml', 'second.yml']); const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; recordingProcess.kill = jest.fn(() => true); @@ -87,61 +202,249 @@ describe('Maestro iOS runner', () => { 'flows/', 'Hermes', ); - await new Promise(resolve => jest.requireActual('timers').setImmediate(resolve), ); expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); + expect(flowCalls()).toHaveLength(1); expect(childProcess.spawn).toHaveBeenCalledTimes(1); recordingProcess.emit('exit', 0, null); await execution; - expect(childProcess.execSync).toHaveBeenCalledTimes(2); + expect(flowCalls()).toHaveLength(2); expect(childProcess.spawn).toHaveBeenCalledTimes(2); }); it('skips helper directories while recursing into flow directories', async () => { fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => !path.endsWith('.yml'), + fs.lstatSync.mockImplementation(file => ({ + isDirectory: () => !file.endsWith('.yml'), })); - fs.readdirSync.mockImplementation(path => - path === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], + fs.readdirSync.mockImplementation(directory => + directory === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], ); await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); expect(fs.readdirSync).not.toHaveBeenCalledWith('flows/helpers'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/nested/flow.yml"', + expect(flowCalls()).toHaveLength(1); + expect(flowCalls()[0][1][3]).toBe('flows/nested/flow.yml'); + }); + + it('partitions the sorted flow list without omissions or duplicates', async () => { + const files = ['e.yml', 'd.yaml', 'c.yml', 'b.yml', 'a.yml']; + const partitions = []; + for (let shardIndex = 0; shardIndex < 4; shardIndex++) { + mockFlows(files); + childProcess.execFileSync.mockClear(); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + shardIndex, + shardCount: 4, + }); + partitions.push(flowCalls().map(([, args]) => args[3])); + } + + expect(partitions).toEqual([ + ['flows/a.yml', 'flows/e.yml'], + ['flows/b.yml'], + ['flows/c.yml'], + ['flows/d.yaml'], + ]); + expect(new Set(partitions.flat()).size).toBe(files.length); + }); + + it.each([ + {shardIndex: -1, shardCount: 4}, + {shardIndex: 4, shardCount: 4}, + {shardIndex: 0, shardCount: 0}, + {shardIndex: 0.5, shardCount: 4}, + {shardIndex: 0, shardCount: NaN}, + ])('rejects invalid sharding options: %p', async options => { + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes', options), + ).rejects.toThrow('Invalid shard'); + expect(childProcess.execFileSync).not.toHaveBeenCalled(); + }); + + it('fails an empty suite instead of passing without coverage', async () => { + mockFlows(['readme.txt']); + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes'), + ).rejects.toThrow('No Maestro flows'); + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + + it('allows empty shards when there are fewer flows than shards', async () => { + mockFlows(['first.yml']); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + shardIndex: 3, + shardCount: 4, + }); + expect(childProcess.spawn).not.toHaveBeenCalled(); + expect(JSON.parse(fs.writeFileSync.mock.calls[0][1])).toEqual({ + shardIndex: 3, + shardCount: 4, + flows: [], + }); + }); + + it('resets before every retry and every subsequent flow', async () => { + mockFlows(['first.yml', 'second.yml']); + let attempts = 0; + childProcess.execFileSync.mockImplementation(file => { + if (file !== 'xcrun' && attempts++ === 0) { + throw new Error('Flow left the app dirty'); + } + }); + + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + resetApp: true, + appPath: '/tmp/RNTester.app', + }); + + const resetCommands = [ + ['simctl', 'uninstall', 'device-id', 'com.example'], + ['simctl', 'keychain', 'device-id', 'reset'], + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'privacy', 'device-id', 'reset', 'all', 'com.example'], + ['simctl', 'ui', 'device-id', 'appearance', 'light'], + ]; + const calls = childProcess.execFileSync.mock.calls; + expect(calls).toHaveLength(18); + for (let attempt = 0; attempt < 3; attempt++) { + expect( + calls.slice(attempt * 6, attempt * 6 + 5).map(([, args]) => args), + ).toEqual(resetCommands); + expect(calls[attempt * 6 + 5][0]).not.toBe('xcrun'); + } + const wrappers = fs.writeFileSync.mock.calls.filter(([file]) => + file.endsWith('/flow.yml'), + ); + expect(wrappers).toHaveLength(3); + for (const [, content] of wrappers) { + expect(content).toContain( + '- setOrientation: portrait\n- launchApp\n- runFlow:', + ); + expect(content).not.toContain('clearState'); + } + expect(wrappers[0][1]).toContain( + JSON.stringify(path.resolve('flows/first.yml')), + ); + expect(wrappers[2][1]).toContain( + JSON.stringify(path.resolve('flows/second.yml')), ); }); - it('rejects after exhausting retries and stops every recorder', async () => { - const consoleError = jest - .spyOn(console, 'error') - .mockImplementation(() => {}); + it('does not execute a flow when reset fails', async () => { fs.existsSync.mockReturnValue(false); - const error = new Error('Maestro driver failed'); - childProcess.execSync.mockImplementation(() => { + const error = new Error('Unable to uninstall app'); + childProcess.execFileSync.mockImplementation(() => { throw error; }); await expect( - executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'), - ).rejects.toBe(error); + executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes', { + resetApp: true, + appPath: '/tmp/RNTester.app', + }), + ).rejects.toMatchObject({errors: [error]}); + + expect(childProcess.execFileSync).toHaveBeenCalledTimes(5); + expect(flowCalls()).toHaveLength(0); + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + + it('requires the original app artifact for reset', async () => { + await expect( + executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes', { + resetApp: true, + }), + ).rejects.toThrow('requires an app path'); + expect(childProcess.execFileSync).not.toHaveBeenCalled(); + }); + + it('keeps reports, logs, screenshots, and videos unique for every attempt', async () => { + mockFlows(['first.yml', 'second.yml']); + childProcess.execFileSync.mockImplementationOnce(() => { + throw new Error('Retry'); + }); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); + + for (const flag of ['--output', '--debug-output', '--test-output-dir']) { + const outputs = flowCalls().map( + ([, args]) => args[args.indexOf(flag) + 1], + ); + expect(new Set(outputs).size).toBe(3); + expect(outputs.every(output => output.includes('maestro-reports'))).toBe( + true, + ); + } + const videos = childProcess.spawn.mock.calls.map(([, args]) => args.at(-1)); + expect(new Set(videos).size).toBe(3); + expect(videos[0]).toContain('0-first.yml/attempt-1/video.mov'); + expect(videos[1]).toContain('0-first.yml/attempt-2/video.mov'); + expect(videos[2]).toContain('1-second.yml/attempt-1/video.mov'); + for (const [, args] of flowCalls()) { + expect(args).toContain('APP_ID=com.example'); + } + }); + + it('hard-kills a hung flow and retries after stopping its recorder', async () => { + const actualChildProcess = jest.requireActual('child_process'); + let timeoutError; + childProcess.execFileSync.mockImplementationOnce((file, args, options) => { + expect(options.killSignal).toBe('SIGKILL'); + try { + actualChildProcess.execFileSync( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stderr.write("ready"); setInterval(() => {}, 1000);', + ], + {...options, timeout: 500, stdio: 'pipe'}, + ); + } catch (error) { + timeoutError = error; + throw error; + } + }); + + await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + + expect(timeoutError).toMatchObject({code: 'ETIMEDOUT', signal: 'SIGKILL'}); + expect(String(timeoutError.stderr)).toContain('ready'); + expect(flowCalls()).toHaveLength(2); + expect(childProcess.spawn).toHaveBeenCalledTimes(2); + for (const {value: recordingProcess} of childProcess.spawn.mock.results) { + expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); + } + }); + + it('continues after an exhausted flow and reports the failure at the end', async () => { + mockFlows(['first.yml', 'second.yml']); + const error = new Error('Maestro driver failed'); + childProcess.execFileSync.mockImplementation((file, args) => { + if (args[3] === 'flows/first.yml') { + throw error; + } + }); + + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes'), + ).rejects.toMatchObject({errors: [error]}); - expect(childProcess.execSync).toHaveBeenCalledTimes(5); - expect(childProcess.spawn).toHaveBeenCalledTimes(5); + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + ...Array(5).fill('flows/first.yml'), + 'flows/second.yml', + ]); + expect(childProcess.spawn).toHaveBeenCalledTimes(6); for (const {value: recordingProcess} of childProcess.spawn.mock.results) { expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); } - expect(consoleError).toHaveBeenCalledWith( - 'Failed to execute flow flow.yml after 5 attempts.', + expect(console.error).toHaveBeenCalledWith( + 'Failed to execute flow flows/first.yml after 5 attempts.', ); }); @@ -157,7 +460,6 @@ describe('Maestro iOS runner', () => { }, }), ); - expect(findAvailableSimulator()).toEqual({ name: 'iPhone 17 Pro', udid: 'new-pro', @@ -178,7 +480,6 @@ describe('Maestro iOS runner', () => { }, }), ); - expect(findAvailableSimulator('iPhone-17-Pro', 'iOS-26-2')).toEqual({ name: 'iPhone 17 Pro', udid: 'expected', @@ -187,7 +488,6 @@ describe('Maestro iOS runner', () => { it('fails when the configured simulator is unavailable', () => { childProcess.execSync.mockReturnValue(JSON.stringify({devices: {}})); - expect(() => findAvailableSimulator('iPhone-17-Pro', 'iOS-26-2')).toThrow( 'Unable to find iPhone 17 Pro simulator on iOS-26-2', ); diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index a9782ec6506f..838afdbdd865 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -9,6 +9,7 @@ const childProcess = require('child_process'); const fs = require('fs'); +const path = require('path'); const usage = ` === Usage === @@ -65,20 +66,24 @@ function findAvailableSimulator(deviceModel, deviceOS) { function launchSimulator(simulator) { console.log(`Launching simulator ${simulator.name} (${simulator.udid})`); - try { - childProcess.execSync(`xcrun simctl boot "${simulator.udid}"`); - } catch (error) { - if ( - !error.message.includes('Unable to boot device in current state: Booted') - ) { - throw error; - } - } + childProcess.execFileSync( + 'xcrun', + ['simctl', 'bootstatus', simulator.udid, '-b'], + { + stdio: 'inherit', + timeout: 10 * 60 * 1000, + killSignal: 'SIGKILL', + }, + ); } function installAppOnSimulator(appPath, udid) { console.log(`Installing app at path ${appPath}`); - childProcess.execSync(`xcrun simctl install "${udid}" "${appPath}"`); + childProcess.execFileSync('xcrun', ['simctl', 'install', udid, appPath], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); } function bringSimulatorInForeground() { @@ -94,7 +99,11 @@ function sleep(ms) { async function launchAppOnSimulator(appId, udid, isDebug) { console.log('Launch the app'); - childProcess.execSync(`xcrun simctl launch "${udid}" "${appId}"`); + childProcess.execFileSync('xcrun', ['simctl', 'launch', udid, appId], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); if (isDebug) { console.log('Wait for metro to warm'); @@ -102,19 +111,17 @@ async function launchAppOnSimulator(appId, udid, isDebug) { } } -function startVideoRecording(udid, currentAttempt) { - console.log( - `Start video record using pid: video_record_${currentAttempt}.pid`, - ); +function startVideoRecording(udid, outputPath) { + console.log(`Start video recording: ${outputPath}`); - const recordingArgs = - `simctl io ${udid} recordVideo --force video_record_${currentAttempt}.mov`.split( - ' ', - ); - const recordingProcess = childProcess.spawn('xcrun', recordingArgs, { - detached: true, - stdio: 'ignore', - }); + const recordingProcess = childProcess.spawn( + 'xcrun', + ['simctl', 'io', udid, 'recordVideo', '--force', outputPath], + { + detached: true, + stdio: 'ignore', + }, + ); return recordingProcess; } @@ -179,60 +186,167 @@ async function executeFlowWithRetries( appId, udid, flow, - jsengine, - currentAttempt, + reportDirectory, + options, ) { - const recProcess = startVideoRecording(udid, currentAttempt); - try { - const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit -e APP_ID="${appId}"`; - console.info(`Executing flow: ${flow} (attempt ${currentAttempt})`); - console.log(command); - childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { - stdio: 'inherit', - timeout, - }); - - await stopVideoRecording(recProcess); - } catch (error) { - await stopVideoRecording(recProcess); + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const attemptDirectory = path.resolve( + reportDirectory, + `attempt-${attempt}`, + ); + fs.mkdirSync(attemptDirectory, {recursive: true}); + let recordingProcess; + + try { + let flowPath = flow; + if (options.resetApp) { + // Uninstall first so the old process cannot repopulate the keychain. + for (const args of [ + ['uninstall', udid, appId], + ['keychain', udid, 'reset'], + ['install', udid, options.appPath], + ['privacy', udid, 'reset', 'all', appId], + ['ui', udid, 'appearance', 'light'], + ]) { + childProcess.execFileSync('xcrun', ['simctl', ...args], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); + } + + flowPath = path.join(attemptDirectory, 'flow.yml'); + fs.writeFileSync( + flowPath, + `appId: ${JSON.stringify(appId)} +--- +- setOrientation: portrait +- launchApp +- runFlow: ${JSON.stringify(path.resolve(flow))} +`, + ); + } - if (currentAttempt < MAX_ATTEMPTS) { - console.info(`Retrying flow: ${flow}`); - await executeFlowWithRetries( - appId, + recordingProcess = startVideoRecording( udid, - flow, - jsengine, - currentAttempt + 1, + path.join(attemptDirectory, 'video.mov'), ); - } else { - console.error( - `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, + console.info(`Executing flow: ${flow} (attempt ${attempt})`); + childProcess.execFileSync( + path.join(process.env.HOME, '.maestro', 'bin', 'maestro'), + [ + '--udid', + udid, + 'test', + flowPath, + '--format', + 'junit', + '--output', + path.join(attemptDirectory, 'report.xml'), + '--debug-output', + path.join(attemptDirectory, 'logs'), + '--test-output-dir', + attemptDirectory, + '-e', + `APP_ID=${appId}`, + ], + { + env: {...process.env, MAESTRO_DRIVER_STARTUP_TIMEOUT: '1500000'}, + stdio: 'inherit', + timeout: 1000 * 60 * 10, + killSignal: 'SIGKILL', + }, ); - throw error; + return; + } catch (error) { + fs.writeFileSync(path.join(attemptDirectory, 'error.txt'), String(error)); + if (attempt === MAX_ATTEMPTS) { + console.error( + `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, + ); + throw error; + } + console.info(`Retrying flow: ${flow}`); + } finally { + await stopVideoRecording(recordingProcess); } } } -async function executeFlows(appId, udid, maestroFlow, jsengine) { - if (!fs.existsSync(maestroFlow) || !fs.lstatSync(maestroFlow).isDirectory()) { - await executeFlowWithRetries(appId, udid, maestroFlow, jsengine, 1); - return; +async function executeFlows(appId, udid, maestroFlow, jsengine, options = {}) { + const { + shardIndex = 0, + shardCount = 1, + reportDirectory = 'maestro-reports', + } = options; + if ( + !Number.isInteger(shardCount) || + shardCount < 1 || + !Number.isInteger(shardIndex) || + shardIndex < 0 || + shardIndex >= shardCount + ) { + throw new Error(`Invalid shard ${shardIndex} of ${shardCount}`); + } + if (options.resetApp && !options.appPath) { + throw new Error('Resetting app state requires an app path'); } - for (const file of fs.readdirSync(maestroFlow).sort()) { - const filePath = `${maestroFlow.replace(/\/$/, '')}/${file}`; - if (fs.lstatSync(filePath).isDirectory()) { - // Fragments pulled in via `runFlow`; they have no `launchApp` of their - // own and fail when run standalone. - if (file === 'helpers') { - continue; + function collectFlows(directory) { + if (!fs.existsSync(directory) || !fs.lstatSync(directory).isDirectory()) { + return [directory]; + } + const flows = []; + for (const file of fs.readdirSync(directory).sort()) { + const filePath = path.join(directory, file); + if (fs.lstatSync(filePath).isDirectory()) { + // Helper fragments are only runnable via their parent flow. + if (file !== 'helpers') { + flows.push(...collectFlows(filePath)); + } + } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { + flows.push(filePath); } - await executeFlows(appId, udid, filePath, jsengine); - } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { - await executeFlowWithRetries(appId, udid, filePath, jsengine, 1); } + return flows; + } + + const flows = collectFlows(maestroFlow); + if (flows.length === 0) { + throw new Error(`No Maestro flows found at ${maestroFlow}`); + } + const selectedFlows = flows.filter( + (_, index) => index % shardCount === shardIndex, + ); + fs.mkdirSync(reportDirectory, {recursive: true}); + fs.writeFileSync( + path.join(reportDirectory, 'manifest.json'), + JSON.stringify({shardIndex, shardCount, flows: selectedFlows}, null, 2), + ); + console.info( + `Shard ${shardIndex + 1}/${shardCount}: ${selectedFlows.length}/${flows.length} flows`, + ); + + const failures = []; + for (let index = 0; index < selectedFlows.length; index++) { + const flow = selectedFlows[index]; + try { + await executeFlowWithRetries( + appId, + udid, + flow, + path.join(reportDirectory, `${index}-${path.basename(flow)}`), + options, + ); + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) { + throw new AggregateError( + failures, + `${failures.length} Maestro flow(s) failed`, + ); } } @@ -249,6 +363,12 @@ async function main(args = process.argv.slice(2)) { const workingDirectory = args[5]; const deviceModel = args[6] || null; const deviceOS = args[7] || null; + const options = { + appPath, + resetApp: process.env.MAESTRO_RESET_APP === 'true', + shardIndex: Number(process.env.MAESTRO_SHARD_INDEX ?? 0), + shardCount: Number(process.env.MAESTRO_SHARD_COUNT ?? 1), + }; console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -267,7 +387,7 @@ async function main(args = process.argv.slice(2)) { installAppOnSimulator(appPath, simulator.udid); bringSimulatorInForeground(); await launchAppOnSimulator(appId, simulator.udid, isDebug); - await executeFlows(appId, simulator.udid, maestroFlow, jsengine); + await executeFlows(appId, simulator.udid, maestroFlow, jsengine, options); console.log('Test finished'); } @@ -278,4 +398,5 @@ if (require.main === module) { module.exports = { executeFlows, findAvailableSimulator, + main, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index 0660257487bf..895e771e1790 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -17,12 +17,11 @@ on: jobs: test: runs-on: macos-26-large - outputs: - status: ${{ steps.report-status.outputs.status }} strategy: fail-fast: false matrix: flavor: [Debug, Release] + shard: [0, 1, 2, 3] steps: - name: Checkout uses: actions/checkout@v6 @@ -37,7 +36,6 @@ jobs: run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests - continue-on-error: true uses: ./.github/actions/maestro-ios with: app-path: '/tmp/RNTesterBuild/RNTester.app' @@ -46,10 +44,9 @@ jobs: flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 - - name: Report status - id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT + shard-index: ${{ matrix.shard }} + shard-count: '4' + reset-app: 'true' report: needs: test @@ -60,8 +57,7 @@ jobs: steps: - id: check run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then + if [[ "${{ needs.test.result }}" != "success" ]]; then echo "status=failure" >> $GITHUB_OUTPUT else echo "status=success" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index afa299e09a1c..d04ffc9fb4ac 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -193,18 +193,6 @@ jobs: needs: test_ios_rntester if: ${{ needs.test_ios_rntester.result == 'success' }} uses: ./.github/workflows/e2e-ios-rntester.yml - secrets: inherit - - test_e2e_ios_rntester_retry_1: - needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} - uses: ./.github/workflows/e2e-ios-rntester.yml - secrets: inherit - - test_e2e_ios_rntester_retry_2: - needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} - uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true secrets: inherit