diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml
index 19ddea8417f..8e1a11c9a25 100644
--- a/.github/actions/maestro-ios/action.yml
+++ b/.github/actions/maestro-ios/action.yml
@@ -30,6 +30,10 @@ inputs:
required: false
default: ''
description: The path used to persist per-flow test results between retries
+ exclude-tags:
+ required: false
+ default: ''
+ description: Comma-separated flow tags to exclude from the test run
runs:
using: composite
@@ -78,7 +82,8 @@ runs:
"${{ inputs.working-directory }}" \
"${{ inputs.device-model }}" \
"${{ inputs.device-os }}" \
- "${{ inputs.test-state-path }}"
+ "${{ inputs.test-state-path }}" \
+ "${{ inputs.exclude-tags }}"
- name: Store Logs
if: always()
uses: ./.github/actions/upload-artifact
@@ -86,5 +91,5 @@ runs:
name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch
overwrite: true
if-no-files-found: ignore
- # Contains the recording, report and debug output of failed attempts only
+ # Only failed runs leave their report, debug output and recording here
path: /tmp/MaestroLogs
diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js
index f01ae945119..63627894016 100644
--- a/.github/workflow-scripts/__tests__/maestro-ios-test.js
+++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js
@@ -21,11 +21,27 @@ const path = require('path');
const {
collectFlows,
executeFlowSuite,
+ filterFlowsByTags,
findAvailableSimulator,
loadState,
+ parseJUnitReport,
runMaestroFlow,
} = require('../maestro-ios');
+// Simulates a single Maestro run in which the flows matching `failing` fail.
+function suiteResults(failing = () => false) {
+ return jest.fn(async (appId, udid, flows) => {
+ return new Map(
+ flows.map(flow => [
+ path.relative(process.cwd(), flow),
+ failing(flow)
+ ? {passed: false, error: 'failed assertion'}
+ : {passed: true},
+ ]),
+ );
+ });
+}
+
describe('Maestro iOS runner', () => {
let temporaryDirectory;
@@ -71,14 +87,54 @@ describe('Maestro iOS runner', () => {
]);
});
- it('retries a failing flow once on a rebooted simulator and keeps going', async () => {
+ it('excludes flows with matching tags', () => {
+ const androidOnlyFlow = path.join(temporaryDirectory, 'android-only.yml');
+ const regularFlow = path.join(temporaryDirectory, 'regular.yml');
+ fs.writeFileSync(
+ androidOnlyFlow,
+ 'appId: x\ntags:\n - android-only\n---\n- launchApp\n',
+ );
+ fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n');
+
+ expect(
+ filterFlowsByTags([androidOnlyFlow, regularFlow], ['android-only']),
+ ).toEqual([regularFlow]);
+ });
+
+ it('parses flow results from a Maestro JUnit report', () => {
+ const report = `
+
+
+
+
+ Assertion is false: "Cheesecake" is visible
+
+
+
+
+`;
+
+ expect(parseJUnitReport(report)).toEqual(
+ new Map([
+ ['flows/first.yml', {passed: true, error: 'SUCCESS'}],
+ [
+ 'flows/second.yml',
+ {passed: false, error: 'Assertion is false: "Cheesecake" is visible'},
+ ],
+ ['flows/third.yml', {passed: true, error: 'WARNING'}],
+ ]),
+ );
+ });
+
+ it('runs all flows together, then retries each failure once on a rebooted simulator', async () => {
const flows = ['first.yml', 'second.yml', 'third.yml'].map(file =>
path.join(temporaryDirectory, file),
);
const statePath = path.join(temporaryDirectory, 'state', 'results.json');
+ const executeSuite = suiteResults(flow => !flow.endsWith('first.yml'));
const executeFlow = jest.fn(async (appId, udid, flow) => {
if (flow.endsWith('second.yml')) {
- throw new Error('failed assertion');
+ throw new Error('failed again');
}
});
const resetSimulator = jest.fn();
@@ -90,33 +146,32 @@ describe('Maestro iOS runner', () => {
udid: 'device-id',
state: loadState(statePath),
statePath,
+ executeSuite,
executeFlow,
resetSimulator,
}),
).rejects.toThrow('1 Maestro flow(s) failed');
+ expect(executeSuite).toHaveBeenCalledTimes(1);
+ expect(executeSuite.mock.calls[0][2]).toEqual(flows);
expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([
- [flows[0], 1],
- [flows[1], 1],
[flows[1], 2],
- [flows[2], 1],
+ [flows[2], 2],
]);
- expect(resetSimulator).toHaveBeenCalledTimes(1);
+ expect(resetSimulator).toHaveBeenCalledTimes(2);
expect(resetSimulator).toHaveBeenCalledWith('device-id');
expect(Object.values(loadState(statePath).flows)).toEqual([
{status: 'passed', attempts: 1},
- {status: 'failed', attempts: 2, error: 'failed assertion'},
- {status: 'passed', attempts: 1},
+ {status: 'failed', attempts: 2, error: 'failed again'},
+ {status: 'passed', attempts: 2},
]);
});
- it('passes a flow that succeeds on its retry', async () => {
- const flow = path.join(temporaryDirectory, 'flaky.yml');
+ it('does not retry when every flow passes', async () => {
+ const flow = path.join(temporaryDirectory, 'first.yml');
const statePath = path.join(temporaryDirectory, 'results.json');
- const executeFlow = jest
- .fn()
- .mockRejectedValueOnce(new Error('flaky'))
- .mockResolvedValueOnce();
+ const executeFlow = jest.fn();
+ const resetSimulator = jest.fn();
await executeFlowSuite({
flows: [flow],
@@ -124,12 +179,15 @@ describe('Maestro iOS runner', () => {
udid: 'device-id',
state: loadState(statePath),
statePath,
+ executeSuite: suiteResults(),
executeFlow,
- resetSimulator: jest.fn(),
+ resetSimulator,
});
+ expect(executeFlow).not.toHaveBeenCalled();
+ expect(resetSimulator).not.toHaveBeenCalled();
expect(Object.values(loadState(statePath).flows)).toEqual([
- {status: 'passed', attempts: 2},
+ {status: 'passed', attempts: 1},
]);
});
@@ -145,28 +203,26 @@ describe('Maestro iOS runner', () => {
udid: 'device-id',
state: loadState(statePath),
statePath,
- executeFlow: jest.fn(async (appId, udid, flow) => {
- if (flow.endsWith('second.yml')) {
- throw new Error('failed assertion');
- }
- }),
+ executeSuite: suiteResults(flow => flow.endsWith('second.yml')),
+ executeFlow: jest.fn().mockRejectedValue(new Error('failed again')),
resetSimulator: jest.fn(),
}),
).rejects.toThrow();
- const retry = jest.fn();
+ const executeSuite = suiteResults();
await executeFlowSuite({
flows,
appId: 'com.example',
udid: 'device-id',
state: loadState(statePath),
statePath,
- executeFlow: retry,
+ executeSuite,
+ executeFlow: jest.fn(),
resetSimulator: jest.fn(),
});
- expect(retry).toHaveBeenCalledTimes(1);
- expect(retry.mock.calls[0][2]).toBe(flows[1]);
+ expect(executeSuite).toHaveBeenCalledTimes(1);
+ expect(executeSuite.mock.calls[0][2]).toEqual([flows[1]]);
});
it('waits for the recorder to exit before returning', async () => {
diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js
index b39051f6527..fb2c26836d1 100644
--- a/.github/workflow-scripts/maestro-ios.js
+++ b/.github/workflow-scripts/maestro-ios.js
@@ -13,7 +13,7 @@ const path = require('path');
const usage = `
=== Usage ===
-node maestro-ios.js [device_model] [device_os] [test_state_path]
+node maestro-ios.js [device_model] [device_os] [test_state_path] [exclude_tags]
@param {string} appPath - Path to the app APK
@param {string} appId - App ID that needs to be launched
@@ -24,6 +24,7 @@ node maestro-ios.js {
+ const tags = getFlowTags(flow);
+ const excludedTag = tags.find(tag => excludeTags.includes(tag));
+ if (excludedTag != null) {
+ console.info(`Skipping flow tagged ${excludedTag}: ${flow}`);
+ return false;
+ }
+ return true;
+ });
+}
+
+function decodeXmlEntities(value) {
+ return value
+ .replaceAll('"', '"')
+ .replaceAll(''', "'")
+ .replaceAll('<', '<')
+ .replaceAll('>', '>')
+ .replaceAll('&', '&');
+}
+
+// Maps each flow key in a Maestro JUnit report to whether it passed, plus the
+// failure message for flows that did not.
+function parseJUnitReport(contents) {
+ const results = new Map();
+ const testCasePattern = /]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
+ for (const [, attributes, body = ''] of contents.matchAll(testCasePattern)) {
+ const attribute = name =>
+ attributes.match(new RegExp(`\\b${name}="([^"]*)"`))?.[1];
+ const file = attribute('file');
+ if (file == null) {
+ continue;
+ }
+ const status = attribute('status');
+ const failure = body.match(/]*>([\s\S]*?)<\/failure>/)?.[1];
+ results.set(getFlowKey(decodeXmlEntities(file)), {
+ passed: status === 'SUCCESS' || status === 'WARNING',
+ error: failure != null ? decodeXmlEntities(failure.trim()) : status,
+ });
+ }
+ return results;
+}
+
+function maestroCommand(udid, flows, appId, outputPath) {
+ const flowArguments = flows.map(flow => `"${flow}"`).join(' ');
+ return `MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 $HOME/.maestro/bin/maestro --udid="${udid}" test ${flowArguments} --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`;
+}
+
+// Runs every flow in a single Maestro invocation, so the CLI and the XCTest
+// driver start once instead of once per flow. Flows missing from the report
+// (e.g. because Maestro crashed or timed out) count as failed.
+function runMaestroSuite(appId, udid, flows) {
+ const outputPath = path.join(MAESTRO_LOG_DIRECTORY, 'suite');
+ fs.mkdirSync(outputPath, {recursive: true});
+
+ const command = maestroCommand(udid, flows, appId, outputPath);
+ console.info(`Executing ${flows.length} flow(s) in a single Maestro run`);
+ console.log(command);
+ let runError = null;
+ try {
+ childProcess.execSync(command, {
+ stdio: 'inherit',
+ timeout: flows.length * FLOW_TIMEOUT_MS,
+ });
+ } catch (error) {
+ runError = error;
+ }
+
+ const reportPath = path.join(outputPath, 'report.xml');
+ const report = fs.existsSync(reportPath)
+ ? parseJUnitReport(fs.readFileSync(reportPath, 'utf8'))
+ : new Map();
+ const missingError =
+ runError instanceof Error
+ ? runError.message
+ : 'Flow did not report a result';
+ const results = new Map(
+ flows.map(flow => [
+ getFlowKey(flow),
+ report.get(getFlowKey(flow)) ?? {passed: false, error: missingError},
+ ]),
+ );
+ if ([...results.values()].every(result => result.passed)) {
+ fs.rmSync(outputPath, {recursive: true, force: true});
+ }
+ return results;
+}
+
async function runMaestroFlow(appId, udid, flow, attempt) {
const outputName = `${path.basename(flow, path.extname(flow))}-attempt-${attempt}`;
const outputPath = path.join(MAESTRO_LOG_DIRECTORY, outputName);
@@ -293,13 +404,12 @@ async function runMaestroFlow(appId, udid, flow, attempt) {
const videoPath = path.join(outputPath, 'recording.mov');
const recProcess = startVideoRecording(udid, videoPath);
try {
- const timeout = 1000 * 60 * 10; // 10 minutes
- const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit --output "${outputPath}/report.xml" --debug-output "${outputPath}" --flatten-debug-output -e APP_ID="${appId}"`;
+ const command = maestroCommand(udid, [flow], appId, outputPath);
console.info(`Executing flow: ${flow} (attempt ${attempt})`);
console.log(command);
- childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, {
+ childProcess.execSync(command, {
stdio: 'inherit',
- timeout,
+ timeout: FLOW_TIMEOUT_MS,
});
} catch (error) {
await stopVideoRecording(recProcess);
@@ -317,6 +427,7 @@ async function executeFlowSuite({
udid,
state,
statePath,
+ executeSuite = runMaestroSuite,
executeFlow = runMaestroFlow,
resetSimulator = rebootSimulator,
}) {
@@ -327,30 +438,50 @@ async function executeFlowSuite({
}
saveState(statePath, state);
- const failedFlows = [];
- for (let index = 0; index < flows.length; index++) {
- const flow = flows[index];
- const flowKey = flowKeys[index];
- const result = state.flows[flowKey];
-
- if (result.status === 'passed') {
+ const flowsToRun = flows.filter((flow, index) => {
+ if (state.flows[flowKeys[index]].status === 'passed') {
console.info(`Skipping previously passed flow: ${flow}`);
- continue;
+ return false;
}
+ return true;
+ });
- for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
- if (attempt > 1) {
- console.info(`Retrying flow: ${flow}`);
- resetSimulator(udid);
+ // First attempt: every flow in one Maestro run.
+ if (flowsToRun.length > 0) {
+ const results = await executeSuite(appId, udid, flowsToRun);
+ for (const flow of flowsToRun) {
+ const result = state.flows[getFlowKey(flow)];
+ const {passed, error} = results.get(getFlowKey(flow));
+ result.attempts += 1;
+ result.status = passed ? 'passed' : 'failed';
+ if (passed) {
+ delete result.error;
+ } else {
+ result.error = error;
}
+ }
+ saveState(statePath, state);
+ }
+
+ // Retries: each failed flow on its own, on a rebooted simulator.
+ const failedFlows = [];
+ for (const flow of flowsToRun) {
+ const flowKey = getFlowKey(flow);
+ const result = state.flows[flowKey];
+
+ for (
+ let attempt = 2;
+ attempt <= MAX_ATTEMPTS && result.status === 'failed';
+ attempt++
+ ) {
+ console.info(`Retrying flow: ${flow}`);
+ resetSimulator(udid);
result.attempts += 1;
try {
await executeFlow(appId, udid, flow, attempt);
result.status = 'passed';
delete result.error;
- break;
} catch (error) {
- result.status = 'failed';
result.error = error instanceof Error ? error.message : String(error);
} finally {
saveState(statePath, state);
@@ -375,7 +506,7 @@ async function executeFlowSuite({
}
async function main(args = process.argv.slice(2)) {
- if (args.length < 6 || args.length > 9) {
+ if (args.length < 6 || args.length > 10) {
throw new Error(`Invalid number of arguments.\n${usage}`);
}
@@ -388,6 +519,10 @@ async function main(args = process.argv.slice(2)) {
const deviceModel = args[6] || null;
const deviceOS = args[7] || null;
const statePath = args[8] || DEFAULT_STATE_PATH;
+ const excludeTags = (args[9] ?? '')
+ .split(',')
+ .map(tag => tag.trim())
+ .filter(Boolean);
console.info('\n==============================');
console.info('Running tests for iOS with the following parameters:');
@@ -400,9 +535,10 @@ async function main(args = process.argv.slice(2)) {
console.info(`DEVICE_MODEL: ${deviceModel ?? ''}`);
console.info(`DEVICE_OS: ${deviceOS ?? ''}`);
console.info(`TEST_STATE_PATH: ${statePath}`);
+ console.info(`EXCLUDE_TAGS: ${excludeTags.join(',') || ''}`);
console.info('==============================\n');
- const flows = collectFlows(maestroFlow);
+ const flows = filterFlowsByTags(collectFlows(maestroFlow), excludeTags);
const state = loadState(statePath);
const simulator = findAvailableSimulator(deviceModel, deviceOS);
@@ -431,8 +567,9 @@ if (require.main === module) {
module.exports = {
collectFlows,
executeFlowSuite,
+ filterFlowsByTags,
findAvailableSimulator,
- formatResults,
loadState,
+ parseJUnitReport,
runMaestroFlow,
};
diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml
index f5c7c0c1bba..4fba45819de 100644
--- a/.github/workflows/e2e-ios-rntester.yml
+++ b/.github/workflows/e2e-ios-rntester.yml
@@ -77,6 +77,7 @@ jobs:
device-model: iPhone-17-Pro
device-os: iOS-26-2
test-state-path: /tmp/maestro-ios-state/results.json
+ exclude-tags: android-only
- name: Store per-flow test state
if: always()
uses: ./.github/actions/upload-artifact