Skip to content

Commit ae06dad

Browse files
committed
Run iOS E2E flows in a single Maestro invocation
The runner started a new `maestro test` process for every flow. Each one paid for JVM startup and an XCTest driver reinstall (~24s) plus stopping the screen recording (~3s): about 21 minutes of every ~75 minute RNTester job. - Run every pending flow in one `maestro test` invocation and read the per-flow results from its JUnit report. Flows missing from the report (Maestro crashed or timed out) count as failed. - Retry each failed flow on its own, on a rebooted simulator, with a screen recording. Passing runs don't produce recordings any more; failures keep Maestro's screenshots and debug output. - Add an `exclude-tags` input, and skip the `android-only` flows in RNTester. They only contain an Android-gated `runFlow`, but still paid the per-flow startup cost on iOS.
1 parent 7f6ddd3 commit ae06dad

4 files changed

Lines changed: 248 additions & 49 deletions

File tree

‎.github/actions/maestro-ios/action.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ inputs:
3030
required: false
3131
default: ''
3232
description: The path used to persist per-flow test results between retries
33+
exclude-tags:
34+
required: false
35+
default: ''
36+
description: Comma-separated flow tags to exclude from the test run
3337

3438
runs:
3539
using: composite
@@ -78,13 +82,14 @@ runs:
7882
"${{ inputs.working-directory }}" \
7983
"${{ inputs.device-model }}" \
8084
"${{ inputs.device-os }}" \
81-
"${{ inputs.test-state-path }}"
85+
"${{ inputs.test-state-path }}" \
86+
"${{ inputs.exclude-tags }}"
8287
- name: Store Logs
8388
if: always()
8489
uses: ./.github/actions/upload-artifact
8590
with:
8691
name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch
8792
overwrite: true
8893
if-no-files-found: ignore
89-
# Contains the recording, report and debug output of failed attempts only
94+
# Only failed runs leave their report, debug output and recording here
9095
path: /tmp/MaestroLogs

‎.github/workflow-scripts/__tests__/maestro-ios-test.js‎

Lines changed: 81 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,27 @@ const path = require('path');
2121
const {
2222
collectFlows,
2323
executeFlowSuite,
24+
filterFlowsByTags,
2425
findAvailableSimulator,
2526
loadState,
27+
parseJUnitReport,
2628
runMaestroFlow,
2729
} = require('../maestro-ios');
2830

31+
// Simulates a single Maestro run in which the flows matching `failing` fail.
32+
function suiteResults(failing = () => false) {
33+
return jest.fn(async (appId, udid, flows) => {
34+
return new Map(
35+
flows.map(flow => [
36+
path.relative(process.cwd(), flow),
37+
failing(flow)
38+
? {passed: false, error: 'failed assertion'}
39+
: {passed: true},
40+
]),
41+
);
42+
});
43+
}
44+
2945
describe('Maestro iOS runner', () => {
3046
let temporaryDirectory;
3147

@@ -71,14 +87,54 @@ describe('Maestro iOS runner', () => {
7187
]);
7288
});
7389

74-
it('retries a failing flow once on a rebooted simulator and keeps going', async () => {
90+
it('excludes flows with matching tags', () => {
91+
const androidOnlyFlow = path.join(temporaryDirectory, 'android-only.yml');
92+
const regularFlow = path.join(temporaryDirectory, 'regular.yml');
93+
fs.writeFileSync(
94+
androidOnlyFlow,
95+
'appId: x\ntags:\n - android-only\n---\n- launchApp\n',
96+
);
97+
fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n');
98+
99+
expect(
100+
filterFlowsByTags([androidOnlyFlow, regularFlow], ['android-only']),
101+
).toEqual([regularFlow]);
102+
});
103+
104+
it('parses flow results from a Maestro JUnit report', () => {
105+
const report = `<?xml version='1.0' encoding='UTF-8'?>
106+
<testsuites>
107+
<testsuite name="Test Suite" device="iPhone 17 Pro" tests="3" failures="1">
108+
<testcase id="first" name="first" classname="first" file="flows/first.yml" time="3.0" status="SUCCESS"/>
109+
<testcase id="second" name="second" classname="second" file="flows/second.yml" time="5.0" status="ERROR">
110+
<failure>Assertion is false: &quot;Cheesecake&quot; is visible</failure>
111+
</testcase>
112+
<testcase id="third" name="third" classname="third" file="flows/third.yml" status="WARNING">
113+
</testcase>
114+
</testsuite>
115+
</testsuites>`;
116+
117+
expect(parseJUnitReport(report)).toEqual(
118+
new Map([
119+
['flows/first.yml', {passed: true, error: 'SUCCESS'}],
120+
[
121+
'flows/second.yml',
122+
{passed: false, error: 'Assertion is false: "Cheesecake" is visible'},
123+
],
124+
['flows/third.yml', {passed: true, error: 'WARNING'}],
125+
]),
126+
);
127+
});
128+
129+
it('runs all flows together, then retries each failure once on a rebooted simulator', async () => {
75130
const flows = ['first.yml', 'second.yml', 'third.yml'].map(file =>
76131
path.join(temporaryDirectory, file),
77132
);
78133
const statePath = path.join(temporaryDirectory, 'state', 'results.json');
134+
const executeSuite = suiteResults(flow => !flow.endsWith('first.yml'));
79135
const executeFlow = jest.fn(async (appId, udid, flow) => {
80136
if (flow.endsWith('second.yml')) {
81-
throw new Error('failed assertion');
137+
throw new Error('failed again');
82138
}
83139
});
84140
const resetSimulator = jest.fn();
@@ -90,46 +146,48 @@ describe('Maestro iOS runner', () => {
90146
udid: 'device-id',
91147
state: loadState(statePath),
92148
statePath,
149+
executeSuite,
93150
executeFlow,
94151
resetSimulator,
95152
}),
96153
).rejects.toThrow('1 Maestro flow(s) failed');
97154

155+
expect(executeSuite).toHaveBeenCalledTimes(1);
156+
expect(executeSuite.mock.calls[0][2]).toEqual(flows);
98157
expect(executeFlow.mock.calls.map(call => [call[2], call[3]])).toEqual([
99-
[flows[0], 1],
100-
[flows[1], 1],
101158
[flows[1], 2],
102-
[flows[2], 1],
159+
[flows[2], 2],
103160
]);
104-
expect(resetSimulator).toHaveBeenCalledTimes(1);
161+
expect(resetSimulator).toHaveBeenCalledTimes(2);
105162
expect(resetSimulator).toHaveBeenCalledWith('device-id');
106163
expect(Object.values(loadState(statePath).flows)).toEqual([
107164
{status: 'passed', attempts: 1},
108-
{status: 'failed', attempts: 2, error: 'failed assertion'},
109-
{status: 'passed', attempts: 1},
165+
{status: 'failed', attempts: 2, error: 'failed again'},
166+
{status: 'passed', attempts: 2},
110167
]);
111168
});
112169

113-
it('passes a flow that succeeds on its retry', async () => {
114-
const flow = path.join(temporaryDirectory, 'flaky.yml');
170+
it('does not retry when every flow passes', async () => {
171+
const flow = path.join(temporaryDirectory, 'first.yml');
115172
const statePath = path.join(temporaryDirectory, 'results.json');
116-
const executeFlow = jest
117-
.fn()
118-
.mockRejectedValueOnce(new Error('flaky'))
119-
.mockResolvedValueOnce();
173+
const executeFlow = jest.fn();
174+
const resetSimulator = jest.fn();
120175

121176
await executeFlowSuite({
122177
flows: [flow],
123178
appId: 'com.example',
124179
udid: 'device-id',
125180
state: loadState(statePath),
126181
statePath,
182+
executeSuite: suiteResults(),
127183
executeFlow,
128-
resetSimulator: jest.fn(),
184+
resetSimulator,
129185
});
130186

187+
expect(executeFlow).not.toHaveBeenCalled();
188+
expect(resetSimulator).not.toHaveBeenCalled();
131189
expect(Object.values(loadState(statePath).flows)).toEqual([
132-
{status: 'passed', attempts: 2},
190+
{status: 'passed', attempts: 1},
133191
]);
134192
});
135193

@@ -145,28 +203,26 @@ describe('Maestro iOS runner', () => {
145203
udid: 'device-id',
146204
state: loadState(statePath),
147205
statePath,
148-
executeFlow: jest.fn(async (appId, udid, flow) => {
149-
if (flow.endsWith('second.yml')) {
150-
throw new Error('failed assertion');
151-
}
152-
}),
206+
executeSuite: suiteResults(flow => flow.endsWith('second.yml')),
207+
executeFlow: jest.fn().mockRejectedValue(new Error('failed again')),
153208
resetSimulator: jest.fn(),
154209
}),
155210
).rejects.toThrow();
156211

157-
const retry = jest.fn();
212+
const executeSuite = suiteResults();
158213
await executeFlowSuite({
159214
flows,
160215
appId: 'com.example',
161216
udid: 'device-id',
162217
state: loadState(statePath),
163218
statePath,
164-
executeFlow: retry,
219+
executeSuite,
220+
executeFlow: jest.fn(),
165221
resetSimulator: jest.fn(),
166222
});
167223

168-
expect(retry).toHaveBeenCalledTimes(1);
169-
expect(retry.mock.calls[0][2]).toBe(flows[1]);
224+
expect(executeSuite).toHaveBeenCalledTimes(1);
225+
expect(executeSuite.mock.calls[0][2]).toEqual([flows[1]]);
170226
});
171227

172228
it('waits for the recorder to exit before returning', async () => {

0 commit comments

Comments
 (0)