diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 921403489..2f63a7620 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -149,6 +149,12 @@ jobs: run: | XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" test -n "$XCTESTRUN_PATH" + # The preparation flag is process-scoped; isolate this ordering test + # from other tests that may already have synthesized input. + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes xcodebuild test-without-building \ -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ diff --git a/.github/workflows/xctest-nightly.yml b/.github/workflows/xctest-nightly.yml index 91ba7a564..93da5e7b6 100644 --- a/.github/workflows/xctest-nightly.yml +++ b/.github/workflows/xctest-nightly.yml @@ -2,9 +2,9 @@ name: XCTest Nightly # The full iOS runner XCTest suite (#1781 A7). The PR lane (ios.yml) names a hand-written # subset of the target's methods in an `-only-testing:` list; everything outside that list ran -# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only -# `testCommand` — which is the runner's server entry point rather than a test (see the step -# below). +# nowhere at all. This lane runs the test plan whole, except `testCommand` (the runner's +# server entry point). The process-scoped input-preparation regression runs in isolation; +# its result is merged with the remaining suite before checking the executed count. # # No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split # (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are @@ -117,8 +117,21 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro - # Same command as ios.yml's targeted step minus every `-only-testing:` flag, so the - # xctestrun's own test plan decides what runs — with one exception. + # The preparation regression needs a fresh process before any other synthesized + # input. Run it separately, then merge both bundles for the source-derived count. + - name: Run isolated synthesized-input preparation regression + run: | + set -euo pipefail + XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" + test -n "$XCTESTRUN_PATH" + mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")" + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes \ + -resultBundlePath "${RESULT_BUNDLE_PATH%.xcresult}-isolated.xcresult" + + # The remaining suite follows the xctestrun plan, excluding the isolated test. # # `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it # opens an NWListener and blocks in `XCTWaiter.wait(timeout: 24 * 60 * 60)` until a @@ -143,7 +156,16 @@ jobs: -xctestrun "$XCTESTRUN_PATH" \ -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \ - -resultBundlePath "$RESULT_BUNDLE_PATH" + -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes \ + -resultBundlePath "${RESULT_BUNDLE_PATH%.xcresult}-remaining.xcresult" + + - name: Merge isolated and remaining XCTest results + if: always() + run: | + set -euo pipefail + xcrun xcresulttool merge --output-path "$RESULT_BUNDLE_PATH" \ + "${RESULT_BUNDLE_PATH%.xcresult}-isolated.xcresult" \ + "${RESULT_BUNDLE_PATH%.xcresult}-remaining.xcresult" # Best-effort and never the job's verdict on its own; the step below is what asserts. # `--compact` first because a red night's summary is the large one, and the job summary diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m index dbbf4ed5b..f58f91414 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m @@ -2,6 +2,7 @@ #import "RunnerXCTestEventBridge.h" #import +#import #import static NSString *const RunnerGestureSynthesisSurface = @"event"; @@ -309,6 +310,21 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { return nil; } +static id RunnerAllocateGestureRecord( + const RunnerGestureEventBridge *bridge, + NSString *name, + NSInteger orientation, + NSInteger processID +) { + id record = ((RunnerMsgSendInitRecord)objc_msgSend)( + [bridge->core.recordClass alloc], bridge->initRecordSelector, name, orientation + ); + if (record != nil) { + ((RunnerMsgSendSetInteger)objc_msgSend)(record, bridge->core.setTargetProcessIDSelector, processID); + } + return record; +} + static NSString * _Nullable RunnerCreateEventRecord( id application, NSString *recordName, @@ -326,22 +342,39 @@ + (NSInteger)interfaceOrientationForApplication:(id)application { return @"private XCTest event synthesis unavailable: could not resolve target process ID"; } - id eventRecord = ((RunnerMsgSendInitRecord)objc_msgSend)( - [bridge->core.recordClass alloc], - bridge->initRecordSelector, - recordName, - interfaceOrientation - ); - if (eventRecord == nil) { - return @"private XCTest event synthesis failed: could not create event record"; +#if TARGET_OS_IOS + // Prepare at the shared record boundary, before any route constructs timed + // pointer paths. An empty record carries no contact (including no status-bar + // tap), and reuses the caller's resolved orientation/PID without AX or images. + static BOOL didPrepare = NO; + @synchronized ([RunnerSynthesizedGesture class]) { + if (!didPrepare) { + NSTimeInterval startedAt = NSProcessInfo.processInfo.systemUptime; + NSString *preparationError = nil; + @try { + id preparation = RunnerAllocateGestureRecord( + bridge, @"agent-device-input-preparation", interfaceOrientation, targetProcessID + ); + preparationError = preparation == nil + ? @"could not create preparation record" + : RunnerSynthesizeEventRecord(bridge, preparation); + didPrepare = preparationError == nil; + } @catch (NSException *exception) { + preparationError = RunnerFormatXCTestException(exception, @"input preparation failed"); + } + NSLog( + @"AGENT_DEVICE_RUNNER_SYNTHESIZED_INPUT_WARMUP outcome=%@ elapsedMs=%.0f detail=%@", + didPrepare ? @"performed" : @"unsupported", + (NSProcessInfo.processInfo.systemUptime - startedAt) * 1000, + preparationError ?: @"" + ); + } } - ((RunnerMsgSendSetInteger)objc_msgSend)( - eventRecord, - bridge->core.setTargetProcessIDSelector, - targetProcessID - ); - *record = eventRecord; - return nil; +#endif + // Failed preparation is best-effort and remains eligible on the next request. + // The requested record is always fresh and retains its original timing. + *record = RunnerAllocateGestureRecord(bridge, recordName, interfaceOrientation, targetProcessID); + return *record == nil ? @"private XCTest event synthesis failed: could not create event record" : nil; } static NSString * _Nullable RunnerSynthesizeEventRecord( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInputWarmupTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInputWarmupTests.swift new file mode 100644 index 000000000..8d2fa6525 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInputWarmupTests.swift @@ -0,0 +1,74 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +private enum SynthesizedInputSpy { + static var paths: [ObjectIdentifier: Int] = [:] + static var submittedPathCounts: [Int] = [] + static var rejectNext = true +} + +private final class SynthesizedInputTarget: NSObject { + @objc var processID: Int { 42 } + @objc var interfaceOrientation: Int { 1 } +} + +private final class SynthesizedInputRecordSpy: NSObject { + @objc(addPointerEventPath:) + func addPointerEventPath(_ path: AnyObject) { + SynthesizedInputSpy.paths[ObjectIdentifier(self), default: 0] += 1 + } + + @objc(synthesizeWithError:) + func synthesizeWithError(_ error: UnsafeMutablePointer?) -> Bool { + SynthesizedInputSpy.submittedPathCounts.append( + SynthesizedInputSpy.paths.removeValue(forKey: ObjectIdentifier(self)) ?? 0 + ) + if SynthesizedInputSpy.rejectNext { + SynthesizedInputSpy.rejectNext = false + error?.pointee = NSError(domain: "WarmupTest", code: 1) + return false + } + return true + } +} +#endif + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + func testSynthesizedInputPreparationDoesNotDeliverContactsAndOrdersMixedRoutes() throws { + let recordClass = try XCTUnwrap(NSClassFromString("XCSynthesizedEventRecord")) + var originals: [(Method, IMP)] = [] + for name in ["addPointerEventPath:", "synthesizeWithError:"] { + let selector = NSSelectorFromString(name) + let method = try XCTUnwrap(class_getInstanceMethod(recordClass, selector)) + let spy = try XCTUnwrap(class_getInstanceMethod(SynthesizedInputRecordSpy.self, selector)) + originals.append((method, method_getImplementation(method))) + method_setImplementation(method, method_getImplementation(spy)) + } + defer { + for (method, implementation) in originals { method_setImplementation(method, implementation) } + SynthesizedInputSpy.paths = [:] + SynthesizedInputSpy.submittedPathCounts = [] + SynthesizedInputSpy.rejectNext = true + } + let target = SynthesizedInputTarget() + // Exercise the actual bridge entry points used by gesture/sequence, scroll, + // synthesized drag and coordinate tap. No real event reaches the simulator. + let samples: [[[String: NSNumber]]] = [[ + ["x": 10, "y": 10, "offsetMs": 0], + ["x": 20, "y": 20, "offsetMs": 100], + ]] + // A failed preparation must leave the real request available and permit the + // next route to prepare again. Only a successful empty synthesis consumes it. + XCTAssertNil(RunnerSynthesizedGesture.synthesizeGesture(withApplication: target, pointerSamples: samples)) + XCTAssertNil(RunnerSynthesizedGesture.synthesizeControlledScroll(withApplication: target, x: 10, y: 10, x2: 20, y2: 20, durationMs: 100)) + XCTAssertNil(RunnerSynthesizedGesture.synthesizeContinuousDrag(withApplication: target, x: 10, y: 10, x2: 20, y2: 20, durationMs: 100)) + XCTAssertNil(RunnerSynthesizedGesture.synthesizeTap(withApplication: target, x: 10, y: 10)) + XCTAssertNil(RunnerSynthesizedGesture.synthesizeSwipe(withApplication: target, x: 10, y: 10, x2: 20, y2: 20, durationMs: 100)) + XCTAssertNil(RunnerSynthesizedGesture.synthesizeGesture(withApplication: target, pointerSamples: samples)) + XCTAssertEqual(SynthesizedInputSpy.submittedPathCounts, [0, 1, 0, 1, 1, 1, 1, 1]) + } +#endif +} diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 177b5726e..87bfc356a 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -84,13 +84,20 @@ describe('the real tree', () => { test('the whole-bundle lanes skip the runner server entry point, which is not a test', () => { // The whole reason -skip-testing: exists in this repo. `testCommand` opens an // NWListener and waits 24 hours; an unfiltered run would hang the lane to its timeout. - const skipped = loadReport(repoRoot).flagged.filter((entry) => entry.flag === 'skip-testing'); + const report = loadReport(repoRoot); + const skipped = report.flagged.filter((entry) => entry.flag === 'skip-testing'); for (const entry of LANES.filter((entry) => entry.selection === 'whole')) { expect( skipped.filter((flag) => flag.workflow === entry.workflow).map((flag) => flag.identifier), ).toContain(ENTRY_POINT); } - for (const entry of skipped) expect(entry.identifier).toBe(ENTRY_POINT); + for (const entry of skipped) { + if (entry.identifier === ENTRY_POINT) continue; + // A real test may be excluded from the shared process only if another invocation + // on the same lane still reaches it. Removing the isolated run must fail this. + const current = LANES.find((lane) => lane.workflow === entry.workflow)!; + expect(report.reach[current.id].has(entry.identifier)).toBe(true); + } }); test('the simulator-only tests are exactly the ones the macOS build compiles out', () => { @@ -296,6 +303,40 @@ describe('a planted guard', () => { }); }); +describe('isolated invocations', () => { + const isolated = `${TARGET}/RunnerTests/testIsolated`; + const swift = source(`${ENTRY_SOURCE}extension RunnerTests {\n func testIsolated() {}\n}\n`); + const shared = `xcodebuild test-without-building -skip-testing:${ENTRY_POINT} -skip-testing:${isolated}`; + + test('a separate selected invocation restores a test skipped by the shared process', () => { + const report = buildReport(TARGET, swift, [ + ...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE), + { + workflow: NIGHTLY_WORKFLOW_FILE, + text: `xcodebuild test-without-building -only-testing:${isolated}\n${shared}`, + }, + ]); + expect([...report.reach.nightly]).toEqual([isolated]); + expect(report.entryPointReachedBy).toEqual([]); + }); + + test('removing the isolated invocation leaves that test unreachable on nightly', () => { + const report = buildReport(TARGET, swift, [ + ...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE), + { workflow: NIGHTLY_WORKFLOW_FILE, text: shared }, + ]); + expect(report.reach.nightly.has(isolated)).toBe(false); + }); + + test('a skip still wins over an only flag in the same invocation', () => { + const report = buildReport(TARGET, swift, [ + ...laneWorkflows().filter((entry) => entry.workflow !== NIGHTLY_WORKFLOW_FILE), + { workflow: NIGHTLY_WORKFLOW_FILE, text: `${shared} -only-testing:${isolated}` }, + ]); + expect(report.reach.nightly.has(isolated)).toBe(false); + }); +}); + describe('the workflow scan', () => { test('reads both flags on the multi-line xcodebuild invocation', () => { expect( diff --git a/scripts/check-xctest-selection.ts b/scripts/check-xctest-selection.ts index 6b8c36bbd..acade2e18 100644 --- a/scripts/check-xctest-selection.ts +++ b/scripts/check-xctest-selection.ts @@ -188,19 +188,26 @@ function identifiers( ); } -/** What one lane reaches: its platform's compiled set, narrowed by its flags. */ +/** Union the invocations: a skip in one process must not erase an isolated run. */ function laneReach( entry: Lane, declaredTests: readonly DeclaredTest[], - flagged: readonly FlaggedTest[], + workflowText: string, ): Set { - const only = identifiers(flagged, entry.workflow, 'only-testing'); - const skipped = identifiers(flagged, entry.workflow, 'skip-testing'); + const parts = workflowText.split(/^\s*(?:run:\s*)?xcodebuild\s+test(?:-without-building)?\b/m); + const invocations = parts.length > 1 ? parts.slice(1) : parts; + const compiled = declaredTests + .filter((test) => test.platforms.includes(entry.platform)) + .map((test) => test.identifier); return new Set( - declaredTests - .filter((test) => test.platforms.includes(entry.platform)) - .map((test) => test.identifier) - .filter((id) => (entry.selection === 'whole' || only.has(id)) && !skipped.has(id)), + invocations.flatMap((text) => { + const flagged = parseFlaggedTests(entry.workflow, text); + const only = identifiers(flagged, entry.workflow, 'only-testing'); + const skipped = identifiers(flagged, entry.workflow, 'skip-testing'); + // An explicit list narrows even a whole-bundle lane's isolated invocation. + const whole = entry.selection === 'whole' && only.size === 0; + return compiled.filter((id) => (whole || only.has(id)) && !skipped.has(id)); + }), ); } @@ -220,7 +227,14 @@ export function buildReport( // and cannot speak for anything else a workflow might select. const owned = flagged.filter((entry) => entry.identifier.startsWith(`${target}/`)); const reach = Object.fromEntries( - LANES.map((entry) => [entry.id, laneReach(entry, declaredTests, flagged)]), + LANES.map((entry) => [ + entry.id, + laneReach( + entry, + declaredTests, + workflows.find((workflow) => workflow.workflow === entry.workflow)?.text ?? '', + ), + ]), ) as Record>; const entryPoint = `${target}/${ENTRY_POINT_METHOD}`; const reachedAnywhere = new Set(LANES.flatMap((entry) => [...reach[entry.id]]));