diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c798baa0c..9904684681 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This is the log of notable changes to EAS CLI and related packages. - [build-tools] Add `launch_app_identifier`, `launch_args`, and `open_url` inputs to the serve-sim, agent-device, argent, and Appium remote sessions, so serve-sim launches an installed app before the stream starts. The inputs are iOS only, and Android sessions keep using `eas/launch_application`. ([#4324](https://github.com/expo/eas-cli/pull/4324) by [@gwdp](https://github.com/gwdp)) - [build-tools] Stream iOS Simulator previews at 1600px and 10 Mbps. ([#4439](https://github.com/expo/eas-cli/pull/4439) by [@gwdp](https://github.com/gwdp)) - [eas-build-job] Add the `DeviceRunSession.Job` payload for EAS Simulator sessions. ([#4412](https://github.com/expo/eas-cli/pull/4412) by [@szdziedzic](https://github.com/szdziedzic)) +- [build-tools] Add a device run session runner that plans EAS Simulator sessions as a task graph, so device-independent work overlaps the boot. ([#4413](https://github.com/expo/eas-cli/pull/4413) by [@szdziedzic](https://github.com/szdziedzic)) ### 🐛 Bug fixes diff --git a/packages/build-tools/src/__tests__/context.test.ts b/packages/build-tools/src/__tests__/context.test.ts index 3ab029e7ed..02b4837670 100644 --- a/packages/build-tools/src/__tests__/context.test.ts +++ b/packages/build-tools/src/__tests__/context.test.ts @@ -13,6 +13,7 @@ import { vol } from 'memfs'; import { createMockLogger } from './utils/logger'; import { BuildContext } from '../context'; import { Datadog } from '../datadog'; +import { getWorkflowInterpolationContext } from '../utils/workflowInterpolationContext'; jest.mock('fs'); jest.mock('fs-extra'); @@ -134,7 +135,7 @@ describe('BuildContext', () => { ctx.updateJobInformation({} as Job, {} as Metadata); - expect(ctx.job.workflowInterpolationContext).toEqual({ + expect(getWorkflowInterpolationContext(ctx.job)).toEqual({ foo: 'bar', }); }); diff --git a/packages/build-tools/src/context.ts b/packages/build-tools/src/context.ts index 35f7393e59..8b8961eb28 100644 --- a/packages/build-tools/src/context.ts +++ b/packages/build-tools/src/context.ts @@ -23,6 +23,7 @@ import { Datadog } from './datadog'; import { readAppConfig } from './utils/appConfig'; import { createTemporaryEnvironmentSecretFile } from './utils/environmentSecrets'; import { PackageManager, resolvePackageManager } from './utils/packageManager'; +import { getWorkflowInterpolationContext } from './utils/workflowInterpolationContext'; export type Artifacts = Partial>; @@ -271,7 +272,7 @@ export class BuildContext { ...this._job, ...job, workflowInterpolationContext: - job.workflowInterpolationContext ?? this.job.workflowInterpolationContext, + getWorkflowInterpolationContext(job) ?? getWorkflowInterpolationContext(this.job), triggeredBy: this._job.triggeredBy, secrets: { ...this.job.secrets, diff --git a/packages/build-tools/src/customBuildContext.ts b/packages/build-tools/src/customBuildContext.ts index 113c597ea0..d500b6040a 100644 --- a/packages/build-tools/src/customBuildContext.ts +++ b/packages/build-tools/src/customBuildContext.ts @@ -22,6 +22,7 @@ import path from 'path'; import { ArtifactToUpload, BuildContext } from './context'; import { reportWorkflowHookMetricToDatadog } from './utils/hookMetrics'; import { uploadStepMetricsToWwwAsync } from './utils/stepMetrics'; +import { getWorkflowInterpolationContext } from './utils/workflowInterpolationContext'; const platformToBuildRuntimePlatform: Record = { [Platform.ANDROID]: BuildRuntimePlatform.LINUX, @@ -117,7 +118,7 @@ export class CustomBuildContext implements ExternalBuild // We omit steps, because CustomBuildContext does not have steps. public staticContext(): Omit { return { - ...this.job.workflowInterpolationContext, + ...getWorkflowInterpolationContext(this.job), expoApiServerURL: this.env.__API_SERVER_URL, job: this.job, metadata: this.metadata ?? null, @@ -138,7 +139,7 @@ export class CustomBuildContext implements ExternalBuild ...this.job, ...job, workflowInterpolationContext: - job.workflowInterpolationContext ?? this.job.workflowInterpolationContext, + getWorkflowInterpolationContext(job) ?? getWorkflowInterpolationContext(this.job), triggeredBy: this.job.triggeredBy, secrets: { ...this.job.secrets, diff --git a/packages/build-tools/src/deviceRunSession/__tests__/graph.test.ts b/packages/build-tools/src/deviceRunSession/__tests__/graph.test.ts new file mode 100644 index 0000000000..0d0dd95e28 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/__tests__/graph.test.ts @@ -0,0 +1,244 @@ +import { TaskGraphDefinitionError, executeTaskGraphAsync, validateTaskGraph } from '../graph'; +import { TaskDefinition, TaskResult } from '../types'; + +type Context = { signal: AbortSignal }; + +function deferred(): { + promise: Promise; + resolve: () => void; + reject: (err: unknown) => void; +} { + let resolve!: () => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function task( + id: string, + overrides: Partial> & { run?: TaskDefinition['run'] } = {} +): TaskDefinition { + return { + id, + displayName: id, + onFailure: 'fail-session', + run: async () => {}, + ...overrides, + }; +} + +async function runGraphAsync( + tasks: TaskDefinition[], + { signal = new AbortController().signal }: { signal?: AbortSignal } = {} +): Promise<{ results: Map; fatalError: unknown; order: string[] }> { + const order: string[] = []; + const execution = await executeTaskGraphAsync(tasks, { + signal, + createContext: (_task, taskSignal) => ({ signal: taskSignal }), + onTaskStart: started => order.push(started.id), + }); + return { ...execution, order }; +} + +describe(validateTaskGraph, () => { + it('rejects duplicate ids', () => { + expect(() => validateTaskGraph([task('a'), task('a')])).toThrow(TaskGraphDefinitionError); + }); + + it('rejects unknown dependencies', () => { + expect(() => validateTaskGraph([task('a', { needs: ['missing'] })])).toThrow( + /depends on "missing"/ + ); + }); + + it('rejects cycles', () => { + expect(() => + validateTaskGraph([task('a', { needs: ['b'] }), task('b', { after: ['a'] })]) + ).toThrow(/cycle/); + }); + + it('accepts a diamond', () => { + expect(() => + validateTaskGraph([ + task('root'), + task('left', { needs: ['root'] }), + task('right', { needs: ['root'] }), + task('join', { needs: ['left', 'right'] }), + ]) + ).not.toThrow(); + }); +}); + +describe(executeTaskGraphAsync, () => { + it('runs independent tasks concurrently and dependents after their needs', async () => { + const bootGate = deferred(); + const downloadGate = deferred(); + let installStartedAt: number | undefined; + const tasks = [ + task('boot', { run: () => bootGate.promise }), + task('download', { run: () => downloadGate.promise }), + task('install', { + needs: ['boot', 'download'], + run: async () => { + installStartedAt = Date.now(); + }, + }), + ]; + + const execution = runGraphAsync(tasks); + // Give the scheduler a tick to start the roots. + await new Promise(resolve => setImmediate(resolve)); + expect(installStartedAt).toBeUndefined(); + bootGate.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(installStartedAt).toBeUndefined(); + downloadGate.resolve(); + + const { results, fatalError, order } = await execution; + expect(fatalError).toBeUndefined(); + expect(order.slice(0, 2).sort()).toEqual(['boot', 'download']); + expect(order[2]).toBe('install'); + expect([...results.values()].map(result => result.outcome)).toEqual([ + 'success', + 'success', + 'success', + ]); + }); + + it('waits for "after" dependencies to finish with any outcome', async () => { + const tasks = [ + task('launch', { + onFailure: 'degrade-application', + run: async () => { + throw new Error('no app'); + }, + }), + task('publish', { after: ['launch'] }), + ]; + const { results, fatalError, order } = await runGraphAsync(tasks); + expect(fatalError).toBeUndefined(); + expect(order).toEqual(['launch', 'publish']); + expect(results.get('launch')?.outcome).toBe('failed'); + expect(results.get('publish')?.outcome).toBe('success'); + }); + + it('skips tasks whose needs failed and keeps the rest of the session going', async () => { + const tasks = [ + task('boot'), + task('download', { + onFailure: 'degrade-application', + run: async () => { + throw new Error('404'); + }, + }), + task('install', { needs: ['boot', 'download'], onFailure: 'degrade-application' }), + task('launch', { needs: ['install'], onFailure: 'degrade-application' }), + task('preview', { needs: ['boot'] }), + task('publish', { needs: ['preview'], after: ['launch'] }), + task('hold', { needs: ['publish'] }), + ]; + const { results, fatalError } = await runGraphAsync(tasks); + expect(fatalError).toBeUndefined(); + expect(results.get('download')?.outcome).toBe('failed'); + expect(results.get('install')).toMatchObject({ + outcome: 'skipped', + skipReason: '"download" did not succeed', + }); + expect(results.get('launch')).toMatchObject({ + outcome: 'skipped', + skipReason: '"install" did not succeed', + }); + expect(results.get('publish')?.outcome).toBe('success'); + expect(results.get('hold')?.outcome).toBe('success'); + }); + + it('stops the graph when a fail-session task fails', async () => { + const error = new Error('boot failed'); + const slowGate = deferred(); + let observedAbort = false; + const tasks = [ + task('boot', { + run: async () => { + throw error; + }, + }), + task('slow', { + onFailure: 'warn', + run: async ({ signal }) => { + signal.addEventListener('abort', () => { + observedAbort = true; + slowGate.resolve(); + }); + await slowGate.promise; + }, + }), + task('install', { needs: ['boot'] }), + task('publish', { after: ['slow'] }), + ]; + const { results, fatalError } = await runGraphAsync(tasks); + expect(fatalError).toBe(error); + expect(observedAbort).toBe(true); + expect(results.get('boot')?.outcome).toBe('failed'); + expect(results.get('slow')?.outcome).toBe('success'); + expect(results.get('install')?.outcome).toBe('skipped'); + expect(results.get('publish')).toMatchObject({ + outcome: 'skipped', + skipReason: 'the session failed before this task could start', + }); + }); + + it('does not treat warn or degrade failures as fatal', async () => { + const tasks = [ + task('prefetch', { + onFailure: 'warn', + run: async () => { + throw new Error('npm down'); + }, + }), + task('preview', { after: ['prefetch'] }), + ]; + const { results, fatalError } = await runGraphAsync(tasks); + expect(fatalError).toBeUndefined(); + expect(results.get('prefetch')?.outcome).toBe('failed'); + expect(results.get('preview')?.outcome).toBe('success'); + }); + + it('aborts when the outside signal aborts and skips what has not started', async () => { + const controller = new AbortController(); + const holdGate = deferred(); + const tasks = [ + task('hold', { + run: async ({ signal }) => { + signal.addEventListener('abort', () => holdGate.resolve()); + await holdGate.promise; + }, + }), + task('later', { needs: ['hold'] }), + ]; + const execution = runGraphAsync(tasks, { signal: controller.signal }); + await new Promise(resolve => setImmediate(resolve)); + const reason = new Error('worker cancelled the job'); + controller.abort(reason); + const { results, fatalError } = await execution; + expect(fatalError).toBe(reason); + expect(results.get('hold')?.outcome).toBe('success'); + expect(results.get('later')?.outcome).toBe('skipped'); + }); + + it('reports durations and calls finish hooks once per task', async () => { + const finished: string[] = []; + const execution = await executeTaskGraphAsync([task('a'), task('b', { needs: ['a'] })], { + signal: new AbortController().signal, + createContext: (_task, signal) => ({ signal }), + onTaskFinish: (finishedTask, result) => { + finished.push(finishedTask.id); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + }, + }); + expect(finished).toEqual(['a', 'b']); + expect(execution.results.size).toBe(2); + }); +}); diff --git a/packages/build-tools/src/deviceRunSession/__tests__/plan.test.ts b/packages/build-tools/src/deviceRunSession/__tests__/plan.test.ts new file mode 100644 index 0000000000..b6fe6f23c0 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/__tests__/plan.test.ts @@ -0,0 +1,153 @@ +import { ArchiveSourceType, BuildTrigger, DeviceRunSession, Platform } from '@expo/eas-build-job'; +import { randomUUID } from 'node:crypto'; + +import { validateTaskGraph } from '../graph'; +import { planDeviceRunSession } from '../plan'; +import { type SessionTask } from '../runtime'; + +function createJob({ + controller = DeviceRunSession.Controller.WEB_PREVIEW_ONLY, + platform = Platform.IOS, + withApplication = true, + egress, +}: { + controller?: DeviceRunSession.Controller; + platform?: Platform; + withApplication?: boolean; + egress?: DeviceRunSession.Egress; +} = {}): DeviceRunSession.Job { + return { + type: DeviceRunSession.JobType.DEVICE_RUN_SESSION, + triggeredBy: BuildTrigger.GIT_BASED_INTEGRATION, + projectArchive: { type: ArchiveSourceType.NONE }, + secrets: { robotAccessToken: 'token', environmentSecrets: [] }, + expoDevUrl: 'https://expo.dev/', + builderEnvironment: { image: 'latest', env: {} }, + initiatingUserId: randomUUID(), + appId: randomUUID(), + session: { + id: randomUUID(), + controller, + maxDurationSeconds: 1200, + ngrokTunnelDomain: 'sim.example.test', + }, + device: + platform === Platform.IOS + ? { platform: Platform.IOS } + : { + platform: Platform.ANDROID, + deviceIdentifier: 'medium_phone', + systemImagePackage: 'system-images;android-35;google_apis;x86_64', + }, + ...(withApplication ? { application: { source: { buildId: randomUUID() } } } : {}), + ...(egress ? { egress } : {}), + }; +} + +function byId(tasks: SessionTask[]): Record { + return Object.fromEntries(tasks.map(task => [task.id, task])); +} + +describe(planDeviceRunSession, () => { + it('plans an iOS web preview session that installs an application', () => { + const tasks = planDeviceRunSession(createJob()); + validateTaskGraph(tasks); + + expect(tasks.map(task => task.id)).toEqual([ + 'select_xcode', + 'boot_device', + 'prefetch_tooling', + 'fetch_turn_credentials', + 'start_pollers', + 'download_build', + 'start_preview', + 'install_build', + 'launch_application', + 'publish_remote_config', + 'hold_session', + ]); + const plan = byId(tasks); + expect(plan.boot_device.needs).toEqual(['select_xcode']); + expect(plan.download_build.needs ?? []).toEqual([]); + expect(plan.start_preview).toMatchObject({ + needs: ['boot_device'], + after: ['prefetch_tooling', 'fetch_turn_credentials'], + }); + expect(plan.install_build.needs).toEqual(['boot_device', 'download_build']); + expect(plan.launch_application.needs).toEqual(['install_build']); + expect(plan.publish_remote_config).toMatchObject({ + needs: ['start_preview'], + after: ['launch_application'], + }); + expect(plan.hold_session.needs).toEqual(['publish_remote_config']); + }); + + it('lets the device stay usable when the application cannot be installed', () => { + const plan = byId(planDeviceRunSession(createJob())); + expect(plan.download_build.onFailure).toBe('degrade-application'); + expect(plan.install_build.onFailure).toBe('degrade-application'); + expect(plan.launch_application.onFailure).toBe('degrade-application'); + expect(plan.prefetch_tooling.onFailure).toBe('warn'); + expect(plan.fetch_turn_credentials.onFailure).toBe('warn'); + expect(plan.boot_device.onFailure).toBe('fail-session'); + expect(plan.start_preview.onFailure).toBe('fail-session'); + expect(plan.publish_remote_config.onFailure).toBe('fail-session'); + }); + + it('starts local egress before the device boots on iOS', () => { + const plan = byId(planDeviceRunSession(createJob({ egress: DeviceRunSession.Egress.LOCAL }))); + expect(plan.start_local_egress.onFailure).toBe('fail-session'); + expect(plan.boot_device.needs).toEqual(['select_xcode', 'start_local_egress']); + }); + + it('plans an Android agent-device session without an application', () => { + const tasks = planDeviceRunSession( + createJob({ + controller: DeviceRunSession.Controller.AGENT_DEVICE, + platform: Platform.ANDROID, + withApplication: false, + }) + ); + validateTaskGraph(tasks); + + const ids = tasks.map(task => task.id); + expect(ids).not.toContain('select_xcode'); + expect(ids).not.toContain('start_pollers'); + expect(ids).not.toContain('download_build'); + const plan = byId(tasks); + expect(plan.boot_device.needs).toEqual([]); + // The agent-device daemon does not need the device to start. + expect(plan.start_controller).toMatchObject({ needs: [], after: ['prefetch_tooling'] }); + expect(plan.publish_remote_config).toMatchObject({ + needs: ['start_preview', 'start_controller'], + after: [], + }); + }); + + it.each([DeviceRunSession.Controller.ARGENT, DeviceRunSession.Controller.APPIUM])( + 'makes the %s controller wait for the device', + controller => { + const plan = byId(planDeviceRunSession(createJob({ controller }))); + expect(plan.start_controller.needs).toEqual(['select_xcode', 'boot_device']); + } + ); + + it('produces a valid graph for every controller, platform, application, and egress combination', () => { + for (const controller of Object.values(DeviceRunSession.Controller)) { + for (const platform of [Platform.IOS, Platform.ANDROID]) { + for (const withApplication of [true, false]) { + for (const egress of [undefined, DeviceRunSession.Egress.LOCAL]) { + if (egress && platform !== Platform.IOS) { + continue; + } + const tasks = planDeviceRunSession( + createJob({ controller, platform, withApplication, egress }) + ); + expect(() => validateTaskGraph(tasks)).not.toThrow(); + expect(tasks.at(-1)?.id).toBe('hold_session'); + } + } + } + } + }); +}); diff --git a/packages/build-tools/src/deviceRunSession/__tests__/runDeviceRunSessionJobAsync.test.ts b/packages/build-tools/src/deviceRunSession/__tests__/runDeviceRunSessionJobAsync.test.ts new file mode 100644 index 0000000000..4e7ae29b63 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/__tests__/runDeviceRunSessionJobAsync.test.ts @@ -0,0 +1,424 @@ +import { + ArchiveSourceType, + BuildTrigger, + DeviceRunSession, + Platform, + SystemError, +} from '@expo/eas-build-job'; +import { BuildRuntimePlatform } from '@expo/steps'; +import spawn from '@expo/turtle-spawn'; +import { randomUUID } from 'node:crypto'; + +import { createMockLogger } from '../../__tests__/utils/logger'; +import { BuildContext } from '../../context'; +import { CustomBuildContext } from '../../customBuildContext'; +import { Datadog } from '../../datadog'; +import { collectAndUploadServeSimMetricsAsync } from '../../steps/functions/collectServeSimMetrics'; +import { downloadBuildAsync } from '../../steps/functions/downloadBuild'; +import { installBuildAsync } from '../../steps/functions/installBuild'; +import { launchApplicationAsync } from '../../steps/functions/launchApplication'; +import { startAgentDeviceControllerAsync } from '../../steps/functions/startAgentDeviceRemoteSession'; +import { bootIosSimulatorAsync } from '../../steps/functions/startIosSimulator'; +import { startLocalEgressAsync } from '../../steps/functions/startLocalEgress'; +import { uploadIosSimulatorRecordingsAsync } from '../../steps/functions/uploadDeviceRunSessionScreenRecordings'; +import { IosSimulatorRecordingUtils } from '../../steps/utils/IosSimulatorRecordingUtils'; +import { stopLocalEgressResourcesAsync } from '../../steps/utils/localEgress'; +import { uploadRemoteSessionConfigWithLocalEgressAsync } from '../../steps/utils/localEgressSession'; +import { + fetchWebPreviewTurnArgsAsync, + selectXcodeDeveloperDirectoryAsync, + startDeviceWebPreviewWithTunnelAsync, + waitForDeviceRunSessionStoppedAsync, +} from '../../steps/utils/remoteDeviceRunSession'; +import { ServeSimMetricsRecorder } from '../../steps/utils/serveSimMetricsRecorder'; +import { runDeviceRunSessionJobAsync } from '../index'; + +jest.mock('@expo/turtle-spawn', () => ({ __esModule: true, default: jest.fn() })); +jest.mock('../../datadog', () => ({ Datadog: { distribution: jest.fn() } })); +jest.mock('../../steps/functions/startIosSimulator', () => ({ bootIosSimulatorAsync: jest.fn() })); +jest.mock('../../steps/functions/startAndroidEmulator', () => ({ + startAndroidEmulatorAsync: jest.fn(), + assertAndroidEmulatorHostSupportAsync: jest.fn(), +})); +jest.mock('../../steps/functions/downloadBuild', () => ({ downloadBuildAsync: jest.fn() })); +jest.mock('../../steps/functions/installBuild', () => ({ installBuildAsync: jest.fn() })); +jest.mock('../../steps/functions/launchApplication', () => ({ launchApplicationAsync: jest.fn() })); +jest.mock('../../steps/functions/startAgentDeviceRemoteSession', () => ({ + startAgentDeviceControllerAsync: jest.fn(), + createAgentDevicePackageSpec: (version?: string) => `agent-device@${version ?? 'latest'}`, +})); +jest.mock('../../steps/functions/startArgentRemoteSession', () => ({ + startArgentControllerAsync: jest.fn(), + ARGENT_PACKAGE_NAME: '@swmansion/argent', +})); +jest.mock('../../steps/functions/startAppiumRemoteSession', () => ({ + startAppiumControllerAsync: jest.fn(), + installAppiumAsync: jest.fn(), + resolveAppium3VersionSpec: (version?: string) => version ?? '^3', + resolveAppiumDriverName: () => 'xcuitest', +})); +jest.mock('../../steps/functions/startLocalEgress', () => ({ startLocalEgressAsync: jest.fn() })); +jest.mock('../../steps/functions/uploadDeviceRunSessionScreenRecordings', () => ({ + uploadIosSimulatorRecordingsAsync: jest.fn(), +})); +jest.mock('../../steps/functions/collectServeSimMetrics', () => ({ + collectAndUploadServeSimMetricsAsync: jest.fn(), +})); +jest.mock('../../steps/utils/IosSimulatorRecordingUtils', () => ({ + IosSimulatorRecordingUtils: { startAsync: jest.fn(), finishAsync: jest.fn() }, +})); +jest.mock('../../steps/utils/serveSimMetricsRecorder', () => ({ + ServeSimMetricsRecorder: { startAsync: jest.fn() }, +})); +jest.mock('../../steps/utils/remoteDeviceRunSession', () => ({ + startDeviceWebPreviewWithTunnelAsync: jest.fn(), + fetchWebPreviewTurnArgsAsync: jest.fn(), + selectXcodeDeveloperDirectoryAsync: jest.fn(), + waitForDeviceRunSessionStoppedAsync: jest.fn(), + getNgrokAuthtokenOrThrow: jest.fn(() => 'ngrok-token'), +})); +jest.mock('../../steps/utils/localEgressSession', () => ({ + uploadRemoteSessionConfigWithLocalEgressAsync: jest.fn(), +})); +jest.mock('../../steps/utils/localEgress', () => ({ stopLocalEgressResourcesAsync: jest.fn() })); + +const mocked = { + spawn: jest.mocked(spawn), + boot: jest.mocked(bootIosSimulatorAsync), + download: jest.mocked(downloadBuildAsync), + install: jest.mocked(installBuildAsync), + launch: jest.mocked(launchApplicationAsync), + agentDevice: jest.mocked(startAgentDeviceControllerAsync), + localEgress: jest.mocked(startLocalEgressAsync), + uploadRecordings: jest.mocked(uploadIosSimulatorRecordingsAsync), + collectMetrics: jest.mocked(collectAndUploadServeSimMetricsAsync), + recordings: jest.mocked(IosSimulatorRecordingUtils), + metricsRecorder: jest.mocked(ServeSimMetricsRecorder), + preview: jest.mocked(startDeviceWebPreviewWithTunnelAsync), + turn: jest.mocked(fetchWebPreviewTurnArgsAsync), + selectXcode: jest.mocked(selectXcodeDeveloperDirectoryAsync), + wait: jest.mocked(waitForDeviceRunSessionStoppedAsync), + publish: jest.mocked(uploadRemoteSessionConfigWithLocalEgressAsync), + stopEgress: jest.mocked(stopLocalEgressResourcesAsync), + datadog: jest.mocked(Datadog), +}; + +const previewStopAsync = jest.fn(); +const controllerStopAsync = jest.fn(); + +function createJob(overrides: Partial = {}): DeviceRunSession.Job { + return { + type: DeviceRunSession.JobType.DEVICE_RUN_SESSION, + triggeredBy: BuildTrigger.GIT_BASED_INTEGRATION, + projectArchive: { type: ArchiveSourceType.NONE }, + secrets: { robotAccessToken: 'robot-token', environmentSecrets: [] }, + expoDevUrl: 'https://expo.dev/', + builderEnvironment: { image: 'latest', env: {} }, + initiatingUserId: randomUUID(), + appId: randomUUID(), + session: { + id: 'session-id', + controller: DeviceRunSession.Controller.WEB_PREVIEW_ONLY, + maxDurationSeconds: 900, + ngrokTunnelDomain: 'sim.example.test', + }, + device: { platform: Platform.IOS, deviceIdentifier: 'iPhone 16' }, + application: { + source: { buildId: '9a2f2f1e-5d1e-4a6e-9a1f-9e2c1c6c0b11' }, + launchArgs: ['-flag'], + openUrl: 'exp://127.0.0.1:8081', + }, + ...overrides, + }; +} + +function createContext( + job: DeviceRunSession.Job, + env: Record = {} +): BuildContext { + return new BuildContext(job, { + env: { __API_SERVER_URL: 'http://api.expo.test', ...env }, + logBuffer: { getLogs: () => [], getPhaseLogs: () => [] }, + logger: createMockLogger(), + uploadArtifact: jest.fn(), + workingdir: '', + }); +} + +function callOrder(fn: jest.Mock | jest.MockedFunction): number { + return fn.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY; +} + +describe(runDeviceRunSessionJobAsync, () => { + beforeEach(() => { + jest.clearAllMocks(); + jest + .spyOn(CustomBuildContext.prototype, 'runtimePlatform', 'get') + .mockReturnValue(BuildRuntimePlatform.DARWIN); + mocked.spawn.mockResolvedValue({ stdout: '', stderr: '' } as any); + mocked.boot.mockResolvedValue({ + udid: 'UDID' as never, + deviceIdentifier: 'iPhone 16' as never, + displayName: 'iPhone 16 (18.0)', + }); + mocked.download.mockResolvedValue({ artifactPath: '/tmp/app.app' }); + mocked.install.mockResolvedValue({ applicationIdentifier: 'dev.expo.app' }); + mocked.launch.mockResolvedValue(undefined); + mocked.preview.mockResolvedValue({ + previewPageUrl: 'https://expo.dev/simulator-preview/preview-id', + apiUrl: 'https://web-preview.example.test', + previewToken: 'preview-token', + stopAsync: previewStopAsync, + }); + mocked.turn.mockResolvedValue(['--stun-url', 'stun:example.test']); + mocked.selectXcode.mockResolvedValue(undefined); + mocked.wait.mockResolvedValue(undefined); + mocked.publish.mockResolvedValue(undefined); + mocked.stopEgress.mockResolvedValue(undefined); + mocked.recordings.startAsync.mockResolvedValue(undefined); + mocked.recordings.finishAsync.mockResolvedValue([]); + mocked.metricsRecorder.startAsync.mockResolvedValue(undefined); + mocked.uploadRecordings.mockResolvedValue(undefined); + mocked.collectMetrics.mockResolvedValue(undefined); + mocked.agentDevice.mockResolvedValue({ + remoteConfig: { + agentDeviceRemoteSessionUrl: 'https://agent-device.example.test', + agentDeviceRemoteSessionToken: 'daemon-token', + }, + daemonPort: 4321, + getLastEventObservedAt: () => undefined, + stopAsync: controllerStopAsync, + }); + previewStopAsync.mockResolvedValue(undefined); + controllerStopAsync.mockResolvedValue(undefined); + }); + + it('runs an iOS web preview session with an application in dependency order', async () => { + await runDeviceRunSessionJobAsync(createContext(createJob())); + + // Xcode is selected before anything boots or polls. + expect(callOrder(mocked.selectXcode)).toBeLessThan(callOrder(mocked.boot)); + expect(callOrder(mocked.selectXcode)).toBeLessThan(callOrder(mocked.recordings.startAsync)); + // The download does not wait for the boot; the install waits for both. + expect(mocked.download).toHaveBeenCalledWith( + expect.objectContaining({ + buildId: '9a2f2f1e-5d1e-4a6e-9a1f-9e2c1c6c0b11', + extensions: ['app'], + robotAccessToken: 'robot-token', + }) + ); + expect(callOrder(mocked.install)).toBeGreaterThan(callOrder(mocked.boot)); + expect(callOrder(mocked.install)).toBeGreaterThan(callOrder(mocked.download)); + expect(mocked.install).toHaveBeenCalledWith( + expect.objectContaining({ + artifactPath: '/tmp/app.app', + runtimePlatform: BuildRuntimePlatform.DARWIN, + }) + ); + expect(mocked.launch).toHaveBeenCalledWith( + expect.objectContaining({ + applicationIdentifier: 'dev.expo.app', + launchArgs: ['-flag'], + openUrl: 'exp://127.0.0.1:8081', + }) + ); + // The preview starts after the boot, with the pre-fetched TURN args and the typed config. + expect(callOrder(mocked.preview)).toBeGreaterThan(callOrder(mocked.boot)); + expect(mocked.preview).toHaveBeenCalledWith( + expect.any(CustomBuildContext), + expect.objectContaining({ + runtimePlatform: BuildRuntimePlatform.DARWIN, + baseDomain: 'sim.example.test', + deviceRunSessionId: 'session-id', + turnArgs: ['--stun-url', 'stun:example.test'], + timeoutMs: 60_000, + }) + ); + // The launch_application task launches the app, so serve-sim gets no launch options. + expect(mocked.preview.mock.calls[0][1]).not.toHaveProperty('launchAppIdentifier'); + // The remote session is published after the launch, then held. + expect(callOrder(mocked.publish)).toBeGreaterThan(callOrder(mocked.launch)); + expect(mocked.publish).toHaveBeenCalledWith( + expect.objectContaining({ + deviceRunSessionId: 'session-id', + remoteConfig: { + previewUrl: 'https://expo.dev/simulator-preview/preview-id', + previewApiUrl: 'https://web-preview.example.test', + previewToken: 'preview-token', + }, + }) + ); + expect(callOrder(mocked.wait)).toBeGreaterThan(callOrder(mocked.publish)); + expect(mocked.wait).toHaveBeenCalledWith( + expect.objectContaining({ + deviceRunSessionId: 'session-id', + maxDurationSeconds: 900, + idleTimeout: undefined, + }) + ); + // Teardown runs after the session and uploads what the pollers collected. + expect(callOrder(previewStopAsync)).toBeGreaterThan(callOrder(mocked.wait)); + expect(mocked.uploadRecordings).toHaveBeenCalledTimes(1); + expect(mocked.collectMetrics).toHaveBeenCalledWith( + expect.any(CustomBuildContext), + expect.objectContaining({ deviceRunSessionId: 'session-id' }) + ); + expect(mocked.stopEgress).toHaveBeenCalledTimes(1); + // Every task reports its duration. + expect(mocked.datadog.distribution).toHaveBeenCalledWith( + 'device_run_session.task_duration_ms', + expect.any(Number), + expect.objectContaining({ + task: 'boot_device', + result: 'success', + controller: 'web-preview-only', + }) + ); + expect(mocked.datadog.distribution).toHaveBeenCalledWith( + 'device_run_session.time_to_remote_config_ms', + expect.any(Number), + expect.objectContaining({ device_platform: 'ios' }) + ); + }); + + it('prefetches the preview package while the device boots', async () => { + await runDeviceRunSessionJobAsync(createContext(createJob())); + + expect(mocked.spawn).toHaveBeenCalledWith( + 'npx', + ['--yes', '--package', '@expo/serve-sim@latest', '--call', 'true'], + expect.anything() + ); + }); + + it('prefetches with the package manager the session overrides', async () => { + await runDeviceRunSessionJobAsync( + createContext(createJob(), { EAS_OVERRIDE_PACKAGE_MANAGER: 'bun' }) + ); + + expect(mocked.spawn).toHaveBeenCalledWith( + 'bun', + ['add', '@expo/serve-sim@latest'], + expect.objectContaining({ cwd: expect.stringContaining('eas-session-prefetch-') }) + ); + expect(mocked.spawn).not.toHaveBeenCalledWith('npx', expect.anything(), expect.anything()); + }); + + it('keeps the device usable when the application cannot be downloaded', async () => { + mocked.download.mockRejectedValue(new Error('404 Not Found')); + + await expect(runDeviceRunSessionJobAsync(createContext(createJob()))).resolves.toBeUndefined(); + + expect(mocked.install).not.toHaveBeenCalled(); + expect(mocked.launch).not.toHaveBeenCalled(); + expect(mocked.publish).toHaveBeenCalledTimes(1); + expect(mocked.wait).toHaveBeenCalledTimes(1); + expect(mocked.datadog.distribution).toHaveBeenCalledWith( + 'device_run_session.task_duration_ms', + expect.any(Number), + expect.objectContaining({ task: 'download_build', result: 'failed' }) + ); + }); + + it('fails the session when the device does not boot, and still cleans up', async () => { + const bootError = new Error('simctl boot failed'); + mocked.boot.mockRejectedValue(bootError); + + await expect(runDeviceRunSessionJobAsync(createContext(createJob()))).rejects.toBe(bootError); + + expect(mocked.install).not.toHaveBeenCalled(); + expect(mocked.preview).not.toHaveBeenCalled(); + expect(mocked.publish).not.toHaveBeenCalled(); + expect(mocked.stopEgress).toHaveBeenCalledTimes(1); + expect(mocked.collectMetrics).toHaveBeenCalledTimes(1); + }); + + it('merges the controller and preview into the remote config and applies the idle timeout', async () => { + const job = createJob({ + session: { + id: 'session-id', + controller: DeviceRunSession.Controller.AGENT_DEVICE, + maxDurationSeconds: 900, + maxIdleTimeMinutes: 5, + packageVersion: '0.9.0', + ngrokTunnelDomain: 'sim.example.test', + }, + application: undefined, + }); + + await runDeviceRunSessionJobAsync(createContext(job)); + + expect(mocked.agentDevice).toHaveBeenCalledWith( + expect.any(CustomBuildContext), + expect.objectContaining({ + deviceRunSessionId: 'session-id', + packageVersion: '0.9.0', + ngrokTunnelDomain: 'sim.example.test', + ngrokAuthtoken: 'ngrok-token', + }) + ); + // Other controllers preview with the latest preview package, not their own version. + expect(mocked.preview).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ packageVersion: undefined }) + ); + // agent-device installs with Bun unless the session overrides the package manager. + expect(mocked.spawn).toHaveBeenCalledWith( + 'bun', + ['add', 'agent-device@0.9.0'], + expect.objectContaining({ cwd: expect.stringContaining('eas-session-prefetch-') }) + ); + expect(mocked.publish).toHaveBeenCalledWith( + expect.objectContaining({ + remoteConfig: { + agentDeviceRemoteSessionUrl: 'https://agent-device.example.test', + agentDeviceRemoteSessionToken: 'daemon-token', + webPreviewUrl: 'https://expo.dev/simulator-preview/preview-id', + previewApiUrl: 'https://web-preview.example.test', + webPreviewToken: 'preview-token', + }, + }) + ); + expect(mocked.wait).toHaveBeenCalledWith( + expect.objectContaining({ + idleTimeout: expect.objectContaining({ maxIdleTimeMinutes: 5 }), + }) + ); + expect(callOrder(controllerStopAsync)).toBeGreaterThan(callOrder(mocked.wait)); + }); + + it('starts local egress before the boot when requested', async () => { + mocked.localEgress.mockResolvedValue(undefined); + + await runDeviceRunSessionJobAsync( + createContext(createJob({ egress: DeviceRunSession.Egress.LOCAL })) + ); + + expect(mocked.localEgress).toHaveBeenCalledWith( + expect.objectContaining({ + ngrokTunnelDomain: 'sim.example.test', + ngrokAuthtoken: 'ngrok-token', + }) + ); + expect(callOrder(mocked.localEgress)).toBeLessThan(callOrder(mocked.boot)); + }); + + it('rejects an invalid job before starting anything', async () => { + const job = { ...createJob(), session: undefined } as unknown as DeviceRunSession.Job; + + await expect(runDeviceRunSessionJobAsync(createContext(job))).rejects.toThrow(SystemError); + + expect(mocked.boot).not.toHaveBeenCalled(); + }); + + it('refuses to run a session for another platform than the worker', async () => { + jest + .spyOn(CustomBuildContext.prototype, 'runtimePlatform', 'get') + .mockReturnValue(BuildRuntimePlatform.LINUX); + + await expect(runDeviceRunSessionJobAsync(createContext(createJob()))).rejects.toThrow( + /wrong resource class/ + ); + expect(mocked.boot).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/build-tools/src/deviceRunSession/__tests__/runtime.test.ts b/packages/build-tools/src/deviceRunSession/__tests__/runtime.test.ts new file mode 100644 index 0000000000..d6bfe7d056 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/__tests__/runtime.test.ts @@ -0,0 +1,71 @@ +import { createMockLogger } from '../../__tests__/utils/logger'; +import { TeardownStack, formatDuration } from '../runtime'; + +describe(TeardownStack, () => { + it('runs actions in reverse order of registration', async () => { + const stack = new TeardownStack(); + const order: string[] = []; + stack.push('first', async () => { + order.push('first'); + }); + stack.push('second', async () => { + order.push('second'); + }); + stack.push('third', async () => { + order.push('third'); + }); + + await stack.runAsync({ logger: createMockLogger() }); + + expect(order).toEqual(['third', 'second', 'first']); + expect(stack.size).toBe(0); + }); + + it('keeps going when an action fails and logs the failure', async () => { + const stack = new TeardownStack(); + const logger = createMockLogger(); + const order: string[] = []; + stack.push('first', async () => { + order.push('first'); + }); + stack.push('broken', async () => { + throw new Error('boom'); + }); + + await stack.runAsync({ logger }); + + expect(order).toEqual(['first']); + expect(logger.warn).toHaveBeenCalledWith( + expect.objectContaining({ err: expect.any(Error) }), + 'Could not clean up broken.' + ); + }); + + it('gives up on an action that outlives the deadline and still runs the rest', async () => { + const stack = new TeardownStack(); + const logger = createMockLogger(); + const order: string[] = []; + stack.push('first', async () => { + order.push('first'); + }); + stack.push('stuck', () => new Promise(() => {})); + + await stack.runAsync({ logger, deadlineMs: 50 }); + + expect(logger.warn).toHaveBeenCalledWith(expect.anything(), 'Could not clean up stuck.'); + // The deadline has passed, so the remaining action is skipped with a warning + // instead of running without a bound. + expect(logger.warn).toHaveBeenCalledWith( + 'Skipping cleanup of first: the cleanup deadline has passed.' + ); + expect(order).toEqual([]); + }); +}); + +describe(formatDuration, () => { + it('formats milliseconds and seconds', () => { + expect(formatDuration(250)).toBe('250ms'); + expect(formatDuration(1500)).toBe('1.5s'); + expect(formatDuration(65_000)).toBe('65.0s'); + }); +}); diff --git a/packages/build-tools/src/deviceRunSession/graph.ts b/packages/build-tools/src/deviceRunSession/graph.ts new file mode 100644 index 0000000000..a6520bfdf0 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/graph.ts @@ -0,0 +1,193 @@ +import { TaskDefinition, TaskResult } from './types'; + +export class TaskGraphDefinitionError extends Error {} + +export interface TaskGraphExecution { + results: Map; + /** Set when a `fail-session` task failed or the graph was aborted from outside. */ + fatalError: unknown; +} + +/** + * Runs tasks as soon as their dependencies allow, concurrently. + * + * A task starts when every `needs` task succeeded and every `after` task has + * finished. A task whose `needs` include a failed or skipped task is skipped. + * A failed `fail-session` task, or an aborted `signal`, stops the graph: no new + * task starts, running tasks see the aborted context signal, and the execution + * resolves once they have settled. The caller decides what to do with + * `fatalError`; the graph itself never throws for task failures. + */ +export async function executeTaskGraphAsync( + tasks: readonly TaskDefinition[], + { + signal, + createContext, + onTaskStart, + onTaskFinish, + }: { + signal: AbortSignal; + createContext: (task: TaskDefinition, signal: AbortSignal) => TContext; + onTaskStart?: (task: TaskDefinition) => void; + onTaskFinish?: (task: TaskDefinition, result: TaskResult) => void; + } +): Promise { + validateTaskGraph(tasks); + + const tasksById = new Map(tasks.map(task => [task.id, task])); + const results = new Map(); + const pending = new Set(tasks.map(task => task.id)); + const running = new Map>(); + const graphAbortController = new AbortController(); + const taskSignal = AbortSignal.any([signal, graphAbortController.signal]); + let fatalError: unknown; + let hasFatalError = false; + + const recordFatalError = (error: unknown): void => { + if (!hasFatalError) { + hasFatalError = true; + fatalError = error; + graphAbortController.abort(error); + } + }; + + const settle = (task: TaskDefinition, result: TaskResult): void => { + results.set(task.id, result); + pending.delete(task.id); + onTaskFinish?.(task, result); + }; + + const skip = (task: TaskDefinition, skipReason: string): void => { + settle(task, { id: task.id, outcome: 'skipped', durationMs: 0, skipReason }); + }; + + const start = (task: TaskDefinition): void => { + pending.delete(task.id); + onTaskStart?.(task); + const startedAt = Date.now(); + const promise = (async () => { + try { + await task.run(createContext(task, taskSignal)); + settle(task, { id: task.id, outcome: 'success', durationMs: Date.now() - startedAt }); + } catch (error) { + settle(task, { id: task.id, outcome: 'failed', durationMs: Date.now() - startedAt, error }); + if (task.onFailure === 'fail-session') { + recordFatalError(error); + } + } finally { + running.delete(task.id); + } + })(); + running.set(task.id, promise); + }; + + const onAbort = (): void => { + recordFatalError(signal.reason ?? new Error('The device run session was aborted.')); + }; + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener('abort', onAbort, { once: true }); + } + + try { + while (pending.size > 0 || running.size > 0) { + // Settle every task that can be decided without running anything, then + // start every task that is ready. Repeat until nothing changes. + let changed = true; + while (changed) { + changed = false; + for (const id of [...pending]) { + const task = tasksById.get(id)!; + if (hasFatalError) { + skip(task, 'the session failed before this task could start'); + changed = true; + continue; + } + const blockedBy = (task.needs ?? []).find(need => { + const result = results.get(need); + return result !== undefined && result.outcome !== 'success'; + }); + if (blockedBy !== undefined) { + skip(task, `"${blockedBy}" did not succeed`); + changed = true; + continue; + } + const isReady = + (task.needs ?? []).every(need => results.get(need)?.outcome === 'success') && + (task.after ?? []).every(dependency => results.has(dependency)); + if (isReady) { + start(task); + changed = true; + } + } + } + + if (running.size === 0) { + // Only possible with a dependency on a task that never settles, which + // validation rules out. Fail loudly rather than hang. + for (const id of [...pending]) { + skip(tasksById.get(id)!, 'no path to start this task'); + } + break; + } + await Promise.race(running.values()); + } + } finally { + signal.removeEventListener('abort', onAbort); + } + + return { results, fatalError }; +} + +/** Rejects duplicate ids, unknown dependencies, and cycles. */ +export function validateTaskGraph(tasks: readonly TaskDefinition[]): void { + const ids = new Set(); + for (const task of tasks) { + if (ids.has(task.id)) { + throw new TaskGraphDefinitionError(`Task "${task.id}" is defined twice.`); + } + ids.add(task.id); + } + for (const task of tasks) { + for (const dependency of [...(task.needs ?? []), ...(task.after ?? [])]) { + if (!ids.has(dependency)) { + throw new TaskGraphDefinitionError( + `Task "${task.id}" depends on "${dependency}", which is not defined.` + ); + } + } + } + + // Kahn's algorithm: every task must be reachable from the roots. + const remainingDependencies = new Map( + tasks.map(task => [task.id, new Set([...(task.needs ?? []), ...(task.after ?? [])])]) + ); + const dependents = new Map(); + for (const task of tasks) { + for (const dependency of remainingDependencies.get(task.id)!) { + dependents.set(dependency, [...(dependents.get(dependency) ?? []), task.id]); + } + } + const ready = tasks.filter(task => remainingDependencies.get(task.id)!.size === 0).map(t => t.id); + let visited = 0; + while (ready.length > 0) { + const id = ready.pop()!; + visited += 1; + for (const dependent of dependents.get(id) ?? []) { + const remaining = remainingDependencies.get(dependent)!; + remaining.delete(id); + if (remaining.size === 0) { + ready.push(dependent); + } + } + } + if (visited !== tasks.length) { + const cyclic = [...remainingDependencies.entries()] + .filter(([, remaining]) => remaining.size > 0) + .map(([id]) => id); + throw new TaskGraphDefinitionError( + `Tasks form a dependency cycle: ${cyclic.map(id => `"${id}"`).join(', ')}.` + ); + } +} diff --git a/packages/build-tools/src/deviceRunSession/index.ts b/packages/build-tools/src/deviceRunSession/index.ts new file mode 100644 index 0000000000..3fa45255c7 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/index.ts @@ -0,0 +1,125 @@ +import { DeviceRunSession, SystemError } from '@expo/eas-build-job'; +import { bunyan } from '@expo/logger'; +import { z } from 'zod'; + +import { BuildContext } from '../context'; +import { CustomBuildContext } from '../customBuildContext'; +import { stopLocalEgressResourcesAsync } from '../steps/utils/localEgress'; +import { executeTaskGraphAsync } from './graph'; +import { planDeviceRunSession } from './plan'; +import { SessionRuntime, type SessionTask } from './runtime'; +import { type TaskResult } from './types'; + +export { planDeviceRunSession } from './plan'; +export { + SessionRuntime, + type SessionState, + type SessionTask, + type SessionTaskContext, +} from './runtime'; +export { executeTaskGraphAsync, validateTaskGraph, TaskGraphDefinitionError } from './graph'; +export type { TaskDefinition, TaskFailurePolicy, TaskOutcome, TaskResult } from './types'; + +const TEARDOWN_TASK = { id: 'teardown', displayName: 'Clean up', onFailure: 'warn' } as const; + +/** + * Runs an EAS Simulator device run session from a `DeviceRunSession.Job`. + * + * Plans the session as a task graph, runs it with the concurrency the + * dependencies allow, and tears everything down in reverse order whatever the + * outcome. Resolves when the session has ended; rejects with the first + * session-failing error, which the worker reports on the job run. + */ +export async function runDeviceRunSessionJobAsync( + ctx: BuildContext, + { signal = new AbortController().signal }: { signal?: AbortSignal } = {} +): Promise { + const parsed = DeviceRunSession.JobZ.safeParse(ctx.job); + if (!parsed.success) { + throw new SystemError( + `The device run session job is invalid, so the worker cannot run it. This is a bug in the API server that created the job.\n${z.prettifyError(parsed.error)}` + ); + } + + const customBuildCtx = new CustomBuildContext(ctx); + const runtime = new SessionRuntime(customBuildCtx); + if (customBuildCtx.runtimePlatform !== runtime.runtimePlatform) { + throw new SystemError( + `This worker runs ${customBuildCtx.runtimePlatform} but the session requests a ${runtime.device.platform} device. The API server scheduled the job on the wrong resource class.` + ); + } + + const tasks = planDeviceRunSession(runtime.job); + runtime.logger.info( + { + controller: runtime.session.controller, + device: runtime.device, + application: runtime.job.application ?? null, + egress: runtime.job.egress ?? null, + tasks: tasks.map(task => ({ + id: task.id, + needs: task.needs ?? [], + after: task.after ?? [], + onFailure: task.onFailure, + })), + }, + 'Planned the device run session.' + ); + + const taskLoggers = new Map(); + const loggerFor = (task: SessionTask): bunyan => { + let logger = taskLoggers.get(task.id); + if (!logger) { + logger = runtime.createTaskLogger(task); + taskLoggers.set(task.id, logger); + } + return logger; + }; + + const results: TaskResult[] = []; + let fatalError: unknown; + try { + const execution = await executeTaskGraphAsync(tasks, { + signal, + createContext: (task, taskSignal) => ({ + logger: loggerFor(task), + signal: taskSignal, + runtime, + }), + onTaskStart: task => runtime.logTaskStart(task, loggerFor(task)), + onTaskFinish: (task, result) => { + results.push(result); + runtime.logTaskFinish(task, result, loggerFor(task)); + runtime.reportTaskMetric(result); + }, + }); + fatalError = execution.fatalError; + } finally { + const teardownLogger = runtime.createTaskLogger(TEARDOWN_TASK); + const startedAt = Date.now(); + runtime.logTaskStart(TEARDOWN_TASK, teardownLogger); + await runtime.teardown.runAsync({ logger: teardownLogger }); + // The egress task may have registered resources without finishing. + await stopLocalEgressResourcesAsync(teardownLogger); + await customBuildCtx.drainPendingMetricUploads(); + runtime.logTaskFinish( + TEARDOWN_TASK, + { id: TEARDOWN_TASK.id, outcome: 'success', durationMs: Date.now() - startedAt }, + teardownLogger + ); + } + + const degraded = results.filter(result => result.outcome === 'failed' && !isFatal(result)); + if (degraded.length > 0) { + runtime.logger.warn( + `The session ran without: ${degraded.map(result => result.id).join(', ')}.` + ); + } + if (fatalError !== undefined) { + throw fatalError instanceof Error ? fatalError : new Error(String(fatalError)); + } + + function isFatal(result: TaskResult): boolean { + return result.error !== undefined && result.error === fatalError; + } +} diff --git a/packages/build-tools/src/deviceRunSession/plan.ts b/packages/build-tools/src/deviceRunSession/plan.ts new file mode 100644 index 0000000000..1730b0a2b3 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/plan.ts @@ -0,0 +1,108 @@ +import { DeviceRunSession, Platform } from '@expo/eas-build-job'; + +import { type SessionTask } from './runtime'; +import { + DOWNLOAD_BUILD_TASK_ID, + INSTALL_BUILD_TASK_ID, + LAUNCH_APPLICATION_TASK_ID, + createDownloadBuildTask, + createInstallBuildTask, + createLaunchApplicationTask, +} from './tasks/application'; +import { START_CONTROLLER_TASK_ID, createStartControllerTask } from './tasks/controller'; +import { BOOT_DEVICE_TASK_ID, createBootDeviceTask } from './tasks/device'; +import { START_LOCAL_EGRESS_TASK_ID, createStartLocalEgressTask } from './tasks/egress'; +import { createHoldSessionTask } from './tasks/hold'; +import { + FETCH_TURN_CREDENTIALS_TASK_ID, + SELECT_XCODE_TASK_ID, + createFetchTurnCredentialsTask, + createSelectXcodeTask, +} from './tasks/network'; +import { createStartPollersTask } from './tasks/pollers'; +import { START_PREVIEW_TASK_ID, createStartPreviewTask } from './tasks/preview'; +import { PUBLISH_REMOTE_CONFIG_TASK_ID, createPublishRemoteConfigTask } from './tasks/publish'; +import { PREFETCH_TOOLING_TASK_ID, createPrefetchToolingTask } from './tasks/tooling'; + +/** + * Turns a session job into the task graph the runner executes. + * + * The graph encodes the session's real dependencies instead of a step order. + * Everything that needs no device (downloading the application, caching tool + * packages, fetching TURN credentials, starting pollers) runs while the device + * boots. The preview and the controller start as soon as the device is ready, + * concurrently with the application install. The remote session is published + * once the preview and controller are up and the application launch has finished. + */ +export function planDeviceRunSession(job: DeviceRunSession.Job): SessionTask[] { + const isIos = job.device.platform === Platform.IOS; + const hasApplication = job.application !== undefined; + const { controller } = job.session; + const hasController = controller !== DeviceRunSession.Controller.WEB_PREVIEW_ONLY; + + const tasks: SessionTask[] = []; + + // Anything that must be in place before the device boots. + const bootNeeds: string[] = []; + if (isIos) { + // The selected Xcode decides which Simulator runtime boots and which tools + // the preview and controller use, so select it once, before everything else. + tasks.push(createSelectXcodeTask()); + bootNeeds.push(SELECT_XCODE_TASK_ID); + } + if (job.egress === DeviceRunSession.Egress.LOCAL) { + // The Simulator reads the system proxy at boot. + tasks.push(createStartLocalEgressTask()); + bootNeeds.push(START_LOCAL_EGRESS_TASK_ID); + } + tasks.push(createBootDeviceTask({ needs: bootNeeds })); + + // Device-independent work that overlaps the boot. + tasks.push(createPrefetchToolingTask()); + tasks.push(createFetchTurnCredentialsTask()); + if (isIos) { + tasks.push(createStartPollersTask({ needs: [SELECT_XCODE_TASK_ID] })); + } + if (hasApplication) { + tasks.push(createDownloadBuildTask()); + } + + // Work that needs the booted device. + tasks.push( + createStartPreviewTask({ + needs: [BOOT_DEVICE_TASK_ID], + after: [PREFETCH_TOOLING_TASK_ID, FETCH_TURN_CREDENTIALS_TASK_ID], + }) + ); + if (hasController) { + tasks.push( + createStartControllerTask({ + controller, + // The agent-device daemon attaches to the device lazily; Argent and Appium + // resolve the booted device when they start. + needs: [ + ...(isIos ? [SELECT_XCODE_TASK_ID] : []), + ...(controller === DeviceRunSession.Controller.AGENT_DEVICE ? [] : [BOOT_DEVICE_TASK_ID]), + ], + after: [PREFETCH_TOOLING_TASK_ID], + }) + ); + } + if (hasApplication) { + tasks.push(createInstallBuildTask({ needs: [BOOT_DEVICE_TASK_ID, DOWNLOAD_BUILD_TASK_ID] })); + tasks.push(createLaunchApplicationTask({ needs: [INSTALL_BUILD_TASK_ID] })); + } + + // Hand-off and the session itself. + tasks.push( + createPublishRemoteConfigTask({ + needs: [START_PREVIEW_TASK_ID, ...(hasController ? [START_CONTROLLER_TASK_ID] : [])], + // A failed application install or launch does not block the hand-off; the + // device stays usable and the failure is visible in the job log. + after: hasApplication ? [LAUNCH_APPLICATION_TASK_ID] : [], + }) + ); + tasks.push(createHoldSessionTask({ needs: [PUBLISH_REMOTE_CONFIG_TASK_ID] })); + + return tasks; +} diff --git a/packages/build-tools/src/deviceRunSession/runtime.ts b/packages/build-tools/src/deviceRunSession/runtime.ts new file mode 100644 index 0000000000..6bca7324ff --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/runtime.ts @@ -0,0 +1,210 @@ +import { DeviceRunSession, Platform } from '@expo/eas-build-job'; +import { bunyan } from '@expo/logger'; +import { + BuildRuntimePlatform, + BuildStepEnv, + BuildStepLogMarker, + BuildStepStatus, +} from '@expo/steps'; +import { setTimeout as setTimeoutAsync } from 'node:timers/promises'; + +import { CustomBuildContext } from '../customBuildContext'; +import { Datadog } from '../datadog'; +import { type AppiumInstallation } from '../steps/functions/startAppiumRemoteSession'; +import { type DeviceWebPreviewHandle } from '../steps/utils/remoteDeviceRunSession'; +import { TaskDefinition, TaskResult } from './types'; + +/** Whatever tool exposes the device to clients: agent-device, Argent, or Appium. */ +export type ControllerHandle = { + /** Fields merged into the session's remote config next to the web preview fields. */ + remoteConfig: Record; + /** Arrival time of the newest observed controller event, for idle detection. */ + getLastEventObservedAt: () => Date | undefined; + stopAsync: () => Promise; +}; + +/** + * Typed hand-offs between tasks. The plan's `needs` edges guarantee that a task + * reading a field runs after the task that writes it. + */ +export type SessionState = { + device?: { displayName: string; udid?: string; serialId?: string }; + turnArgs?: string[]; + download?: { artifactPath: string }; + install?: { applicationIdentifier: string; activityName?: string }; + appiumInstallation?: AppiumInstallation; + preview?: DeviceWebPreviewHandle; + controller?: ControllerHandle; + remoteConfigPublishedAt?: Date; +}; + +export type SessionTaskContext = { + /** Child logger whose lines the website groups under this task. */ + logger: bunyan; + /** Aborted when the session fails or the worker cancels the job. */ + signal: AbortSignal; + runtime: SessionRuntime; +}; + +export type SessionTask = TaskDefinition; + +const DEFAULT_TEARDOWN_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Cleanup actions registered by tasks, run in reverse order of registration. + * Every action runs even when an earlier one fails; failures are logged. + */ +export class TeardownStack { + private readonly actions: { name: string; run: (logger: bunyan) => Promise }[] = []; + + public push(name: string, run: (logger: bunyan) => Promise): void { + this.actions.push({ name, run }); + } + + public get size(): number { + return this.actions.length; + } + + public async runAsync({ + logger, + deadlineMs = DEFAULT_TEARDOWN_DEADLINE_MS, + }: { + logger: bunyan; + deadlineMs?: number; + }): Promise { + const deadline = Date.now() + deadlineMs; + while (this.actions.length > 0) { + const { name, run } = this.actions.pop()!; + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + logger.warn(`Skipping cleanup of ${name}: the cleanup deadline has passed.`); + continue; + } + const timeoutController = new AbortController(); + try { + await Promise.race([ + run(logger), + setTimeoutAsync(remainingMs, undefined, { signal: timeoutController.signal }).then(() => { + throw new Error( + `Cleanup of ${name} did not finish within ${Math.round(remainingMs / 1000)}s.` + ); + }), + ]); + } catch (err) { + logger.warn({ err }, `Could not clean up ${name}.`); + } finally { + timeoutController.abort(); + } + } + } +} + +/** Shared state for one device run session: the job, the context, hand-offs, and cleanup. */ +export class SessionRuntime { + public readonly job: DeviceRunSession.Job; + public readonly session: DeviceRunSession.Session; + public readonly device: DeviceRunSession.Device; + public readonly runtimePlatform: BuildRuntimePlatform; + public readonly startedAt = Date.now(); + public readonly state: SessionState = {}; + public readonly teardown = new TeardownStack(); + + constructor(public readonly ctx: CustomBuildContext) { + this.job = ctx.job; + this.session = ctx.job.session; + this.device = ctx.job.device; + this.runtimePlatform = + ctx.job.device.platform === Platform.IOS + ? BuildRuntimePlatform.DARWIN + : BuildRuntimePlatform.LINUX; + } + + public get env(): BuildStepEnv { + return this.ctx.env; + } + + /** Session-level logger. Lines logged here are not grouped under a task. */ + public get logger(): bunyan { + return this.ctx.logger; + } + + public get deviceRunSessionId(): string { + return this.session.id; + } + + public createTaskLogger(task: Pick): bunyan { + return this.ctx.logger.child({ + buildStepId: task.id, + buildStepDisplayName: task.displayName, + }); + } + + public logTaskStart(task: Pick, logger: bunyan): void { + logger.info({ marker: BuildStepLogMarker.START_STEP }, `Starting "${task.displayName}".`); + } + + public logTaskFinish( + task: Pick, + result: TaskResult, + logger: bunyan + ): void { + const duration = formatDuration(result.durationMs); + switch (result.outcome) { + case 'success': + logger.info( + { marker: BuildStepLogMarker.END_STEP, result: BuildStepStatus.SUCCESS }, + `Finished "${task.displayName}" in ${duration}.` + ); + break; + case 'failed': + if (task.onFailure === 'fail-session') { + logger.error({ err: result.error }); + logger.error( + { marker: BuildStepLogMarker.END_STEP, result: BuildStepStatus.FAIL }, + `"${task.displayName}" failed after ${duration}. The session cannot continue.` + ); + } else { + logger.warn({ err: result.error }); + logger.warn( + { marker: BuildStepLogMarker.END_STEP, result: BuildStepStatus.WARNING }, + task.onFailure === 'degrade-application' + ? `"${task.displayName}" failed after ${duration}. The device stays usable, but the application will not be available in this session.` + : `"${task.displayName}" failed after ${duration}. The session continues without it.` + ); + } + break; + case 'skipped': + logger.info({ marker: BuildStepLogMarker.START_STEP }, `Starting "${task.displayName}".`); + logger.info( + { marker: BuildStepLogMarker.END_STEP, result: BuildStepStatus.SKIPPED }, + `Skipped "${task.displayName}": ${result.skipReason ?? 'not needed'}.` + ); + break; + } + } + + public reportTaskMetric(result: TaskResult): void { + if (result.outcome === 'skipped') { + return; + } + Datadog.distribution('device_run_session.task_duration_ms', result.durationMs, { + task: result.id, + result: result.outcome, + ...this.metricTags(), + }); + } + + public metricTags(): Record { + return { + controller: this.session.controller, + device_platform: this.device.platform, + }; + } +} + +export function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${Math.round(durationMs)}ms`; + } + return `${(durationMs / 1000).toFixed(1)}s`; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/application.ts b/packages/build-tools/src/deviceRunSession/tasks/application.ts new file mode 100644 index 0000000000..0da2af52ce --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/application.ts @@ -0,0 +1,88 @@ +import { Platform } from '@expo/eas-build-job'; +import nullthrows from 'nullthrows'; + +import { downloadBuildAsync } from '../../steps/functions/downloadBuild'; +import { installBuildAsync } from '../../steps/functions/installBuild'; +import { launchApplicationAsync } from '../../steps/functions/launchApplication'; +import { type SessionTask } from '../runtime'; + +export const DOWNLOAD_BUILD_TASK_ID = 'download_build'; +export const INSTALL_BUILD_TASK_ID = 'install_build'; +export const LAUNCH_APPLICATION_TASK_ID = 'launch_application'; + +/** Downloads and extracts the application while the device boots. */ +export function createDownloadBuildTask(): SessionTask { + return { + id: DOWNLOAD_BUILD_TASK_ID, + displayName: 'Download application', + onFailure: 'degrade-application', + run: async ({ runtime, logger }) => { + const application = nullthrows(runtime.job.application, 'The job requests no application.'); + const source = + 'buildId' in application.source + ? { buildId: application.source.buildId } + : { applicationArchiveUrl: application.source.archiveUrl }; + logger.info( + 'buildId' in source + ? `Downloading build ${source.buildId}...` + : 'Downloading application archive...' + ); + const { artifactPath } = await downloadBuildAsync({ + logger, + ...source, + graphqlClient: runtime.ctx.graphqlClient, + robotAccessToken: runtime.job.secrets.robotAccessToken, + extensions: runtime.device.platform === Platform.IOS ? ['app'] : ['apk'], + }); + runtime.state.download = { artifactPath }; + }, + }; +} + +/** Installs the downloaded application on the booted device. */ +export function createInstallBuildTask({ needs }: { needs: readonly string[] }): SessionTask { + return { + id: INSTALL_BUILD_TASK_ID, + displayName: 'Install application', + needs, + onFailure: 'degrade-application', + run: async ({ runtime, logger }) => { + const { artifactPath } = nullthrows( + runtime.state.download, + 'The application was not downloaded.' + ); + runtime.state.install = await installBuildAsync({ + artifactPath, + runtimePlatform: runtime.runtimePlatform, + env: runtime.env, + logger, + }); + }, + }; +} + +/** Launches the installed application with the requested arguments and URL. */ +export function createLaunchApplicationTask({ needs }: { needs: readonly string[] }): SessionTask { + return { + id: LAUNCH_APPLICATION_TASK_ID, + displayName: 'Launch application', + needs, + onFailure: 'degrade-application', + run: async ({ runtime, logger }) => { + const application = nullthrows(runtime.job.application, 'The job requests no application.'); + const { applicationIdentifier, activityName } = nullthrows( + runtime.state.install, + 'The application was not installed.' + ); + await launchApplicationAsync({ + applicationIdentifier, + activityName, + launchArgs: application.launchArgs, + openUrl: application.openUrl, + runtimePlatform: runtime.runtimePlatform, + env: runtime.env, + logger, + }); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/controller.ts b/packages/build-tools/src/deviceRunSession/tasks/controller.ts new file mode 100644 index 0000000000..17583eefad --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/controller.ts @@ -0,0 +1,72 @@ +import { DeviceRunSession, SystemError } from '@expo/eas-build-job'; + +import { startAgentDeviceControllerAsync } from '../../steps/functions/startAgentDeviceRemoteSession'; +import { startAppiumControllerAsync } from '../../steps/functions/startAppiumRemoteSession'; +import { startArgentControllerAsync } from '../../steps/functions/startArgentRemoteSession'; +import { getNgrokAuthtokenOrThrow } from '../../steps/utils/remoteDeviceRunSession'; +import { type ControllerHandle, type SessionTask } from '../runtime'; + +export const START_CONTROLLER_TASK_ID = 'start_controller'; + +const controllerDisplayNames: Record = { + [DeviceRunSession.Controller.WEB_PREVIEW_ONLY]: 'Start web preview controller', + [DeviceRunSession.Controller.AGENT_DEVICE]: 'Start agent-device', + [DeviceRunSession.Controller.ARGENT]: 'Start Argent', + [DeviceRunSession.Controller.APPIUM]: 'Start Appium', +}; + +/** Starts the tool that lets clients drive the device, and tunnels it. Not used for web-preview-only sessions. */ +export function createStartControllerTask({ + controller, + needs, + after, +}: { + controller: DeviceRunSession.Controller; + needs: readonly string[]; + after: readonly string[]; +}): SessionTask { + return { + id: START_CONTROLLER_TASK_ID, + displayName: controllerDisplayNames[controller], + needs, + after, + onFailure: 'fail-session', + run: async ({ runtime, logger, signal }) => { + const { session, env, runtimePlatform } = runtime; + const options = { + deviceRunSessionId: runtime.deviceRunSessionId, + packageVersion: session.packageVersion, + ngrokTunnelDomain: session.ngrokTunnelDomain, + ngrokAuthtoken: getNgrokAuthtokenOrThrow(env), + env, + logger, + }; + + let handle: ControllerHandle; + switch (controller) { + case DeviceRunSession.Controller.AGENT_DEVICE: + handle = await startAgentDeviceControllerAsync(runtime.ctx, options); + break; + case DeviceRunSession.Controller.ARGENT: + handle = await startArgentControllerAsync(runtime.ctx, { + ...options, + runtimePlatform, + signal, + }); + break; + case DeviceRunSession.Controller.APPIUM: + handle = await startAppiumControllerAsync(runtime.ctx, { + ...options, + runtimePlatform, + installation: runtime.state.appiumInstallation, + }); + break; + case DeviceRunSession.Controller.WEB_PREVIEW_ONLY: + throw new SystemError('Web-preview-only sessions have no controller to start.'); + } + + runtime.state.controller = handle; + runtime.teardown.push(controllerDisplayNames[controller], () => handle.stopAsync()); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/device.ts b/packages/build-tools/src/deviceRunSession/tasks/device.ts new file mode 100644 index 0000000000..9d9e195881 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/device.ts @@ -0,0 +1,74 @@ +import { Platform } from '@expo/eas-build-job'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { + assertAndroidEmulatorHostSupportAsync, + startAndroidEmulatorAsync, +} from '../../steps/functions/startAndroidEmulator'; +import { bootIosSimulatorAsync } from '../../steps/functions/startIosSimulator'; +import { + type AndroidDeviceName, + AndroidEmulatorUtils, + type AndroidVirtualDeviceName, +} from '../../utils/AndroidEmulatorUtils'; +import { type IosSimulatorName, type IosSimulatorUuid } from '../../utils/IosSimulatorUtils'; +import { type SessionTask } from '../runtime'; + +export const BOOT_DEVICE_TASK_ID = 'boot_device'; +const ANDROID_VIRTUAL_DEVICE_NAME = 'EasAndroidDevice01' as AndroidVirtualDeviceName; + +/** Boots the iOS Simulator or Android emulator described by the job and waits until it accepts input. */ +export function createBootDeviceTask({ needs }: { needs: readonly string[] }): SessionTask { + return { + id: BOOT_DEVICE_TASK_ID, + displayName: 'Start device', + needs, + onFailure: 'fail-session', + run: async ({ runtime, logger }) => { + const { device, env } = runtime; + if (device.platform === Platform.IOS) { + const booted = await bootIosSimulatorAsync({ + deviceIdentifier: device.deviceIdentifier as + | IosSimulatorUuid + | IosSimulatorName + | undefined, + env, + logger, + }); + runtime.state.device = { displayName: booted.displayName, udid: booted.udid }; + return; + } + + if (env.EAS_NO_EMULATOR_HOST_SUPPORT_CHECK !== '1') { + await assertAndroidEmulatorHostSupportAsync({ env }); + } + try { + const availableDevices = await AndroidEmulatorUtils.getAvailableDevicesAsync({ env }); + logger.info(`Available Android devices:\n- ${availableDevices.join(`\n- `)}`); + } catch (error) { + logger.info('Failed to list available Android devices.', error); + } + const logcatDirectory = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'eas-android-emulator-logcat-') + ); + const adjustAnimationScale = + env.ANDROID_EMULATOR_ADJUST_ANIMATION_SCALE !== 'false' && + env.ANDROID_EMULATOR_ADJUST_ANIMATION_SCALE !== '0'; + const { serialId } = await startAndroidEmulatorAsync({ + deviceName: ANDROID_VIRTUAL_DEVICE_NAME, + deviceIdentifier: device.deviceIdentifier as AndroidDeviceName, + systemImagePackage: device.systemImagePackage, + lcdWidth: device.lcdWidth ?? null, + lcdHeight: device.lcdHeight ?? null, + lcdDensity: device.lcdDensity ?? null, + adjustAnimationScale, + logcatDirectory, + env, + logger, + }); + runtime.state.device = { displayName: ANDROID_VIRTUAL_DEVICE_NAME, serialId }; + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/egress.ts b/packages/build-tools/src/deviceRunSession/tasks/egress.ts new file mode 100644 index 0000000000..bff270da10 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/egress.ts @@ -0,0 +1,27 @@ +import { startLocalEgressAsync } from '../../steps/functions/startLocalEgress'; +import { getNgrokAuthtokenOrThrow } from '../../steps/utils/remoteDeviceRunSession'; +import { type SessionTask } from '../runtime'; + +export const START_LOCAL_EGRESS_TASK_ID = 'start_local_egress'; + +/** + * Points the host's system proxy at the EAS CLI egress client before the device + * boots, so the Simulator reads it at boot. The runner releases the resources in + * its teardown. macOS only. + */ +export function createStartLocalEgressTask(): SessionTask { + return { + id: START_LOCAL_EGRESS_TASK_ID, + displayName: 'Start local egress', + onFailure: 'fail-session', + run: async ({ runtime, logger, signal }) => { + await startLocalEgressAsync({ + ngrokTunnelDomain: runtime.session.ngrokTunnelDomain, + ngrokAuthtoken: getNgrokAuthtokenOrThrow(runtime.env), + env: runtime.env, + logger, + signal, + }); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/hold.ts b/packages/build-tools/src/deviceRunSession/tasks/hold.ts new file mode 100644 index 0000000000..18ad6eae1d --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/hold.ts @@ -0,0 +1,36 @@ +import { waitForDeviceRunSessionStoppedAsync } from '../../steps/utils/remoteDeviceRunSession'; +import { type SessionTask } from '../runtime'; + +export const HOLD_SESSION_TASK_ID = 'hold_session'; + +/** + * Keeps the job alive until the session is stopped by a client, reaches its + * maximum duration, or idles past its idle timeout. The idle timeout only + * applies to sessions with a controller, which is where activity is observed. + */ +export function createHoldSessionTask({ needs }: { needs: readonly string[] }): SessionTask { + return { + id: HOLD_SESSION_TASK_ID, + displayName: 'Session', + needs, + onFailure: 'fail-session', + run: async ({ runtime, logger, signal }) => { + const { session } = runtime; + const { controller } = runtime.state; + await waitForDeviceRunSessionStoppedAsync({ + ctx: runtime.ctx, + deviceRunSessionId: runtime.deviceRunSessionId, + logger, + maxDurationSeconds: session.maxDurationSeconds, + signal, + idleTimeout: + controller && session.maxIdleTimeMinutes !== undefined + ? { + maxIdleTimeMinutes: session.maxIdleTimeMinutes, + getLastEventObservedAt: controller.getLastEventObservedAt, + } + : undefined, + }); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/network.ts b/packages/build-tools/src/deviceRunSession/tasks/network.ts new file mode 100644 index 0000000000..8e35857d0f --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/network.ts @@ -0,0 +1,36 @@ +import { + fetchWebPreviewTurnArgsAsync, + selectXcodeDeveloperDirectoryAsync, +} from '../../steps/utils/remoteDeviceRunSession'; +import { type SessionTask } from '../runtime'; + +export const FETCH_TURN_CREDENTIALS_TASK_ID = 'fetch_turn_credentials'; +export const SELECT_XCODE_TASK_ID = 'select_xcode'; + +/** Fetches Cloudflare TURN credentials for the web preview. Best effort: the preview falls back to P2P/STUN. */ +export function createFetchTurnCredentialsTask(): SessionTask { + return { + id: FETCH_TURN_CREDENTIALS_TASK_ID, + displayName: 'Fetch TURN credentials', + onFailure: 'warn', + run: async ({ runtime, logger }) => { + runtime.state.turnArgs = await fetchWebPreviewTurnArgsAsync(runtime.ctx, { + env: runtime.env, + logger, + deviceRunSessionId: runtime.deviceRunSessionId, + }); + }, + }; +} + +/** Selects the Xcode every session tool uses, before the device boots under it. macOS only. */ +export function createSelectXcodeTask(): SessionTask { + return { + id: SELECT_XCODE_TASK_ID, + displayName: 'Select Xcode', + onFailure: 'fail-session', + run: async ({ runtime, logger }) => { + await selectXcodeDeveloperDirectoryAsync({ env: runtime.env, logger }); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/pollers.ts b/packages/build-tools/src/deviceRunSession/tasks/pollers.ts new file mode 100644 index 0000000000..10bf6c9997 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/pollers.ts @@ -0,0 +1,42 @@ +import { collectAndUploadServeSimMetricsAsync } from '../../steps/functions/collectServeSimMetrics'; +import { uploadIosSimulatorRecordingsAsync } from '../../steps/functions/uploadDeviceRunSessionScreenRecordings'; +import { IosSimulatorRecordingUtils } from '../../steps/utils/IosSimulatorRecordingUtils'; +import { ServeSimMetricsRecorder } from '../../steps/utils/serveSimMetricsRecorder'; +import { type SessionTask } from '../runtime'; + +export const START_POLLERS_TASK_ID = 'start_pollers'; + +/** + * Starts the screen recording and serve-sim metrics pollers. Both poll for booted + * devices, so they can start before the boot. Their results are uploaded in the + * teardown. macOS only. + */ +export function createStartPollersTask({ needs }: { needs: readonly string[] }): SessionTask { + return { + id: START_POLLERS_TASK_ID, + displayName: 'Start screen recording and metrics collection', + needs, + onFailure: 'warn', + run: async ({ runtime, logger }) => { + const { deviceRunSessionId, ctx } = runtime; + + await IosSimulatorRecordingUtils.startAsync({ env: runtime.env, logger }); + runtime.teardown.push('screen recordings', async teardownLogger => { + const recordings = await IosSimulatorRecordingUtils.finishAsync({ logger: teardownLogger }); + await uploadIosSimulatorRecordingsAsync(ctx, { + deviceRunSessionId, + recordings, + logger: teardownLogger, + }); + }); + + await ServeSimMetricsRecorder.startAsync({ logger }); + runtime.teardown.push('serve-sim metrics', async teardownLogger => { + await collectAndUploadServeSimMetricsAsync(ctx, { + deviceRunSessionId, + logger: teardownLogger, + }); + }); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/preview.ts b/packages/build-tools/src/deviceRunSession/tasks/preview.ts new file mode 100644 index 0000000000..69a4a1192c --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/preview.ts @@ -0,0 +1,52 @@ +import { DeviceRunSession } from '@expo/eas-build-job'; + +import { startDeviceWebPreviewWithTunnelAsync } from '../../steps/utils/remoteDeviceRunSession'; +import { type SessionTask } from '../runtime'; + +export const START_PREVIEW_TASK_ID = 'start_preview'; + +const PREVIEW_STARTUP_TIMEOUT_MS = 60_000; +// Appium sessions on Android share the emulator with expo-device-hub, which takes longer to attach. +const APPIUM_PREVIEW_STARTUP_TIMEOUT_MS = 120_000; + +/** Starts serve-sim (macOS) or expo-device-hub (Linux) against the booted device and tunnels it. */ +export function createStartPreviewTask({ + needs, + after, +}: { + needs: readonly string[]; + after: readonly string[]; +}): SessionTask { + return { + id: START_PREVIEW_TASK_ID, + displayName: 'Start web preview', + needs, + after, + onFailure: 'fail-session', + run: async ({ runtime, logger }) => { + const { session } = runtime; + const preview = await startDeviceWebPreviewWithTunnelAsync(runtime.ctx, { + runtimePlatform: runtime.runtimePlatform, + baseDomain: session.ngrokTunnelDomain, + env: runtime.env, + logger, + timeoutMs: + session.controller === DeviceRunSession.Controller.APPIUM + ? APPIUM_PREVIEW_STARTUP_TIMEOUT_MS + : PREVIEW_STARTUP_TIMEOUT_MS, + // Only web-preview-only sessions pin the preview version; other controllers pin their own tool. + packageVersion: + session.controller === DeviceRunSession.Controller.WEB_PREVIEW_ONLY + ? session.packageVersion + : undefined, + deviceRunSessionId: runtime.deviceRunSessionId, + turnArgs: runtime.state.turnArgs, + // No launch options: the launch_application task launches the app with simctl or adb, + // so the preview can start while the application installs. + }); + runtime.state.preview = preview; + runtime.teardown.push('web preview', () => preview.stopAsync()); + logger.info(`Web preview URL: ${preview.previewPageUrl} (server: ${preview.apiUrl}).`); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/publish.ts b/packages/build-tools/src/deviceRunSession/tasks/publish.ts new file mode 100644 index 0000000000..7a52dbf6ef --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/publish.ts @@ -0,0 +1,61 @@ +import nullthrows from 'nullthrows'; + +import { Datadog } from '../../datadog'; +import { uploadRemoteSessionConfigWithLocalEgressAsync } from '../../steps/utils/localEgressSession'; +import { type SessionTask } from '../runtime'; + +export const PUBLISH_REMOTE_CONFIG_TASK_ID = 'publish_remote_config'; + +/** + * Reports the remote session to the API server, which marks the session as in + * progress. Runs once the preview and controller are up and, when an application + * was requested, once its launch has finished with any outcome, so today's + * "ready" meaning is unchanged: the app is launched when clients get access. + */ +export function createPublishRemoteConfigTask({ + needs, + after, +}: { + needs: readonly string[]; + after: readonly string[]; +}): SessionTask { + return { + id: PUBLISH_REMOTE_CONFIG_TASK_ID, + displayName: 'Publish remote session', + needs, + after, + onFailure: 'fail-session', + run: async ({ runtime, logger, signal }) => { + const preview = nullthrows(runtime.state.preview, 'The web preview is not running.'); + const { controller } = runtime.state; + const remoteConfig = controller + ? { + ...controller.remoteConfig, + webPreviewUrl: preview.previewPageUrl, + previewApiUrl: preview.apiUrl, + ...(preview.previewToken ? { webPreviewToken: preview.previewToken } : {}), + } + : { + // The API server reads previewUrl and previewToken for web-preview-only sessions. + previewUrl: preview.previewPageUrl, + previewApiUrl: preview.apiUrl, + ...(preview.previewToken ? { previewToken: preview.previewToken } : {}), + }; + + await uploadRemoteSessionConfigWithLocalEgressAsync({ + env: runtime.env, + signal, + ctx: runtime.ctx, + deviceRunSessionId: runtime.deviceRunSessionId, + remoteConfig, + logger, + }); + runtime.state.remoteConfigPublishedAt = new Date(); + Datadog.distribution( + 'device_run_session.time_to_remote_config_ms', + Date.now() - runtime.startedAt, + runtime.metricTags() + ); + }, + }; +} diff --git a/packages/build-tools/src/deviceRunSession/tasks/tooling.ts b/packages/build-tools/src/deviceRunSession/tasks/tooling.ts new file mode 100644 index 0000000000..466263e82a --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/tasks/tooling.ts @@ -0,0 +1,199 @@ +import { DeviceRunSession } from '@expo/eas-build-job'; +import { type bunyan } from '@expo/logger'; +import { BuildRuntimePlatform, type BuildStepEnv } from '@expo/steps'; +import spawn from '@expo/turtle-spawn'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createAgentDevicePackageSpec } from '../../steps/functions/startAgentDeviceRemoteSession'; +import { + installAppiumAsync, + resolveAppium3VersionSpec, + resolveAppiumDriverName, +} from '../../steps/functions/startAppiumRemoteSession'; +import { ARGENT_PACKAGE_NAME } from '../../steps/functions/startArgentRemoteSession'; +import { + PackageManager, + resolveConfiguredPackageManager, + resolvePackageAdd, +} from '../../utils/packageManager'; +import { type SessionRuntime, type SessionTask, type SessionTaskContext } from '../runtime'; + +export const PREFETCH_TOOLING_TASK_ID = 'prefetch_tooling'; + +const SERVE_SIM_PACKAGE_NAME = '@expo/serve-sim'; +const EXPO_DEVICE_HUB_PACKAGE_NAME = 'expo-device-hub'; + +/** + * Resolves and caches the controller and preview packages while the device + * boots, so the tasks that start them find the packages installed. Every part is + * best effort: a failure here only means the start task fetches the package + * itself, as it does today. + * + * Each package is warmed with the package manager its start task resolves + * (`EAS_OVERRIDE_PACKAGE_MANAGER`, then `EAS_FALLBACK_PACKAGE_MANAGER`, then the + * tool's default), so the prefetch fills the cache that the start reads. + */ +export function createPrefetchToolingTask(): SessionTask { + return { + id: PREFETCH_TOOLING_TASK_ID, + displayName: 'Prefetch session tooling', + onFailure: 'warn', + run: async context => { + const jobs = planPrefetches(context.runtime).map(async ({ description, run }) => { + context.logger.info(`Prefetching ${description}.`); + try { + await run(context); + } catch (err) { + throw new Error(`Could not prefetch ${description}: ${errorMessage(err)}`, { + cause: err, + }); + } + }); + const failures = (await Promise.allSettled(jobs)).flatMap(result => + result.status === 'rejected' ? [result.reason as Error] : [] + ); + if (failures.length > 0) { + throw new Error(failures.map(failure => failure.message).join('\n')); + } + }, + }; +} + +type Prefetch = { + description: string; + run: (context: SessionTaskContext) => Promise; +}; + +function planPrefetches(runtime: SessionRuntime): Prefetch[] { + const { controller, packageVersion } = runtime.session; + const prefetches: Prefetch[] = []; + + // The web preview is served by serve-sim on macOS and expo-device-hub on Linux. + // Only web-preview-only sessions pin its version; other controllers preview with latest. + const previewVersion = + controller === DeviceRunSession.Controller.WEB_PREVIEW_ONLY ? packageVersion : undefined; + const previewPackageName = + runtime.runtimePlatform === BuildRuntimePlatform.DARWIN + ? SERVE_SIM_PACKAGE_NAME + : EXPO_DEVICE_HUB_PACKAGE_NAME; + const previewPackageSpec = `${previewPackageName}@${previewVersion ?? 'latest'}`; + prefetches.push({ + description: previewPackageSpec, + run: async ({ runtime, logger }) => { + // Matches startWebPreviewWithTunnelAsync, which runs the preview through this manager. + await warmPackageExecCacheAsync({ + packageManager: resolveConfiguredPackageManager(runtime.env, PackageManager.NPM), + packageSpec: previewPackageSpec, + env: runtime.env, + logger, + }); + }, + }); + + switch (controller) { + case DeviceRunSession.Controller.AGENT_DEVICE: { + const spec = createAgentDevicePackageSpec(packageVersion); + prefetches.push({ + description: spec, + run: async ({ runtime, logger }) => { + // The controller start installs the package into a fresh directory with this + // manager; installing it once here fills the manager's package cache. + await warmPackageAddCacheAsync({ + packageManager: resolveConfiguredPackageManager(runtime.env, PackageManager.BUN), + packageSpec: spec, + env: runtime.env, + logger, + }); + }, + }); + break; + } + case DeviceRunSession.Controller.ARGENT: { + const spec = `${ARGENT_PACKAGE_NAME}@${packageVersion ?? 'latest'}`; + prefetches.push({ + description: spec, + run: async ({ runtime, logger }) => { + // Matches startArgentControllerAsync, which runs Argent through this manager. + await warmPackageExecCacheAsync({ + packageManager: resolveConfiguredPackageManager(runtime.env, PackageManager.BUN), + packageSpec: spec, + env: runtime.env, + logger, + }); + }, + }); + break; + } + case DeviceRunSession.Controller.APPIUM: { + const versionSpec = resolveAppium3VersionSpec(packageVersion); + prefetches.push({ + description: `appium@${versionSpec}`, + run: async ({ runtime, logger }) => { + runtime.state.appiumInstallation = await installAppiumAsync({ + versionSpec, + driverName: resolveAppiumDriverName(runtime.runtimePlatform), + env: runtime.env, + logger, + }); + }, + }); + break; + } + case DeviceRunSession.Controller.WEB_PREVIEW_ONLY: + break; + } + + return prefetches; +} + +type WarmCacheOptions = { + packageManager: PackageManager; + packageSpec: string; + env: BuildStepEnv; + logger: bunyan; +}; + +/** Fills the cache that `resolvePackageExec(packageManager, [packageSpec, ...])` reads. */ +async function warmPackageExecCacheAsync(options: WarmCacheOptions): Promise { + const { packageManager, packageSpec, env, logger } = options; + switch (packageManager) { + case PackageManager.NPM: + case PackageManager.YARN: + // Both run through `npx --yes`. `npx --package --call true` installs the + // package into the npx cache without running its binary, and the later + // `npx --yes ` resolves from that cache. + await spawn('npx', ['--yes', '--package', packageSpec, '--call', 'true'], { env, logger }); + return; + case PackageManager.BUN: + case PackageManager.PNPM: + // `bun x` and `pnpm dlx` fetch through the manager's global package cache. + await warmPackageAddCacheAsync(options); + return; + } +} + +/** Installs the package into a throwaway directory, which fills the manager's package cache. */ +async function warmPackageAddCacheAsync({ + packageManager, + packageSpec, + env, + logger, +}: WarmCacheOptions): Promise { + const installDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'eas-session-prefetch-')); + try { + await fs.promises.writeFile( + path.join(installDir, 'package.json'), + `${JSON.stringify({ name: 'eas-session-prefetch', private: true })}\n` + ); + const add = resolvePackageAdd(packageManager, packageSpec); + await spawn(add.command, add.args, { cwd: installDir, env, logger }); + } finally { + await fs.promises.rm(installDir, { recursive: true, force: true }); + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/packages/build-tools/src/deviceRunSession/types.ts b/packages/build-tools/src/deviceRunSession/types.ts new file mode 100644 index 0000000000..739939fbe0 --- /dev/null +++ b/packages/build-tools/src/deviceRunSession/types.ts @@ -0,0 +1,35 @@ +/** + * What happens to the session when a task fails. + * + * - `fail-session`: the whole session fails. Running tasks are aborted and the + * job run errors with this task's error. + * - `degrade-application`: the device stays usable. Tasks that need this one + * are skipped, the failure is logged, and the session continues. + * - `warn`: the failure is logged and everything else proceeds as if the task + * had succeeded. Only for best-effort work such as cache warming. + */ +export type TaskFailurePolicy = 'fail-session' | 'degrade-application' | 'warn'; + +export type TaskOutcome = 'success' | 'failed' | 'skipped'; + +export interface TaskDefinition { + /** Stable snake_case id. Doubles as the log group id on expo.dev. */ + id: string; + /** Human-readable name shown as the log group title. */ + displayName: string; + /** Tasks that must succeed before this one starts. A failed or skipped need skips this task. */ + needs?: readonly string[]; + /** Tasks that must finish, with any outcome, before this one starts. */ + after?: readonly string[]; + onFailure: TaskFailurePolicy; + run: (context: TContext) => Promise; +} + +export interface TaskResult { + id: string; + outcome: TaskOutcome; + durationMs: number; + error?: unknown; + /** Why a task was skipped, for the log. */ + skipReason?: string; +} diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index 382d8151dd..c4beb6dadb 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -31,6 +31,7 @@ export { parseGradleProfile, formatGradleProfileReport } from './android/gradleP export type { GradleProfileTask } from './android/gradleProfile'; export * from './generic'; +export { runDeviceRunSessionJobAsync } from './deviceRunSession'; export { Datadog } from './datadog'; export { Sentry } from './sentry'; diff --git a/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts b/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts index f7c965ba0d..72223cb96c 100644 --- a/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts +++ b/packages/build-tools/src/steps/functions/collectServeSimMetrics.ts @@ -1,3 +1,4 @@ +import { type bunyan } from '@expo/logger'; import { BuildFunction, BuildRuntimePlatform } from '@expo/steps'; import { type CustomBuildContext } from '../../customBuildContext'; @@ -21,15 +22,7 @@ export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContex } try { const deviceRunSessionId = getDeviceRunSessionIdOrThrow(env); - for (const { udid, filePath, metadata } of collected) { - await uploadServeSimMetricsFileAsync(ctx, { - deviceRunSessionId, - udid, - filePath, - metadata, - logger, - }); - } + await uploadCollectedServeSimMetricsAsync(ctx, { deviceRunSessionId, collected, logger }); } catch (err) { const error = err instanceof Error ? err : new Error(String(err)); Sentry.capture('Could not upload serve-sim metrics', error); @@ -38,3 +31,47 @@ export function createCollectServeSimMetricsBuildFunction(ctx: CustomBuildContex }, }); } + +type CollectedServeSimMetrics = Awaited>; + +async function uploadCollectedServeSimMetricsAsync( + ctx: CustomBuildContext, + { + deviceRunSessionId, + collected, + logger, + }: { deviceRunSessionId: string; collected: CollectedServeSimMetrics; logger: bunyan } +): Promise { + for (const { udid, filePath, metadata } of collected) { + await uploadServeSimMetricsFileAsync(ctx, { + deviceRunSessionId, + udid, + filePath, + metadata, + logger, + }); + } +} + +/** + * Stops serve-sim metrics polling and uploads whatever was collected. Upload + * failures are reported and logged; they never fail the session. Shared by the + * `eas/collect_serve_sim_metrics` step and the device run session runner. + */ +export async function collectAndUploadServeSimMetricsAsync( + ctx: CustomBuildContext, + { deviceRunSessionId, logger }: { deviceRunSessionId: string; logger: bunyan } +): Promise { + const collected = await ServeSimMetricsRecorder.finishAsync({ logger }); + if (collected.length === 0) { + logger.info('No serve-sim metrics collected; skipping upload.'); + return; + } + try { + await uploadCollectedServeSimMetricsAsync(ctx, { deviceRunSessionId, collected, logger }); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + Sentry.capture('Could not upload serve-sim metrics', error); + logger.warn({ err: error }, 'Could not upload serve-sim metrics.'); + } +} diff --git a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts index 63c6b4acb6..f4a30ed439 100644 --- a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts @@ -109,29 +109,16 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( await selectXcodeDeveloperDirectoryAsync({ env, logger }); } - logger.info('Launching agent-device daemon.'); - const daemonProcess = await startAgentDeviceDaemonAsync({ packageVersion, env, logger }); - - logger.info(`Waiting for daemon credentials at ${DAEMON_JSON_PATH}.`); - const { port: daemonPort, token: daemonToken } = await waitForDaemonInfoAsync({ - daemonProcess, - }); - logger.info(`Daemon is listening on port ${daemonPort}; loaded auth token.`); - - const agentDeviceTunnel = await startNgrokTunnelAsync({ - port: daemonPort, - subdomainPrefix: 'agent-device', - baseDomain: ngrokTunnelDomain, - authtoken: ngrokAuthtoken, + const controller = await startAgentDeviceControllerAsync(ctx, { + deviceRunSessionId, + packageVersion, + ngrokTunnelDomain, + ngrokAuthtoken, + env, logger, }); - const agentDeviceRemoteSessionUrl = agentDeviceTunnel.url; - logger.info(`Tunnel is ready at ${agentDeviceRemoteSessionUrl}.`); let webPreview: Awaited> | undefined; - let eventCollection: - | Awaited> - | undefined; try { const launchDescription = describeServeSimLaunch(launch); if (launchDescription) { @@ -157,27 +144,13 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( ctx, deviceRunSessionId, remoteConfig: { - agentDeviceRemoteSessionUrl, - agentDeviceRemoteSessionToken: daemonToken, + ...controller.remoteConfig, webPreviewUrl: webPreview.previewPageUrl, previewApiUrl: webPreview.apiUrl, ...(webPreview.previewToken ? { webPreviewToken: webPreview.previewToken } : {}), }, logger, }); - void pollAgentDeviceArtifactsForUploadAsync(ctx, { - deviceRunSessionId, - daemonUrl: `http://127.0.0.1:${daemonPort}`, - daemonToken, - logger, - }); - - eventCollection = await startAgentDeviceEventCollectionAsync({ - ctx, - deviceRunSessionId, - stateDir: AGENT_DEVICE_STATE_DIR, - logger, - }); await waitForDeviceRunSessionStoppedAsync({ ctx, @@ -189,7 +162,7 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( maxIdleTimeMinutes !== undefined && maxIdleTimeMinutes > 0 ? { maxIdleTimeMinutes, - getLastEventObservedAt: eventCollection.getLastEventObservedAt, + getLastEventObservedAt: controller.getLastEventObservedAt, } : undefined, }); @@ -197,20 +170,116 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( if (webPreview) { await webPreview.stopAsync(); } - await agentDeviceTunnel.stopAsync(); - if (eventCollection) { - await stopAgentDeviceEventCollectionSafelyAsync({ - eventCollection, - deviceRunSessionId, - logger, - }); - } - await daemonProcess.stopAsync(); + await controller.stopAsync(); } }), }); } +export type AgentDeviceControllerHandle = { + remoteConfig: { + agentDeviceRemoteSessionUrl: string; + agentDeviceRemoteSessionToken: string; + }; + /** Local port the daemon listens on. */ + daemonPort: number; + /** Arrival time of the newest collected session event, for idle detection. */ + getLastEventObservedAt: () => Date | undefined; + /** Closes the tunnel, stops event collection, and stops the daemon. */ + stopAsync: () => Promise; +}; + +/** + * Starts the agent-device daemon, exposes it through an ngrok tunnel, and starts + * artifact and event collection for the session. Shared by the + * `eas/start_agent_device_remote_session` step and the device run session runner. + */ +export async function startAgentDeviceControllerAsync( + ctx: CustomBuildContext, + { + deviceRunSessionId, + packageVersion, + ngrokTunnelDomain, + ngrokAuthtoken, + env, + logger, + }: { + deviceRunSessionId: string; + packageVersion: string | undefined; + ngrokTunnelDomain: string; + ngrokAuthtoken: string; + env: BuildStepEnv; + logger: bunyan; + } +): Promise { + logger.info('Launching agent-device daemon.'); + const daemonProcess = await startAgentDeviceDaemonAsync({ packageVersion, env, logger }); + + let agentDeviceTunnel: Awaited> | undefined; + let eventCollection: Awaited> | undefined; + try { + logger.info(`Waiting for daemon credentials at ${DAEMON_JSON_PATH}.`); + const { port: daemonPort, token: daemonToken } = await waitForDaemonInfoAsync({ + daemonProcess, + }); + logger.info(`Daemon is listening on port ${daemonPort}; loaded auth token.`); + + agentDeviceTunnel = await startNgrokTunnelAsync({ + port: daemonPort, + subdomainPrefix: 'agent-device', + baseDomain: ngrokTunnelDomain, + authtoken: ngrokAuthtoken, + logger, + }); + const tunnel = agentDeviceTunnel; + logger.info(`Tunnel is ready at ${tunnel.url}.`); + + void pollAgentDeviceArtifactsForUploadAsync(ctx, { + deviceRunSessionId, + daemonUrl: `http://127.0.0.1:${daemonPort}`, + daemonToken, + logger, + }); + + eventCollection = await startAgentDeviceEventCollectionAsync({ + ctx, + deviceRunSessionId, + stateDir: AGENT_DEVICE_STATE_DIR, + logger, + }); + const collection = eventCollection; + + return { + remoteConfig: { + agentDeviceRemoteSessionUrl: tunnel.url, + agentDeviceRemoteSessionToken: daemonToken, + }, + daemonPort, + getLastEventObservedAt: collection.getLastEventObservedAt, + stopAsync: async () => { + await tunnel.stopAsync(); + await stopAgentDeviceEventCollectionSafelyAsync({ + eventCollection: collection, + deviceRunSessionId, + logger, + }); + await daemonProcess.stopAsync(); + }, + }; + } catch (error) { + await agentDeviceTunnel?.stopAsync(); + if (eventCollection) { + await stopAgentDeviceEventCollectionSafelyAsync({ + eventCollection, + deviceRunSessionId, + logger, + }); + } + await daemonProcess.stopAsync(); + throw error; + } +} + export async function startAgentDeviceDaemonAsync({ packageVersion, env, @@ -385,7 +454,7 @@ async function getBunVersionForDiagnosticsAsync(env: BuildStepEnv): Promise { - await spawn('sdkmanager', [systemImagePackage], { - env, - logger, - }); - }, - { - logger, - retryOptions: { - retries: 3, // Retry 3 times - retryIntervalMs: 1_000, - }, - } - ); - - let emulatorPromise = null; - let serialId = null; - await retryAsync( - async attemptCount => { - const timeoutMs = ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS[attemptCount]; - const attempt = attemptCount + 1; - const maxAttempts = ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS.length; - const attemptSuffix = attempt > 1 ? ` (attempt ${attempt}/${maxAttempts})` : ''; - let attemptSerialId = null; - - try { - logger.info(`Creating emulator device${attemptSuffix}.`); - await AndroidEmulatorUtils.createAsync({ - deviceName, - systemImagePackage, - deviceIdentifier: deviceIdentifier ?? null, - lcdWidth: lcdWidth ?? null, - lcdHeight: lcdHeight ?? null, - lcdDensity: lcdDensity ?? null, - env, - logger, - }); - - logger.info(`Starting emulator device${attemptSuffix}.`); - const startResult = await AndroidEmulatorUtils.startAsync({ - deviceName, - env, - logcatDirectory, - }); - attemptSerialId = startResult.serialId; - await AndroidEmulatorUtils.waitForReadyAsync({ - env, - serialId: attemptSerialId, - timeoutMs, - logger, - }); - if (shouldAdjustAnimationScale) { - await AndroidEmulatorUtils.disableWindowAndTransitionAnimationsAsync({ - env, - logger, - serialId: attemptSerialId, - }); - } - logger.info(`${deviceName} is ready.`); - - serialId = attemptSerialId; - emulatorPromise = startResult.emulatorPromise; - } catch (err) { - logger.warn( - { err }, - `${deviceName} failed to start on attempt ${attempt}/${maxAttempts}.` - ); - try { - if (attemptSerialId) { - await AndroidEmulatorUtils.deleteAsync({ - serialId: attemptSerialId, - deviceName, - env, - }); - } else { - await AndroidEmulatorUtils.deleteAsync({ - deviceName, - env, - }); - } - } catch (cleanupErr) { - logger.warn({ err: cleanupErr }, `Failed to clean up ${deviceName}.`); - } - throw err; - } - }, - { - logger, - retryOptions: { - retries: ANDROID_STARTUP_RETRIES_COUNT, - retryIntervalMs: 1_000, - }, - } - ); - - if (!serialId || !emulatorPromise) { - throw new Error(`Failed to start emulator ${deviceName}.`); - } + const { serialId, emulatorPromise } = await startAndroidEmulatorAsync({ + deviceName, + deviceIdentifier: deviceIdentifier ?? null, + systemImagePackage, + lcdWidth: lcdWidth ?? null, + lcdHeight: lcdHeight ?? null, + lcdDensity: lcdDensity ?? null, + adjustAnimationScale: shouldAdjustAnimationScale, + logcatDirectory, + env, + logger, + }); const count = Number(inputs.count.value ?? 1); if (count > 1) { @@ -306,7 +220,140 @@ export function createStartAndroidEmulatorBuildFunction(): BuildFunction { }); } -async function assertAndroidEmulatorHostSupportAsync({ +type AndroidEmulatorStartResult = Awaited>; + +/** + * Installs the system image if needed, then creates, starts, and waits for one + * Android emulator, retrying with growing timeouts. Shared by the + * `eas/start_android_emulator` step and the device run session runner. + */ +export async function startAndroidEmulatorAsync({ + deviceName, + deviceIdentifier, + systemImagePackage, + lcdWidth, + lcdHeight, + lcdDensity, + adjustAnimationScale, + logcatDirectory, + env, + logger, +}: { + deviceName: AndroidVirtualDeviceName; + deviceIdentifier: AndroidDeviceName | null; + systemImagePackage: string; + lcdWidth: number | null; + lcdHeight: number | null; + lcdDensity: number | null; + adjustAnimationScale: boolean; + logcatDirectory: string; + env: BuildStepEnv; + logger: bunyan; +}): Promise<{ + serialId: AndroidEmulatorStartResult['serialId']; + emulatorPromise: AndroidEmulatorStartResult['emulatorPromise']; +}> { + logger.info('Making sure system image is installed'); + await retryAsync( + async () => { + await spawn('sdkmanager', [systemImagePackage], { + env, + logger, + }); + }, + { + logger, + retryOptions: { + retries: 3, // Retry 3 times + retryIntervalMs: 1_000, + }, + } + ); + + let emulatorPromise: AndroidEmulatorStartResult['emulatorPromise'] | null = null; + let serialId: AndroidEmulatorStartResult['serialId'] | null = null; + await retryAsync( + async attemptCount => { + const timeoutMs = ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS[attemptCount]; + const attempt = attemptCount + 1; + const maxAttempts = ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS.length; + const attemptSuffix = attempt > 1 ? ` (attempt ${attempt}/${maxAttempts})` : ''; + let attemptSerialId = null; + + try { + logger.info(`Creating emulator device${attemptSuffix}.`); + await AndroidEmulatorUtils.createAsync({ + deviceName, + systemImagePackage, + deviceIdentifier: deviceIdentifier ?? null, + lcdWidth: lcdWidth ?? null, + lcdHeight: lcdHeight ?? null, + lcdDensity: lcdDensity ?? null, + env, + logger, + }); + + logger.info(`Starting emulator device${attemptSuffix}.`); + const startResult = await AndroidEmulatorUtils.startAsync({ + deviceName, + env, + logcatDirectory, + }); + attemptSerialId = startResult.serialId; + await AndroidEmulatorUtils.waitForReadyAsync({ + env, + serialId: attemptSerialId, + timeoutMs, + logger, + }); + if (adjustAnimationScale) { + await AndroidEmulatorUtils.disableWindowAndTransitionAnimationsAsync({ + env, + logger, + serialId: attemptSerialId, + }); + } + logger.info(`${deviceName} is ready.`); + + serialId = attemptSerialId; + emulatorPromise = startResult.emulatorPromise; + } catch (err) { + logger.warn({ err }, `${deviceName} failed to start on attempt ${attempt}/${maxAttempts}.`); + try { + if (attemptSerialId) { + await AndroidEmulatorUtils.deleteAsync({ + serialId: attemptSerialId, + deviceName, + env, + }); + } else { + await AndroidEmulatorUtils.deleteAsync({ + deviceName, + env, + }); + } + } catch (cleanupErr) { + logger.warn({ err: cleanupErr }, `Failed to clean up ${deviceName}.`); + } + throw err; + } + }, + { + logger, + retryOptions: { + retries: ANDROID_STARTUP_RETRIES_COUNT, + retryIntervalMs: 1_000, + }, + } + ); + + if (!serialId || !emulatorPromise) { + throw new Error(`Failed to start emulator ${deviceName}.`); + } + return { serialId, emulatorPromise }; +} + +export async function assertAndroidEmulatorHostSupportAsync({ env, }: { env: NodeJS.ProcessEnv; diff --git a/packages/build-tools/src/steps/functions/startAppiumRemoteSession.ts b/packages/build-tools/src/steps/functions/startAppiumRemoteSession.ts index 91a2dc0463..f0ac4b4883 100644 --- a/packages/build-tools/src/steps/functions/startAppiumRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startAppiumRemoteSession.ts @@ -89,65 +89,19 @@ export function createStartAppiumRemoteSessionBuildFunction( }, { runtimePlatform } ); - const versionSpec = resolveAppium3VersionSpec(packageVersion); - logger.info( - `Starting Appium remote session (version: ${versionSpec}, runtime: ${runtimePlatform}).` - ); - const device = await resolveAppiumDeviceAsync({ runtimePlatform, env, logger }); - const { appiumHome, appiumBinPath, appiumEnv } = await installAppiumAsync({ - versionSpec, - driverName: device.driverName, + const controller = await startAppiumControllerAsync(ctx, { + deviceRunSessionId, + packageVersion, + runtimePlatform, + ngrokTunnelDomain, + ngrokAuthtoken, env, logger, }); - const appiumProcess = spawnDetached({ - command: appiumBinPath, - args: [ - '--address', - APPIUM_HOST, - '--port', - String(APPIUM_PORT), - '--base-path', - '/', - '--log-level', - 'error', - // Appium 3 gates session listing (GET /appium/sessions) behind the - // session_discovery insecure feature. We rely on it to poll for - // Appium Event Timings, so enable it for all drivers. - '--allow-insecure', - '*:session_discovery', - '--default-capabilities', - JSON.stringify({ 'appium:eventTimings': true }), - ], - env: appiumEnv, - }); - try { - await waitForAppiumReadyAsync({ appiumProcess, logger }); - } catch (error) { - await appiumProcess.stopAsync(); - await fs.promises.rm(appiumHome, { recursive: true, force: true }); - throw error; - } - - const eventCollection = await startAppiumEventCollectionAsync({ - ctx, - deviceRunSessionId, - appiumUrl: `http://${APPIUM_HOST}:${APPIUM_PORT}/`, - logger, - }); - let appiumTunnel: Awaited> | undefined; let webPreview: Awaited> | undefined; try { - appiumTunnel = await startNgrokTunnelAsync({ - port: APPIUM_PORT, - subdomainPrefix: 'appium', - baseDomain: ngrokTunnelDomain, - authtoken: ngrokAuthtoken, - logger, - }); - // expo-device-hub has no serial-selection flag. Device run session workflows must expose // a single booted Android emulator so the Hub and Appium resolve the same device. const launchDescription = describeServeSimLaunch(launch); @@ -171,12 +125,7 @@ export function createStartAppiumRemoteSessionBuildFunction( ctx, deviceRunSessionId, remoteConfig: { - appiumUrl: appiumTunnel.url, - capabilities: { - platformName: device.platformName, - 'appium:automationName': device.automationName, - 'appium:udid': device.udid, - }, + ...controller.remoteConfig, webPreviewUrl: webPreview.previewPageUrl, previewApiUrl: webPreview.apiUrl, ...(webPreview.previewToken ? { webPreviewToken: webPreview.previewToken } : {}), @@ -193,7 +142,7 @@ export function createStartAppiumRemoteSessionBuildFunction( maxIdleTimeMinutes !== undefined && maxIdleTimeMinutes > 0 ? { maxIdleTimeMinutes, - getLastEventObservedAt: eventCollection.getLastEventObservedAt, + getLastEventObservedAt: controller.getLastEventObservedAt, } : undefined, }); @@ -201,17 +150,158 @@ export function createStartAppiumRemoteSessionBuildFunction( if (webPreview) { await webPreview.stopAsync(); } - if (appiumTunnel) { - await appiumTunnel.stopAsync(); - } - await eventCollection.stopAsync(); - await appiumProcess.stopAsync(); - await fs.promises.rm(appiumHome, { recursive: true, force: true }); + await controller.stopAsync(); } }), }); } +export type AppiumInstallation = Awaited>; + +export type AppiumControllerHandle = { + remoteConfig: { + appiumUrl: string; + capabilities: { + platformName: AppiumDevice['platformName']; + 'appium:automationName': AppiumDevice['automationName']; + 'appium:udid': string; + }; + }; + /** Arrival time of the newest collected Appium command, for idle detection. */ + getLastEventObservedAt: () => Date | undefined; + /** Closes the tunnel, stops event collection, stops the server, and removes its home. */ + stopAsync: () => Promise; +}; + +/** Appium driver for the runtime platform; the device itself is resolved after it boots. */ +export function resolveAppiumDriverName( + runtimePlatform: BuildRuntimePlatform +): AppiumDevice['driverName'] { + return runtimePlatform === BuildRuntimePlatform.DARWIN ? 'xcuitest' : 'uiautomator2'; +} + +/** + * Installs Appium (unless a prepared installation is passed), starts its server + * against the booted device, exposes it through an ngrok tunnel, and starts event + * collection. Shared by the `eas/start_appium_remote_session` step and the device + * run session runner. + */ +export async function startAppiumControllerAsync( + ctx: CustomBuildContext, + { + deviceRunSessionId, + packageVersion, + runtimePlatform, + ngrokTunnelDomain, + ngrokAuthtoken, + env, + logger, + installation, + }: { + deviceRunSessionId: string; + packageVersion: string | undefined; + runtimePlatform: BuildRuntimePlatform; + ngrokTunnelDomain: string; + ngrokAuthtoken: string; + env: BuildStepEnv; + logger: bunyan; + /** An installation prepared earlier, for example while the device was booting. */ + installation?: AppiumInstallation; + } +): Promise { + const versionSpec = resolveAppium3VersionSpec(packageVersion); + + logger.info( + `Starting Appium remote session (version: ${versionSpec}, runtime: ${runtimePlatform}).` + ); + const device = await resolveAppiumDeviceAsync({ runtimePlatform, env, logger }); + const { appiumHome, appiumBinPath, appiumEnv } = + installation ?? + (await installAppiumAsync({ + versionSpec, + driverName: device.driverName, + env, + logger, + })); + const removeAppiumHomeAsync = async (): Promise => { + await fs.promises.rm(appiumHome, { recursive: true, force: true }); + }; + + const appiumProcess = spawnDetached({ + command: appiumBinPath, + args: [ + '--address', + APPIUM_HOST, + '--port', + String(APPIUM_PORT), + '--base-path', + '/', + '--log-level', + 'error', + // Appium 3 gates session listing (GET /appium/sessions) behind the + // session_discovery insecure feature. We rely on it to poll for + // Appium Event Timings, so enable it for all drivers. + '--allow-insecure', + '*:session_discovery', + '--default-capabilities', + JSON.stringify({ 'appium:eventTimings': true }), + ], + env: appiumEnv, + }); + try { + await waitForAppiumReadyAsync({ appiumProcess, logger }); + } catch (error) { + await appiumProcess.stopAsync(); + await removeAppiumHomeAsync(); + throw error; + } + + let eventCollection: Awaited> | undefined; + let appiumTunnel: Awaited> | undefined; + try { + eventCollection = await startAppiumEventCollectionAsync({ + ctx, + deviceRunSessionId, + appiumUrl: `http://${APPIUM_HOST}:${APPIUM_PORT}/`, + logger, + }); + const collection = eventCollection; + + appiumTunnel = await startNgrokTunnelAsync({ + port: APPIUM_PORT, + subdomainPrefix: 'appium', + baseDomain: ngrokTunnelDomain, + authtoken: ngrokAuthtoken, + logger, + }); + const tunnel = appiumTunnel; + + return { + remoteConfig: { + appiumUrl: tunnel.url, + capabilities: { + platformName: device.platformName, + 'appium:automationName': device.automationName, + 'appium:udid': device.udid, + }, + }, + getLastEventObservedAt: collection.getLastEventObservedAt, + stopAsync: async () => { + await tunnel.stopAsync(); + await collection.stopAsync(); + await appiumProcess.stopAsync(); + await removeAppiumHomeAsync(); + }, + }; + } catch (error) { + await appiumTunnel?.stopAsync(); + await eventCollection?.stopAsync(); + await appiumProcess.stopAsync(); + await removeAppiumHomeAsync(); + throw error; + } +} + export function resolveAppium3VersionSpec(packageVersion: string | undefined): string { const versionSpec = packageVersion ?? DEFAULT_APPIUM_VERSION; const range = semver.validRange(versionSpec); diff --git a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts index 245103d6d3..19cef022e8 100644 --- a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts @@ -3,6 +3,7 @@ import { type bunyan } from '@expo/logger'; import { BuildFunction, BuildRuntimePlatform, + type BuildStepEnv, BuildStepInput, BuildStepInputValueTypeName, } from '@expo/steps'; @@ -43,7 +44,7 @@ import { waitForDeviceRunSessionStoppedAsync, } from '../utils/remoteDeviceRunSession'; -const ARGENT_PACKAGE_NAME = '@swmansion/argent'; +export const ARGENT_PACKAGE_NAME = '@swmansion/argent'; // 0.16.0 is the first version that exposes the tool-server event log flag; keeping the floor // here lets us enable it (and the artifacts list endpoint) unconditionally below. export const MIN_ARGENT_REMOTE_SESSION_VERSION = '0.16.0'; @@ -104,8 +105,6 @@ export function createStartArgentRemoteSessionBuildFunction( // A missing or non-positive value disables the idle timeout (opt-in feature). const maxIdleTimeMinutes = inputs.max_idle_time_minutes.value as number | undefined; const maxDurationSeconds = inputs.max_duration_seconds?.value as number | undefined; - warnIfArgentPackageVersionCannotBeVerified({ packageVersion, logger }); - const versionSpec = packageVersion ?? 'latest'; const { runtimePlatform } = global; const launch = parseServeSimLaunchInputs( { @@ -116,119 +115,26 @@ export function createStartArgentRemoteSessionBuildFunction( { runtimePlatform } ); logger.info( - `Starting argent remote session (version: ${versionSpec}, runtime: ${runtimePlatform}).` + `Starting argent remote session (version: ${packageVersion ?? 'latest'}, runtime: ${runtimePlatform}).` ); if (runtimePlatform === BuildRuntimePlatform.DARWIN) { await selectXcodeDeveloperDirectoryAsync({ env, logger }); } - // Start the potentially slow installation while Argent is being prepared. - // On Linux expo-device-hub calls this again and awaits the same in-flight - // setup before launching. On macOS this remains non-blocking, so only a - // recording started in the first moments may miss ffmpeg. - // Never rejects, so `void` is safe. - void ensureFfmpegInstalledOnceAsync({ runtimePlatform, env, logger }); - - const packageManager = resolveConfiguredPackageManager(env, PackageManager.BUN); - const argentExec = (args: string[]): { command: string; args: string[] } => - resolvePackageExec(packageManager, args); - - logger.info('Enabling the Argent artifacts list endpoint flag.'); - const enableArtifacts = argentExec([ - `${ARGENT_PACKAGE_NAME}@${versionSpec}`, - 'enable', - ARGENT_ARTIFACTS_LIST_ENDPOINT_FLAG, - ]); - await spawn(enableArtifacts.command, enableArtifacts.args, { env, logger }); - - logger.info('Enabling the Argent tool-server event log flag.'); - const enableEventLog = argentExec([ - `${ARGENT_PACKAGE_NAME}@${versionSpec}`, - 'enable', - ARGENT_EVENT_LOG_FLAG, - ]); - await spawn(enableEventLog.command, enableEventLog.args, { env, logger }); - - const startServer = argentExec([ - `${ARGENT_PACKAGE_NAME}@${versionSpec}`, - 'server', - 'start', - '--port', - '0', - '--idle-timeout', - '0', - '--force', - ]); - logger.info( - `Launching ${ARGENT_PACKAGE_NAME}@${versionSpec} tool-server via ${startServer.command}.` - ); - // Keep Argent itself in foreground mode under the detached process. This preserves - // the npx/bun -> Argent CLI -> tool-server ancestry used to identify the matching state file. - const argentServer = spawnDetached({ - command: startServer.command, - args: startServer.args, - env: { ...env, ARGENT_EVENT_LOG: ARGENT_EVENT_LOG_PATH }, - }); - if (argentServer.pid === undefined) { - throw new SystemError( - 'Failed to start Argent: could not determine the PID of the launched process.' - ); - } - - logger.info(`Waiting for argent tool-server state in ${ARGENT_STATE_DIR}.`); - let toolServerPort: number; - let toolServerToken: string | undefined; - try { - const toolServerState = await waitForArgentToolServerStateAsync({ - stateDir: ARGENT_STATE_DIR, - ancestorPid: argentServer.pid, - timeoutMs: STARTUP_TIMEOUT_MS, - }); - toolServerPort = toolServerState.port; - toolServerToken = toolServerState.token; - } catch (err) { - const output = argentServer.getOutput(); - throw new SystemError( - `${ - err instanceof Error ? err.message : `Timed out waiting for argent tool-server state.` - }${output ? `\nArgent tool-server output:\n${output}` : ''}` - ); - } - logger.info(`Argent tool-server is listening on port ${toolServerPort}.`); - const artifactPollAbortController = new AbortController(); - const artifactPollSignal = signal - ? AbortSignal.any([signal, artifactPollAbortController.signal]) - : artifactPollAbortController.signal; - const artifactPollingPromise = pollArgentArtifactsForUploadAsync(ctx, { + const controller = await startArgentControllerAsync(ctx, { deviceRunSessionId, - toolsUrl: `http://127.0.0.1:${toolServerPort}`, - toolsAuthToken: toolServerToken, + packageVersion, + runtimePlatform, + ngrokTunnelDomain, + ngrokAuthtoken, + env, logger, - signal: artifactPollSignal, + signal, }); - const eventCollection = await startArgentEventCollectionAsync({ - ctx, - deviceRunSessionId, - eventLogPath: ARGENT_EVENT_LOG_PATH, - logger, - }); - - let toolsTunnel: Awaited> | undefined; let webPreview: Awaited> | undefined; try { - toolsTunnel = await startNgrokTunnelAsync({ - port: toolServerPort, - subdomainPrefix: 'argent', - baseDomain: ngrokTunnelDomain, - authtoken: ngrokAuthtoken, - rewriteHostHeader: true, - logger, - }); - const publicToolsUrl = toolsTunnel.url; - logger.info(`Tunnel is ready at ${publicToolsUrl}.`); - const launchDescription = describeServeSimLaunch(launch); if (launchDescription) { logger.info(launchDescription); @@ -253,8 +159,7 @@ export function createStartArgentRemoteSessionBuildFunction( ctx, deviceRunSessionId, remoteConfig: { - toolsUrl: publicToolsUrl, - ...(toolServerToken ? { toolsAuthToken: toolServerToken } : {}), + ...controller.remoteConfig, webPreviewUrl: webPreview.previewPageUrl, previewApiUrl: webPreview.apiUrl, ...(webPreview.previewToken ? { webPreviewToken: webPreview.previewToken } : {}), @@ -272,7 +177,7 @@ export function createStartArgentRemoteSessionBuildFunction( maxIdleTimeMinutes !== undefined && maxIdleTimeMinutes > 0 ? { maxIdleTimeMinutes, - getLastEventObservedAt: eventCollection.getLastEventObservedAt, + getLastEventObservedAt: controller.getLastEventObservedAt, } : undefined, }); @@ -280,24 +185,200 @@ export function createStartArgentRemoteSessionBuildFunction( if (webPreview) { await webPreview.stopAsync(); } - if (toolsTunnel) { - await toolsTunnel.stopAsync(); - } - await stopArgentEventCollectionSafelyAsync({ eventCollection, deviceRunSessionId, logger }); - artifactPollAbortController.abort(); - try { - await artifactPollingPromise; - } catch (err) { - const error = err instanceof Error ? err : new Error(String(err)); - Sentry.capture('Could not finish Argent remote session artifact polling', error); - logger.warn({ err: error }, 'Could not finish Argent remote session artifact polling.'); - } - await argentServer.stopAsync(); + await controller.stopAsync(); } }), }); } +export type ArgentControllerHandle = { + remoteConfig: { + toolsUrl: string; + toolsAuthToken?: string; + }; + /** Arrival time of the newest collected session event, for idle detection. */ + getLastEventObservedAt: () => Date | undefined; + /** Closes the tunnel, stops event and artifact collection, and stops the tool server. */ + stopAsync: () => Promise; +}; + +/** + * Starts the Argent tool server, exposes it through an ngrok tunnel, and starts + * artifact and event collection for the session. Shared by the + * `eas/start_argent_remote_session` step and the device run session runner. + */ +export async function startArgentControllerAsync( + ctx: CustomBuildContext, + { + deviceRunSessionId, + packageVersion, + runtimePlatform, + ngrokTunnelDomain, + ngrokAuthtoken, + env, + logger, + signal, + }: { + deviceRunSessionId: string; + packageVersion: string | undefined; + runtimePlatform: BuildRuntimePlatform; + ngrokTunnelDomain: string; + ngrokAuthtoken: string; + env: BuildStepEnv; + logger: bunyan; + signal?: AbortSignal; + } +): Promise { + warnIfArgentPackageVersionCannotBeVerified({ packageVersion, logger }); + const versionSpec = packageVersion ?? 'latest'; + + // Start the potentially slow installation while Argent is being prepared. + // On Linux expo-device-hub calls this again and awaits the same in-flight + // setup before launching. On macOS this remains non-blocking, so only a + // recording started in the first moments may miss ffmpeg. + // Never rejects, so `void` is safe. + void ensureFfmpegInstalledOnceAsync({ runtimePlatform, env, logger }); + + const packageManager = resolveConfiguredPackageManager(env, PackageManager.BUN); + const argentExec = (args: string[]): { command: string; args: string[] } => + resolvePackageExec(packageManager, args); + + logger.info('Enabling the Argent artifacts list endpoint flag.'); + const enableArtifacts = argentExec([ + `${ARGENT_PACKAGE_NAME}@${versionSpec}`, + 'enable', + ARGENT_ARTIFACTS_LIST_ENDPOINT_FLAG, + ]); + await spawn(enableArtifacts.command, enableArtifacts.args, { env, logger }); + + logger.info('Enabling the Argent tool-server event log flag.'); + const enableEventLog = argentExec([ + `${ARGENT_PACKAGE_NAME}@${versionSpec}`, + 'enable', + ARGENT_EVENT_LOG_FLAG, + ]); + await spawn(enableEventLog.command, enableEventLog.args, { env, logger }); + + const startServer = argentExec([ + `${ARGENT_PACKAGE_NAME}@${versionSpec}`, + 'server', + 'start', + '--port', + '0', + '--idle-timeout', + '0', + '--force', + ]); + logger.info( + `Launching ${ARGENT_PACKAGE_NAME}@${versionSpec} tool-server via ${startServer.command}.` + ); + // Keep Argent itself in foreground mode under the detached process. This preserves + // the npx/bun -> Argent CLI -> tool-server ancestry used to identify the matching state file. + const argentServer = spawnDetached({ + command: startServer.command, + args: startServer.args, + env: { ...env, ARGENT_EVENT_LOG: ARGENT_EVENT_LOG_PATH }, + }); + if (argentServer.pid === undefined) { + throw new SystemError( + 'Failed to start Argent: could not determine the PID of the launched process.' + ); + } + + logger.info(`Waiting for argent tool-server state in ${ARGENT_STATE_DIR}.`); + let toolServerPort: number; + let toolServerToken: string | undefined; + try { + const toolServerState = await waitForArgentToolServerStateAsync({ + stateDir: ARGENT_STATE_DIR, + ancestorPid: argentServer.pid, + timeoutMs: STARTUP_TIMEOUT_MS, + }); + toolServerPort = toolServerState.port; + toolServerToken = toolServerState.token; + } catch (err) { + const output = argentServer.getOutput(); + await argentServer.stopAsync(); + throw new SystemError( + `${ + err instanceof Error ? err.message : `Timed out waiting for argent tool-server state.` + }${output ? `\nArgent tool-server output:\n${output}` : ''}` + ); + } + logger.info(`Argent tool-server is listening on port ${toolServerPort}.`); + + const artifactPollAbortController = new AbortController(); + const artifactPollSignal = signal + ? AbortSignal.any([signal, artifactPollAbortController.signal]) + : artifactPollAbortController.signal; + const artifactPollingPromise = pollArgentArtifactsForUploadAsync(ctx, { + deviceRunSessionId, + toolsUrl: `http://127.0.0.1:${toolServerPort}`, + toolsAuthToken: toolServerToken, + logger, + signal: artifactPollSignal, + }); + const stopArtifactPollingAsync = async (): Promise => { + artifactPollAbortController.abort(); + try { + await artifactPollingPromise; + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + Sentry.capture('Could not finish Argent remote session artifact polling', error); + logger.warn({ err: error }, 'Could not finish Argent remote session artifact polling.'); + } + }; + + let eventCollection: Awaited> | undefined; + let toolsTunnel: Awaited> | undefined; + try { + eventCollection = await startArgentEventCollectionAsync({ + ctx, + deviceRunSessionId, + eventLogPath: ARGENT_EVENT_LOG_PATH, + logger, + }); + const collection = eventCollection; + + toolsTunnel = await startNgrokTunnelAsync({ + port: toolServerPort, + subdomainPrefix: 'argent', + baseDomain: ngrokTunnelDomain, + authtoken: ngrokAuthtoken, + rewriteHostHeader: true, + logger, + }); + const tunnel = toolsTunnel; + logger.info(`Tunnel is ready at ${tunnel.url}.`); + + return { + remoteConfig: { + toolsUrl: tunnel.url, + ...(toolServerToken ? { toolsAuthToken: toolServerToken } : {}), + }, + getLastEventObservedAt: collection.getLastEventObservedAt, + stopAsync: async () => { + await tunnel.stopAsync(); + await stopArgentEventCollectionSafelyAsync({ + eventCollection: collection, + deviceRunSessionId, + logger, + }); + await stopArtifactPollingAsync(); + await argentServer.stopAsync(); + }, + }; + } catch (error) { + await toolsTunnel?.stopAsync(); + if (eventCollection) { + await stopArgentEventCollectionSafelyAsync({ eventCollection, deviceRunSessionId, logger }); + } + await stopArtifactPollingAsync(); + await argentServer.stopAsync(); + throw error; + } +} + export async function stopArgentEventCollectionSafelyAsync({ eventCollection, deviceRunSessionId, diff --git a/packages/build-tools/src/steps/functions/startIosSimulator.ts b/packages/build-tools/src/steps/functions/startIosSimulator.ts index 71ad9f51b7..7b4922a661 100644 --- a/packages/build-tools/src/steps/functions/startIosSimulator.ts +++ b/packages/build-tools/src/steps/functions/startIosSimulator.ts @@ -47,59 +47,18 @@ export function createStartIosSimulatorBuildFunction(): BuildFunction { }), ], fn: async ({ logger }, { inputs, env }) => { - try { - const availableDevices = await IosSimulatorUtils.getAvailableDevicesAsync({ - env, - filter: 'available', - }); - logger.info( - `Available Simulator devices:\n- ${availableDevices - .map(device => device.displayName) - .join(`\n- `)}` - ); - } catch (error) { - logger.info('Failed to list available Simulator devices.', error); - } finally { - logger.info(''); - } - const deviceIdentifierInput = inputs.device_identifier.value?.toString() as | IosSimulatorUuid | IosSimulatorName | undefined; - const originalDeviceIdentifier = - deviceIdentifierInput ?? (await findMostGenericIphoneUuidAsync({ env })); const enableAccessibilitySettings = Boolean(inputs.enable_accessibility_settings.value); - - if (!originalDeviceIdentifier) { - throw new Error('Could not find an iPhone among available simulator devices.'); - } - - if (enableAccessibilitySettings) { - await IosSimulatorUtils.enableAccessibilitySettingsAsync({ - deviceIdentifier: originalDeviceIdentifier, + const { deviceIdentifier: originalDeviceIdentifier, displayName: formattedDevice } = + await bootIosSimulatorAsync({ + deviceIdentifier: deviceIdentifierInput, + enableAccessibilitySettings, env, + logger, }); - } - const udid = await bootWithLocalEgressAsync({ - deviceIdentifier: originalDeviceIdentifier, - env, - logger, - }); - - try { - await IosSimulatorUtils.disableApsdAsync({ udid, env }); - } catch (err) { - logger.warn({ err }, 'Failed to disable apsd in the Simulator.'); - } - - await IosSimulatorUtils.waitForReadyAsync({ udid, env }); - - logger.info(''); - - const device = await IosSimulatorUtils.getDeviceAsync({ udid, env }); - const formattedDevice = device?.displayName ?? originalDeviceIdentifier; - logger.info(`${formattedDevice} is ready.`); const count = Number(inputs.count.value ?? 1); if (count > 1) { @@ -150,6 +109,78 @@ export function createStartIosSimulatorBuildFunction(): BuildFunction { }); } +export type BootedIosSimulator = { + udid: IosSimulatorUuid; + /** The identifier the device was requested by, or the udid picked when none was requested. */ + deviceIdentifier: IosSimulatorUuid | IosSimulatorName; + displayName: string; +}; + +/** + * Boots one iOS Simulator and waits until it accepts input. Shared by the + * `eas/start_ios_simulator` step and the device run session runner. + */ +export async function bootIosSimulatorAsync({ + deviceIdentifier, + enableAccessibilitySettings = false, + env, + logger, +}: { + deviceIdentifier?: IosSimulatorUuid | IosSimulatorName; + enableAccessibilitySettings?: boolean; + env: BuildStepEnv; + logger: bunyan; +}): Promise { + try { + const availableDevices = await IosSimulatorUtils.getAvailableDevicesAsync({ + env, + filter: 'available', + }); + logger.info( + `Available Simulator devices:\n- ${availableDevices + .map(device => device.displayName) + .join(`\n- `)}` + ); + } catch (error) { + logger.info('Failed to list available Simulator devices.', error); + } finally { + logger.info(''); + } + + const resolvedDeviceIdentifier = + deviceIdentifier ?? (await findMostGenericIphoneUuidAsync({ env })); + if (!resolvedDeviceIdentifier) { + throw new Error('Could not find an iPhone among available simulator devices.'); + } + + if (enableAccessibilitySettings) { + await IosSimulatorUtils.enableAccessibilitySettingsAsync({ + deviceIdentifier: resolvedDeviceIdentifier, + env, + }); + } + const udid = await bootWithLocalEgressAsync({ + deviceIdentifier: resolvedDeviceIdentifier, + env, + logger, + }); + + try { + await IosSimulatorUtils.disableApsdAsync({ udid, env }); + } catch (err) { + logger.warn({ err }, 'Failed to disable apsd in the Simulator.'); + } + + await IosSimulatorUtils.waitForReadyAsync({ udid, env }); + + logger.info(''); + + const device = await IosSimulatorUtils.getDeviceAsync({ udid, env }); + const displayName = device?.displayName ?? resolvedDeviceIdentifier; + logger.info(`${displayName} is ready.`); + return { udid, deviceIdentifier: resolvedDeviceIdentifier, displayName }; +} + /** * Boot a device with the local egress environment in place before anything * inside it starts. `simctl boot` returns once the simulator's launchd is up diff --git a/packages/build-tools/src/steps/functions/startLocalEgress.ts b/packages/build-tools/src/steps/functions/startLocalEgress.ts index e0459b4a84..2482bdfd28 100644 --- a/packages/build-tools/src/steps/functions/startLocalEgress.ts +++ b/packages/build-tools/src/steps/functions/startLocalEgress.ts @@ -1,5 +1,5 @@ import { type bunyan } from '@expo/logger'; -import { BuildFunction, BuildRuntimePlatform } from '@expo/steps'; +import { BuildFunction, BuildRuntimePlatform, type BuildStepEnv } from '@expo/steps'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -92,112 +92,134 @@ export function createStartLocalEgressBuildFunction(): BuildFunction { fn: async ({ logger }, { env, signal }) => { const ngrokTunnelDomain = getNgrokTunnelDomainOrThrow(env); const ngrokAuthtoken = getNgrokAuthtokenOrThrow(env); - let workDir: string | undefined; - let server: DetachedProcessHandle | undefined; - let tunnel: NgrokTunnelHandle | undefined; - let finishSetup!: () => void; - const setupFinished = new Promise(resolve => { - finishSetup = resolve; - }); - // Register before acquisition. Cancellation ends setup promptly; pending - // SDK calls retain their own late-result disposal handlers. - const lifetimeSignal = registerLocalEgressResources(async () => { - await setupFinished; - const results = await Promise.allSettled([ - Promise.resolve().then(() => stopLocalEgressGuardRelaysAsync(logger)), - Promise.resolve().then(() => tunnel?.stopAsync()), - Promise.resolve().then(() => server?.stopAsync()), - ]); - for (const result of results) { - if (result.status === 'rejected') { - logger.warn({ err: result.reason }, 'Could not stop a local egress resource.'); - } - } - await Promise.all([ - fs.promises.rm(LOCAL_EGRESS_HANDOFF_PATH, { force: true }), - workDir ? fs.promises.rm(workDir, { recursive: true, force: true }) : undefined, - ]); - }); - const startupSignal = signal ? AbortSignal.any([signal, lifetimeSignal]) : lifetimeSignal; + await startLocalEgressAsync({ ngrokTunnelDomain, ngrokAuthtoken, env, logger, signal }); + }, + }); +} - try { - startupSignal.throwIfAborted(); - workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'eas-local-egress-')); - startupSignal.throwIfAborted(); - logger.info(`Downloading the reverse tunnel server (chisel ${CHISEL_VERSION}).`); - const downloadDir = workDir; - const chiselPath = await awaitLocalEgressAcquisitionAsync( - downloadChiselAsync({ destinationDir: downloadDir, logger }), - startupSignal, - async () => await fs.promises.rm(downloadDir, { recursive: true, force: true }), - logger - ); - startupSignal.throwIfAborted(); +/** + * Starts the local egress resources: the chisel reverse-tunnel server, its ngrok + * tunnel, the system proxy, and the handoff file read when the remote session is + * published. Registers them so `stopLocalEgressResourcesAsync` can release them. + * Shared by the `eas/start_local_egress` step and the device run session runner. + */ +export async function startLocalEgressAsync({ + ngrokTunnelDomain, + ngrokAuthtoken, + env, + logger, + signal, +}: { + ngrokTunnelDomain: string; + ngrokAuthtoken: string; + env: BuildStepEnv; + logger: bunyan; + signal?: AbortSignal; +}): Promise { + let workDir: string | undefined; + let server: DetachedProcessHandle | undefined; + let tunnel: NgrokTunnelHandle | undefined; + let finishSetup!: () => void; + const setupFinished = new Promise(resolve => { + finishSetup = resolve; + }); + // Register before acquisition. Cancellation ends setup promptly; pending + // SDK calls retain their own late-result disposal handlers. + const lifetimeSignal = registerLocalEgressResources(async () => { + await setupFinished; + const results = await Promise.allSettled([ + Promise.resolve().then(() => stopLocalEgressGuardRelaysAsync(logger)), + Promise.resolve().then(() => tunnel?.stopAsync()), + Promise.resolve().then(() => server?.stopAsync()), + ]); + for (const result of results) { + if (result.status === 'rejected') { + logger.warn({ err: result.reason }, 'Could not stop a local egress resource.'); + } + } + await Promise.all([ + fs.promises.rm(LOCAL_EGRESS_HANDOFF_PATH, { force: true }), + workDir ? fs.promises.rm(workDir, { recursive: true, force: true }) : undefined, + ]); + }); + const startupSignal = signal ? AbortSignal.any([signal, lifetimeSignal]) : lifetimeSignal; - const credentials = generateEgressCredentials(); - const authfilePath = path.join(workDir, 'authfile.json'); - await fs.promises.writeFile( - authfilePath, - createChiselAuthfileContents({ ...credentials, port: LOCAL_EGRESS_PROXY_PORT }), - { encoding: 'utf8', mode: 0o600 } - ); - startupSignal.throwIfAborted(); + try { + startupSignal.throwIfAborted(); + workDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'eas-local-egress-')); + startupSignal.throwIfAborted(); + logger.info(`Downloading the reverse tunnel server (chisel ${CHISEL_VERSION}).`); + const downloadDir = workDir; + const chiselPath = await awaitLocalEgressAcquisitionAsync( + downloadChiselAsync({ destinationDir: downloadDir, logger }), + startupSignal, + async () => await fs.promises.rm(downloadDir, { recursive: true, force: true }), + logger + ); + startupSignal.throwIfAborted(); - const controlPort = await findAvailablePortAsync(); - startupSignal.throwIfAborted(); - logger.info(`Starting the reverse tunnel server on 127.0.0.1:${controlPort}.`); - const started = await startChiselServerAsync({ - chiselPath, - controlPort, - authfilePath, - env, - signal: startupSignal, - }); - server = started.process; - startupSignal.throwIfAborted(); - tunnel = await awaitLocalEgressAcquisitionAsync( - startNgrokTunnelAsync({ - port: controlPort, - subdomainPrefix: 'egress', - baseDomain: ngrokTunnelDomain, - authtoken: ngrokAuthtoken, - logger, - }), - startupSignal, - async listener => await listener?.stopAsync(), - logger - ); - startupSignal.throwIfAborted(); + const credentials = generateEgressCredentials(); + const authfilePath = path.join(workDir, 'authfile.json'); + await fs.promises.writeFile( + authfilePath, + createChiselAuthfileContents({ ...credentials, port: LOCAL_EGRESS_PROXY_PORT }), + { encoding: 'utf8', mode: 0o600 } + ); + startupSignal.throwIfAborted(); - const { service } = await configureSystemProxyAsync({ - env, - logger, - port: LOCAL_EGRESS_PROXY_PORT, - signal: startupSignal, - }); - startupSignal.throwIfAborted(); - await writeLocalEgressHandoffAsync({ - url: tunnel.url, - token: credentials.password, - fingerprint: started.fingerprint, - port: LOCAL_EGRESS_PROXY_PORT, - }); - startupSignal.throwIfAborted(); - logger.info( - `Local egress is configured on network service "${service}". HTTP(S) and WebSocket ` + - 'requests that honor the system proxy (WebKit, URLSession and other CFNetwork clients) ' + - 'fail until the EAS CLI egress client connects, then exit from that machine. Once the ' + - 'Simulator boots, proxy environment variables are set inside it for clients that read ' + - 'them (gRPC, libcurl), and the local egress guard is installed so that connections which ' + - 'ignore both are refused in the process that makes them and reported here.' - ); - } catch (error) { - finishSetup(); - await stopLocalEgressResourcesAsync(logger); - throw error; - } finally { - finishSetup(); - } - }, - }); + const controlPort = await findAvailablePortAsync(); + startupSignal.throwIfAborted(); + logger.info(`Starting the reverse tunnel server on 127.0.0.1:${controlPort}.`); + const started = await startChiselServerAsync({ + chiselPath, + controlPort, + authfilePath, + env, + signal: startupSignal, + }); + server = started.process; + startupSignal.throwIfAborted(); + tunnel = await awaitLocalEgressAcquisitionAsync( + startNgrokTunnelAsync({ + port: controlPort, + subdomainPrefix: 'egress', + baseDomain: ngrokTunnelDomain, + authtoken: ngrokAuthtoken, + logger, + }), + startupSignal, + async listener => await listener?.stopAsync(), + logger + ); + startupSignal.throwIfAborted(); + + const { service } = await configureSystemProxyAsync({ + env, + logger, + port: LOCAL_EGRESS_PROXY_PORT, + signal: startupSignal, + }); + startupSignal.throwIfAborted(); + await writeLocalEgressHandoffAsync({ + url: tunnel.url, + token: credentials.password, + fingerprint: started.fingerprint, + port: LOCAL_EGRESS_PROXY_PORT, + }); + startupSignal.throwIfAborted(); + logger.info( + `Local egress is configured on network service "${service}". HTTP(S) and WebSocket ` + + 'requests that honor the system proxy (WebKit, URLSession and other CFNetwork clients) ' + + 'fail until the EAS CLI egress client connects, then exit from that machine. Once the ' + + 'Simulator boots, proxy environment variables are set inside it for clients that read ' + + 'them (gRPC, libcurl), and the local egress guard is installed so that connections which ' + + 'ignore both are refused in the process that makes them and reported here.' + ); + } catch (error) { + finishSetup(); + await stopLocalEgressResourcesAsync(logger); + throw error; + } finally { + finishSetup(); + } } diff --git a/packages/build-tools/src/steps/functions/uploadDeviceRunSessionScreenRecordings.ts b/packages/build-tools/src/steps/functions/uploadDeviceRunSessionScreenRecordings.ts index d168519373..daea50f180 100644 --- a/packages/build-tools/src/steps/functions/uploadDeviceRunSessionScreenRecordings.ts +++ b/packages/build-tools/src/steps/functions/uploadDeviceRunSessionScreenRecordings.ts @@ -1,4 +1,5 @@ import { SystemError } from '@expo/eas-build-job'; +import { type bunyan } from '@expo/logger'; import { BuildFunction, BuildRuntimePlatform, @@ -78,56 +79,82 @@ export function createUploadDeviceRunSessionScreenRecordingsBuildFunction( } const deviceRunSessionId = getDeviceRunSessionIdOrThrow(env); - - const limit = limitFactory(5); - await Promise.all( - recordings.map(recording => - limit(async () => { - try { - const metadata = RecordingManifestSchema.parse( - JSON.parse(await readFile(path.join(recording.directory, 'session.json'), 'utf-8')) - ); - const startedAt = recordingStartTimeFormatter.format( - new Date(metadata.firstFrameWallClock.iso8601) - ); - const shortUdid = `${recording.udid.slice(0, 8)}-…`; - const displayName = `${recording.deviceName} screen recording (${shortUdid}, started at ${startedAt})`; - const recordingPath = path.join(recording.directory, metadata.recording); - const { size } = await stat(recordingPath); - const recordingId = path.basename(recording.directory); - logger.info( - `Uploading screen recording for ${recording.deviceName} (${formatBytes(size)}).` - ); - await uploadDeviceRunSessionArtifactAsync(ctx, { - deviceRunSessionId, - artifactId: recordingId, - name: displayName, - filename: `${recordingId}.mp4`, - kind: 'screen-recording', - metadata: { - __eas_type: 'screen-recording', - __eas_screen_recording: '1', - udid: recording.udid, - deviceName: recording.deviceName, - runtimeDisplayName: recording.runtimeDisplayName, - firstFrameAt: metadata.firstFrameWallClock.iso8601, - width: metadata.width, - height: metadata.height, - }, - size, - stream: createReadStream(recordingPath), - }); - } catch (err) { - const error = err instanceof Error ? err : new Error(String(err)); - Sentry.capture('Could not upload iOS Simulator screen recording', error); - logger.warn( - { err: error }, - `Could not upload screen recording for ${recording.deviceName}.` - ); - } - }) - ) - ); + await uploadIosSimulatorRecordingsAsync(ctx, { deviceRunSessionId, recordings, logger }); }, }); } + +export type IosSimulatorRecording = z.infer[number]; + +/** + * Uploads finished iOS Simulator screen recordings as session artifacts, at most + * five at a time. Failures are reported and logged per recording; they never + * fail the session. Shared by the `eas/upload_device_run_session_screen_recordings` + * step and the device run session runner. + */ +export async function uploadIosSimulatorRecordingsAsync( + ctx: CustomBuildContext, + { + deviceRunSessionId, + recordings, + logger, + }: { + deviceRunSessionId: string; + recordings: IosSimulatorRecording[]; + logger: bunyan; + } +): Promise { + if (recordings.length === 0) { + logger.info('No iOS Simulator recordings found; skipping uploads.'); + return; + } + const limit = limitFactory(5); + await Promise.all( + recordings.map(recording => + limit(async () => { + try { + const metadata = RecordingManifestSchema.parse( + JSON.parse(await readFile(path.join(recording.directory, 'session.json'), 'utf-8')) + ); + const startedAt = recordingStartTimeFormatter.format( + new Date(metadata.firstFrameWallClock.iso8601) + ); + const shortUdid = `${recording.udid.slice(0, 8)}-…`; + const displayName = `${recording.deviceName} screen recording (${shortUdid}, started at ${startedAt})`; + const recordingPath = path.join(recording.directory, metadata.recording); + const { size } = await stat(recordingPath); + const recordingId = path.basename(recording.directory); + logger.info( + `Uploading screen recording for ${recording.deviceName} (${formatBytes(size)}).` + ); + await uploadDeviceRunSessionArtifactAsync(ctx, { + deviceRunSessionId, + artifactId: recordingId, + name: displayName, + filename: `${recordingId}.mp4`, + kind: 'screen-recording', + metadata: { + __eas_type: 'screen-recording', + __eas_screen_recording: '1', + udid: recording.udid, + deviceName: recording.deviceName, + runtimeDisplayName: recording.runtimeDisplayName, + firstFrameAt: metadata.firstFrameWallClock.iso8601, + width: metadata.width, + height: metadata.height, + }, + size, + stream: createReadStream(recordingPath), + }); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + Sentry.capture('Could not upload iOS Simulator screen recording', error); + logger.warn( + { err: error }, + `Could not upload screen recording for ${recording.deviceName}.` + ); + } + }) + ) + ); +} diff --git a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts index e2afbc9f70..44698bc489 100644 --- a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts +++ b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts @@ -476,10 +476,14 @@ export function turnIceServersToWebPreviewArgs(iceServers: TurnIceServers): stri */ export async function fetchWebPreviewTurnArgsAsync( ctx: CustomBuildContext, - { env, logger }: { env: BuildStepEnv; logger: bunyan } + { + env, + logger, + deviceRunSessionId: deviceRunSessionIdOption, + }: { env: BuildStepEnv; logger: bunyan; deviceRunSessionId?: string } ): Promise { try { - const deviceRunSessionId = getDeviceRunSessionIdOrThrow(env); + const deviceRunSessionId = deviceRunSessionIdOption ?? getDeviceRunSessionIdOrThrow(env); const expoApiServerUrl = nullthrows(ctx.env.__API_SERVER_URL, '__API_SERVER_URL is not set'); const robotAccessToken = nullthrows( ctx.job.secrets?.robotAccessToken, @@ -905,6 +909,8 @@ async function startWebPreviewWithTunnelAsync( packageSpec, createArgs, readPreviewTokenAsync, + deviceRunSessionId, + turnArgs: turnArgsOption, }: { baseDomain: string; env: BuildStepEnv; @@ -914,12 +920,17 @@ async function startWebPreviewWithTunnelAsync( packageSpec: string; createArgs: (port: number, turnArgs: string[], previewPageUrl: string) => string[]; readPreviewTokenAsync?: (device: string) => Promise; + deviceRunSessionId?: string; + /** Pre-fetched TURN args; when set, no www request is made here. */ + turnArgs?: string[]; } ): Promise { const subdomainId = randomBytes(16).toString('hex'); const previewPageUrl = simulatorPreviewPageUrl(env, subdomainId); const port = await findAvailablePortAsync(); - const turnArgs = await fetchWebPreviewTurnArgsAsync(ctx, { env, logger }); + const turnArgs = + turnArgsOption ?? + (await fetchWebPreviewTurnArgsAsync(ctx, { env, logger, deviceRunSessionId })); const previewExec = resolvePackageExec( resolveConfiguredPackageManager(env, PackageManager.NPM), createArgs(port, turnArgs, previewPageUrl) @@ -977,6 +988,18 @@ export async function readServeSimPreviewTokenAsync( return servers.find(server => server.udid === udid)?.token; } +export type WebPreviewStartOptions = { + baseDomain: string; + env: BuildStepEnv; + logger: bunyan; + timeoutMs: number; + packageVersion?: string; + /** Session id to use for www calls; defaults to the DEVICE_RUN_SESSION_ID env contract. */ + deviceRunSessionId?: string; + /** Pre-fetched TURN args; when set, no www request is made here. */ + turnArgs?: string[]; +}; + export async function startServeSimWithTunnelAsync( ctx: CustomBuildContext, { @@ -985,16 +1008,12 @@ export async function startServeSimWithTunnelAsync( logger, timeoutMs, packageVersion, + deviceRunSessionId, + turnArgs, launchAppIdentifier, launchArgs, openUrl, - }: { - baseDomain: string; - env: BuildStepEnv; - logger: bunyan; - timeoutMs: number; - packageVersion?: string; - } & ServeSimLaunchOptions + }: WebPreviewStartOptions & ServeSimLaunchOptions ): Promise { const websiteArgs = websiteOriginServeSimArgs(env); return await startWebPreviewWithTunnelAsync(ctx, { @@ -1002,6 +1021,8 @@ export async function startServeSimWithTunnelAsync( env, logger, timeoutMs, + deviceRunSessionId, + turnArgs, serverName: 'serve-sim', packageSpec: createServeSimPackageSpec(packageVersion), createArgs: (port, turnArgs, previewPageUrl) => @@ -1041,14 +1062,9 @@ export async function startExpoDeviceHubWithTunnelAsync( logger, timeoutMs, packageVersion, - }: { - runtimePlatform: BuildRuntimePlatform; - baseDomain: string; - env: BuildStepEnv; - logger: bunyan; - timeoutMs: number; - packageVersion?: string; - } + deviceRunSessionId, + turnArgs, + }: WebPreviewStartOptions & { runtimePlatform: BuildRuntimePlatform } ): Promise { if (runtimePlatform === BuildRuntimePlatform.LINUX) { await ensureFfmpegInstalledOnceAsync({ runtimePlatform, env, logger }); @@ -1058,6 +1074,8 @@ export async function startExpoDeviceHubWithTunnelAsync( env, logger, timeoutMs, + deviceRunSessionId, + turnArgs, serverName: 'expo-device-hub', packageSpec: createExpoDeviceHubPackageSpec(packageVersion), createArgs: (port, turnArgs) => createExpoDeviceHubArgs({ port, turnArgs, packageVersion }), @@ -1072,14 +1090,7 @@ export async function startDeviceWebPreviewWithTunnelAsync( launchArgs, openUrl, ...options - }: { - runtimePlatform: BuildRuntimePlatform; - baseDomain: string; - env: BuildStepEnv; - logger: bunyan; - timeoutMs: number; - packageVersion?: string; - } & ServeSimLaunchOptions + }: WebPreviewStartOptions & ServeSimLaunchOptions & { runtimePlatform: BuildRuntimePlatform } ): Promise { switch (runtimePlatform) { case BuildRuntimePlatform.DARWIN: diff --git a/packages/build-tools/src/utils/outputs.ts b/packages/build-tools/src/utils/outputs.ts index 7e1059e542..b4095ba2d8 100644 --- a/packages/build-tools/src/utils/outputs.ts +++ b/packages/build-tools/src/utils/outputs.ts @@ -9,7 +9,8 @@ export async function uploadJobOutputsToWwwAsync( ctx: BuildStepGlobalContext, { logger, expoApiV2BaseUrl }: { logger: bunyan; expoApiV2BaseUrl: string } ): Promise { - if (!ctx.staticContext.job.outputs) { + const { job } = ctx.staticContext; + if (!('outputs' in job) || !job.outputs) { logger.info('Job defines no outputs, skipping upload'); return; } @@ -22,7 +23,7 @@ export async function uploadJobOutputsToWwwAsync( logger.debug({ dynamicValues: interpolationContext }, 'Using dynamic values'); const outputs = collectJobOutputs({ - jobOutputDefinitions: ctx.staticContext.job.outputs, + jobOutputDefinitions: job.outputs, interpolationContext, }); logger.info('Uploading outputs'); diff --git a/packages/build-tools/src/utils/workflowInterpolationContext.ts b/packages/build-tools/src/utils/workflowInterpolationContext.ts new file mode 100644 index 0000000000..4b151243d1 --- /dev/null +++ b/packages/build-tools/src/utils/workflowInterpolationContext.ts @@ -0,0 +1,11 @@ +import { Job, StaticWorkflowInterpolationContext } from '@expo/eas-build-job'; + +/** + * Build and generic jobs may carry a workflow interpolation context. Device run + * session jobs never do. + */ +export function getWorkflowInterpolationContext( + job: Job +): StaticWorkflowInterpolationContext | undefined { + return 'workflowInterpolationContext' in job ? job.workflowInterpolationContext : undefined; +} diff --git a/packages/eas-build-job/src/index.ts b/packages/eas-build-job/src/index.ts index 22585fe9b0..2bb344fe10 100644 --- a/packages/eas-build-job/src/index.ts +++ b/packages/eas-build-job/src/index.ts @@ -18,6 +18,7 @@ export { Workflow, Platform, Cache, + StaticWorkflowInterpolationContext, WorkflowInterpolationContext, } from './common'; export { Metadata, sanitizeMetadata } from './metadata'; diff --git a/packages/eas-build-job/src/job.ts b/packages/eas-build-job/src/job.ts index 6e31225268..8dc3347856 100644 --- a/packages/eas-build-job/src/job.ts +++ b/packages/eas-build-job/src/job.ts @@ -2,11 +2,12 @@ import Joi from 'joi'; import * as Android from './android'; import { Platform } from './common'; +import { DeviceRunSession } from './deviceRunSessionJob'; import { Generic } from './generic'; import * as Ios from './ios'; export type BuildJob = Android.Job | Ios.Job; -export type Job = BuildJob | Generic.Job; +export type Job = BuildJob | Generic.Job | DeviceRunSession.Job; export const JobSchema = Joi.object({ platform: Joi.string() diff --git a/packages/steps/src/index.ts b/packages/steps/src/index.ts index 418facc9e8..f109b05f23 100644 --- a/packages/steps/src/index.ts +++ b/packages/steps/src/index.ts @@ -11,7 +11,7 @@ export { BuildStepGlobalContext, ExternalBuildContextProvider } from './BuildSte export { BuildWorkflow, executeHookStepsAsync } from './BuildWorkflow'; export { BuildStepEnv } from './BuildStepEnv'; export { BuildFunctionGroup } from './BuildFunctionGroup'; -export { BuildStep } from './BuildStep'; +export { BuildStep, BuildStepLogMarker, BuildStepStatus } from './BuildStep'; export * as errors from './errors'; export * from './interpolation'; export * from './utils/shell/spawn';