feat: support iOS simulator camera videos

This commit is contained in:
Michał Pierzchała
2026-06-10 21:53:22 +02:00
parent af5eeb6835
commit 79c8bc7542
29 changed files with 693 additions and 19 deletions
+1
View File
@@ -159,6 +159,7 @@
"!android-snapshot-helper/dist/*.idsig",
"android-multitouch-helper/dist",
"!android-multitouch-helper/dist/*.idsig",
"third_party/serve-sim-camera",
"src/platforms/linux/atspi-dump.py",
"skills",
"server.json",
+2
View File
@@ -66,12 +66,14 @@ test('apps.open resolves session device identifiers from open response', async (
const result = await client.apps.open({
app: 'Settings',
platform: 'ios',
cameraVideo: './fixtures/back.mp4',
relaunch: true,
});
assert.equal(setup.calls.length, 1);
assert.equal(setup.calls[0]?.command, 'open');
assert.deepEqual(setup.calls[0]?.positionals, ['Settings']);
assert.equal(setup.calls[0]?.flags?.cameraVideo, './fixtures/back.mp4');
assert.equal(result.identifiers.session, 'qa');
assert.equal(result.identifiers.deviceId, 'SIM-001');
assert.equal(result.identifiers.udid, 'SIM-001');
+1
View File
@@ -194,6 +194,7 @@ export type BackendOpenTarget = {
};
export type BackendOpenOptions = {
cameraVideo?: string;
launchArgs?: string[];
relaunch?: boolean;
};
+1
View File
@@ -277,6 +277,7 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags {
androidDeviceAllowlist: options.androidDeviceAllowlist,
surface: options.surface,
activity: options.activity,
cameraVideo: options.cameraVideo,
launchConsole: options.launchConsole,
launchArgs: options.launchArgs,
relaunch: options.relaunch,
+3
View File
@@ -189,6 +189,7 @@ export type AppOpenOptions = AgentDeviceRequestOverrides &
url?: string;
surface?: SessionSurface;
activity?: string;
cameraVideo?: string;
launchConsole?: string;
launchArgs?: string[];
relaunch?: boolean;
@@ -555,6 +556,7 @@ type RepeatedPressOptions = {
export type DeviceBootOptions = DeviceCommandBaseOptions & {
headless?: boolean;
cameraVideo?: string;
cameraFront?: string;
cameraBack?: string;
};
@@ -882,6 +884,7 @@ export type InternalRequestOptions = AgentDeviceClientConfig &
overlayRefs?: boolean;
surface?: SessionSurface;
activity?: string;
cameraVideo?: string;
launchConsole?: string;
launchArgs?: string[];
relaunch?: boolean;
+2
View File
@@ -25,6 +25,7 @@ const MAX_APP_PUSH_PAYLOAD_BYTES = 8 * 1024;
export type OpenAppCommandOptions = CommandContext &
BackendOpenTarget & {
cameraVideo?: string;
launchArgs?: string[];
relaunch?: boolean;
};
@@ -105,6 +106,7 @@ export const openAppCommand: RuntimeCommand<OpenAppCommandOptions, OpenAppComman
toAppBackendContext(runtime, options),
target,
{
...(options.cameraVideo !== undefined ? { cameraVideo: options.cameraVideo } : {}),
launchArgs: options.launchArgs,
relaunch: options.relaunch,
},
+1
View File
@@ -43,6 +43,7 @@ export const appCliReaders = {
url: positionals[1],
surface: flags.surface,
activity: flags.activity,
cameraVideo: flags.cameraVideo,
launchConsole: flags.launchConsole,
launchArgs: flags.launchArgs,
relaunch: flags.relaunch,
+1
View File
@@ -56,6 +56,7 @@ export const clientCommandMetadata = [
url: stringField('Optional URL passed with an app shell.'),
surface: enumField(SESSION_SURFACES),
activity: stringField('Android activity name.'),
cameraVideo: stringField('iOS simulator video file path injected as the app camera stream.'),
launchConsole: stringField('Launch console mode.'),
launchArgs: stringArrayField(
'Launch arguments forwarded verbatim to the platform launch command.',
+46
View File
@@ -77,6 +77,52 @@ test('dispatch open rejects launch arguments without an app target', async () =>
);
});
test('dispatch open rejects camera video without an app target', async () => {
await assert.rejects(
() => dispatchCommand(IOS_SIMULATOR, 'open', [], undefined, { cameraVideo: './back.mp4' }),
(error: unknown) => {
assert.equal(error instanceof AppError, true);
assert.equal((error as AppError).code, 'INVALID_ARGS');
assert.match((error as AppError).message, /requires an app target/i);
return true;
},
);
});
test('dispatch open forwards iOS simulator camera video to openIosApp', async () => {
await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, {
cameraVideo: '/tmp/back.mp4',
});
assert.equal(mockOpenIosApp.mock.calls.length, 1);
assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR);
assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app');
assert.equal(mockOpenIosApp.mock.calls[0]?.[2]?.cameraVideo, '/tmp/back.mp4');
});
test('dispatch open rejects camera video outside iOS simulator', async () => {
const device: DeviceInfo = {
platform: 'android',
id: 'emulator-5554',
name: 'Pixel',
kind: 'emulator',
booted: true,
};
await assert.rejects(
() =>
dispatchCommand(device, 'open', ['com.example.app'], undefined, {
cameraVideo: './back.mp4',
}),
(error: unknown) => {
assert.equal(error instanceof AppError, true);
assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION');
assert.match((error as AppError).message, /iOS simulators/i);
return true;
},
);
});
test('dispatch open forwards Android launch arguments to openAndroidApp', async () => {
const device: DeviceInfo = {
platform: 'android',
+1
View File
@@ -33,6 +33,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
requestId?: string;
appBundleId?: string;
activity?: string;
cameraVideo?: string;
launchConsole?: string;
launchArgs?: string[];
clearAppState?: boolean;
+15
View File
@@ -175,6 +175,7 @@ async function handleOpenCommand(
): Promise<Record<string, unknown>> {
const app = positionals[0];
const url = positionals[1];
const cameraVideo = context?.cameraVideo;
const launchConsole = context?.launchConsole;
const launchArgs = context?.launchArgs;
if (positionals.length > 2) {
@@ -187,9 +188,18 @@ async function handleOpenCommand(
if (launchArgs && launchArgs.length > 0) {
throw new AppError('INVALID_ARGS', '--launch-args requires an app target');
}
if (cameraVideo) {
throw new AppError('INVALID_ARGS', '--camera-video requires an app target');
}
await interactor.openDevice();
return { app: null, ...successText('Opened device') };
}
if (cameraVideo && (device.platform !== 'ios' || device.kind !== 'simulator')) {
throw new AppError(
'UNSUPPORTED_OPERATION',
'--camera-video is supported only for iOS simulators.',
);
}
if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) {
throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE);
}
@@ -212,6 +222,7 @@ async function handleOpenCommand(
await interactor.open(app, {
activity: context?.activity,
appBundleId: context?.appBundleId,
cameraVideo,
launchArgs,
url,
});
@@ -220,6 +231,9 @@ async function handleOpenCommand(
if (launchConsole && isDeepLinkTarget(app)) {
throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE);
}
if (cameraVideo && isDeepLinkTarget(app)) {
throw new AppError('INVALID_ARGS', '--camera-video requires an app target');
}
if (context?.clearAppState) {
if (isDeepLinkTarget(app)) {
throw new AppError(
@@ -232,6 +246,7 @@ async function handleOpenCommand(
await interactor.open(app, {
activity: context?.activity,
appBundleId: context?.appBundleId,
cameraVideo,
launchConsole,
launchArgs,
});
+1
View File
@@ -57,6 +57,7 @@ export type Interactor = {
options?: {
activity?: string;
appBundleId?: string;
cameraVideo?: string;
launchConsole?: string;
launchArgs?: string[];
url?: string;
+1
View File
@@ -29,6 +29,7 @@ export function createAppleInteractor(
open: (app, options) =>
openIosApp(device, app, {
appBundleId: options?.appBundleId,
cameraVideo: options?.cameraVideo,
launchConsole: options?.launchConsole,
launchArgs: options?.launchArgs,
url: options?.url,
+3
View File
@@ -54,6 +54,7 @@ export type OpenAppOptions = {
udid?: NonNullable<DaemonRequest['flags']>['udid'];
serial?: NonNullable<DaemonRequest['flags']>['serial'];
activity?: NonNullable<DaemonRequest['flags']>['activity'];
cameraVideo?: NonNullable<DaemonRequest['flags']>['cameraVideo'];
launchConsole?: NonNullable<DaemonRequest['flags']>['launchConsole'];
launchArgs?: NonNullable<DaemonRequest['flags']>['launchArgs'];
out?: NonNullable<DaemonRequest['flags']>['out'];
@@ -226,6 +227,7 @@ export async function openApp(options: OpenAppOptions = {}): Promise<DaemonRespo
udid,
serial,
activity,
cameraVideo,
launchConsole,
launchArgs,
out,
@@ -248,6 +250,7 @@ export async function openApp(options: OpenAppOptions = {}): Promise<DaemonRespo
...(udid !== undefined ? { udid } : {}),
...(serial !== undefined ? { serial } : {}),
...(activity !== undefined ? { activity } : {}),
...(cameraVideo !== undefined ? { cameraVideo } : {}),
...(launchConsole !== undefined ? { launchConsole } : {}),
...(launchArgs !== undefined ? { launchArgs } : {}),
...(out !== undefined ? { out } : {}),
+6
View File
@@ -20,6 +20,12 @@ test('contextFromFlags forwards generic app-state clearing', () => {
assert.equal(context.clearAppState, true);
});
test('contextFromFlags forwards iOS simulator camera video path', () => {
const flags: CommandFlags = { cameraVideo: './fixtures/camera-feed.mp4' };
const context = contextFromFlags('/tmp/agent-device.log', flags);
assert.equal(context.cameraVideo, './fixtures/camera-feed.mp4');
});
test('contextFromFlags forwards screenshot flags from CLI flags', () => {
const flags: CommandFlags = {
screenshotFullscreen: true,
+1
View File
@@ -23,6 +23,7 @@ export function contextFromFlags(
requestId: effectiveRequestId,
appBundleId,
activity: flags?.activity,
cameraVideo: flags?.cameraVideo,
launchConsole: flags?.launchConsole,
launchArgs: flags?.launchArgs,
clearAppState: flags?.clearAppState,
+16 -1
View File
@@ -101,11 +101,26 @@ function contextForRuntimeLaunchUrl(
traceLogPath?: string,
): ReturnType<typeof contextFromFlags> {
const context = contextFromFlags(logPath, flags, appBundleId, traceLogPath);
delete context.cameraVideo;
delete context.launchConsole;
delete context.launchArgs;
return context;
}
function contextForOpenDispatch(
logPath: string,
flags: DaemonRequest['flags'],
appBundleId: string | undefined,
traceLogPath: string | undefined,
cwd: string | undefined,
): ReturnType<typeof contextFromFlags> {
const context = contextFromFlags(logPath, flags, appBundleId, traceLogPath);
if (context.cameraVideo) {
context.cameraVideo = SessionStore.expandHome(context.cameraVideo, cwd);
}
return context;
}
function buildStartupPerfSample(
startedAtMs: number,
appTarget: string | undefined,
@@ -218,7 +233,7 @@ async function completeOpenCommand(params: {
}
const openDispatchSession = provisionalSession.session ?? existingSession;
await dispatchCommand(device, 'open', openPositionals, req.flags?.out, {
...contextFromFlags(logPath, req.flags, sessionAppBundleId),
...contextForOpenDispatch(logPath, req.flags, sessionAppBundleId, traceLogPath, req.meta?.cwd),
});
timing.openDispatchDurationMs = Math.max(0, Date.now() - openStartedAtMs);
const launchUrlStartedAtMs = Date.now();
@@ -0,0 +1,84 @@
import assert from 'node:assert/strict';
import fsp from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { afterEach, test, vi } from 'vitest';
import { IOS_DEVICE, IOS_SIMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts';
import { AppError } from '../../../utils/errors.ts';
import { runCmdDetached } from '../../../utils/exec.ts';
import {
prepareIosSimulatorCameraVideo,
stopIosSimulatorCameraVideo,
} from '../simulator-camera.ts';
vi.mock('../../../utils/exec.ts', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../../utils/exec.ts')>();
return {
...actual,
runCmdDetached: vi.fn(() => 987_654),
};
});
const mockRunCmdDetached = vi.mocked(runCmdDetached);
afterEach(() => {
mockRunCmdDetached.mockClear();
});
test('prepareIosSimulatorCameraVideo starts vendored helper and returns simctl child env', async () => {
const tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'agent-device-ios-camera-test-'));
const videoPath = path.join(tempDir, 'sample.mp4');
await fsp.writeFile(videoPath, 'fixture');
try {
const launch = await prepareIosSimulatorCameraVideo({
device: IOS_SIMULATOR,
bundleId: 'com.example.camera',
videoPath,
});
assert.equal(mockRunCmdDetached.mock.calls.length, 1);
const [helperPath, helperArgs, helperOptions] = mockRunCmdDetached.mock.calls[0] ?? [];
assert.match(helperPath ?? '', /third_party\/serve-sim-camera\/bin\/camera-helper$/);
assert.deepEqual(helperArgs, [
'--shm',
launch.shmName,
'--source',
'video',
'--arg',
videoPath,
]);
assert.equal(launch.helperPid, 987_654);
assert.equal(launch.videoPath, videoPath);
assert.match(launch.shmName, /^\/ad-camera-[a-f0-9]{12}$/);
assert.deepEqual(helperOptions?.stdio?.[0], 'ignore');
assert.equal(typeof helperOptions?.stdio?.[1], 'number');
assert.equal(helperOptions?.stdio?.[2], helperOptions?.stdio?.[1]);
assert.match(
launch.env.SIMCTL_CHILD_DYLD_INSERT_LIBRARIES ?? '',
/camera-injector\.dylib$/,
);
const shmEnvKey = Object.keys(launch.env).find((key) => key.endsWith('_SHM_NAME'));
const mirrorEnvKey = Object.keys(launch.env).find((key) => key.endsWith('_MIRROR_MODE'));
assert.equal(launch.env[shmEnvKey ?? ''], launch.shmName);
assert.equal(launch.env[mirrorEnvKey ?? ''], 'auto');
} finally {
await stopIosSimulatorCameraVideo(IOS_SIMULATOR, 'com.example.camera');
await fsp.rm(tempDir, { force: true, recursive: true });
}
});
test('prepareIosSimulatorCameraVideo rejects non-simulator devices', async () => {
await assert.rejects(
() =>
prepareIosSimulatorCameraVideo({
device: IOS_DEVICE,
bundleId: 'com.example.camera',
videoPath: '/tmp/sample.mp4',
}),
(error: unknown) => {
assert.equal(error instanceof AppError, true);
assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION');
return true;
},
);
});
+54 -17
View File
@@ -51,6 +51,7 @@ import { buildSimctlArgsForDevice } from './simctl.ts';
import { runAppleToolCommand, runXcrun } from './tool-provider.ts';
import { prepareIosInstallArtifact } from './install-artifact.ts';
import { filterAppleAppsByBundlePrefix } from './app-filter.ts';
import { prepareIosSimulatorCameraVideo, stopIosSimulatorCameraVideo } from './simulator-camera.ts';
import {
closeMacOsApp,
listMacApps,
@@ -174,13 +175,26 @@ function parseUrlScheme(url: string): string | undefined {
export async function openIosApp(
device: DeviceInfo,
app: string,
options?: { appBundleId?: string; launchConsole?: string; launchArgs?: string[]; url?: string },
options?: {
appBundleId?: string;
launchConsole?: string;
launchArgs?: string[];
cameraVideo?: string;
url?: string;
},
): Promise<void> {
const launchConsole = options?.launchConsole?.trim();
const launchArgs = options?.launchArgs;
const cameraVideo = options?.cameraVideo?.trim();
if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) {
throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE);
}
if (cameraVideo && (device.platform !== 'ios' || device.kind !== 'simulator')) {
throw new AppError(
'UNSUPPORTED_OPERATION',
'--camera-video is supported only for iOS simulators.',
);
}
if (device.platform === 'macos') {
if (launchArgs && launchArgs.length > 0) {
throw new AppError(
@@ -203,6 +217,7 @@ export async function openIosApp(
const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app));
await launchIosSimulatorApp(device, bundleId, {
...(launchArgs ? { launchArgs } : {}),
...(cameraVideo ? { cameraVideo } : {}),
});
await openIosSimulatorUrl(device, explicitUrl, undefined);
return;
@@ -224,6 +239,9 @@ export async function openIosApp(
if (launchConsole) {
throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE);
}
if (cameraVideo) {
throw new AppError('INVALID_ARGS', '--camera-video requires an app target.');
}
if (device.kind === 'simulator') {
await openIosSimulatorUrl(device, deepLinkTarget, launchArgs);
return;
@@ -244,6 +262,7 @@ export async function openIosApp(
await launchIosSimulatorApp(device, bundleId, {
...(launchConsole ? { launchConsole } : {}),
...(launchArgs ? { launchArgs } : {}),
...(cameraVideo ? { cameraVideo } : {}),
});
return;
}
@@ -283,20 +302,24 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise<void
if (device.kind === 'simulator') {
await ensureBootedSimulator(device);
const terminateArgs = simctlArgs(device, ['terminate', device.id, bundleId]);
const result = await runXcrun(terminateArgs, {
allowFailure: true,
timeoutMs: IOS_SIMULATOR_TERMINATE_TIMEOUT_MS,
});
if (result.exitCode !== 0) {
const stderr = result.stderr.toLowerCase();
if (stderr.includes('found nothing to terminate')) return;
throw new AppError('COMMAND_FAILED', `xcrun exited with code ${result.exitCode}`, {
cmd: 'xcrun',
args: terminateArgs,
stdout: result.stdout,
stderr: result.stderr,
exitCode: result.exitCode,
try {
const result = await runXcrun(terminateArgs, {
allowFailure: true,
timeoutMs: IOS_SIMULATOR_TERMINATE_TIMEOUT_MS,
});
if (result.exitCode !== 0) {
const stderr = result.stderr.toLowerCase();
if (stderr.includes('found nothing to terminate')) return;
throw new AppError('COMMAND_FAILED', `xcrun exited with code ${result.exitCode}`, {
cmd: 'xcrun',
args: terminateArgs,
stdout: result.stdout,
stderr: result.stderr,
exitCode: result.exitCode,
});
}
} finally {
await stopIosSimulatorCameraVideo(device, bundleId);
}
return;
}
@@ -1089,9 +1112,16 @@ function isIosBiometricCapabilityMissing(stdout: string, stderr: string): boolea
async function launchIosSimulatorApp(
device: DeviceInfo,
bundleId: string,
options?: { launchConsole?: string; launchArgs?: string[] },
options?: { launchConsole?: string; launchArgs?: string[]; cameraVideo?: string },
): Promise<void> {
await ensureBootedSimulator(device);
const cameraLaunch = options?.cameraVideo
? await prepareIosSimulatorCameraVideo({
device,
bundleId,
videoPath: options.cameraVideo,
})
: undefined;
let consecutiveFBSFailures = 0;
const MAX_CONSECUTIVE_FBS_FAILURES = 3;
@@ -1111,9 +1141,10 @@ async function launchIosSimulatorApp(
buildIosSimulatorLaunchArgs(device.id, bundleId, options),
);
const result = options?.launchConsole
? await runIosSimulatorConsoleLaunch(launchArgs, options.launchConsole)
? await runIosSimulatorConsoleLaunch(launchArgs, options.launchConsole, cameraLaunch?.env)
: await runXcrun(launchArgs, {
allowFailure: true,
...(cameraLaunch?.env ? { env: { ...process.env, ...cameraLaunch.env } } : {}),
});
if (result.exitCode === 0) return;
@@ -1139,6 +1170,9 @@ async function launchIosSimulatorApp(
{ deadline: launchDeadline },
);
} catch (error) {
if (cameraLaunch) {
await stopIosSimulatorCameraVideo(device, bundleId).catch(() => {});
}
if (isSimulatorLaunchFBSError(error)) {
const appError = error as AppError;
const probe = await probeSimulatorLaunchContext(device, bundleId);
@@ -1152,10 +1186,11 @@ async function launchIosSimulatorApp(
function buildIosSimulatorLaunchArgs(
deviceId: string,
bundleId: string,
options?: { launchConsole?: string; launchArgs?: string[] },
options?: { launchConsole?: string; launchArgs?: string[]; cameraVideo?: string },
): string[] {
const args = ['launch'];
if (options?.launchConsole) args.push('--console-pty');
if (options?.cameraVideo) args.push('--terminate-running-process');
args.push(deviceId, bundleId);
if (options?.launchArgs && options.launchArgs.length > 0) {
args.push(...options.launchArgs);
@@ -1166,12 +1201,14 @@ function buildIosSimulatorLaunchArgs(
async function runIosSimulatorConsoleLaunch(
launchArgs: string[],
logPath: string,
env?: NodeJS.ProcessEnv,
): Promise<Awaited<ReturnType<typeof runXcrun>>> {
await fs.mkdir(path.dirname(logPath), { recursive: true });
try {
const result = await runXcrun(launchArgs, {
allowFailure: true,
timeoutMs: IOS_SIMULATOR_CONSOLE_CAPTURE_MS,
...(env ? { env: { ...process.env, ...env } } : {}),
});
await writeIosSimulatorConsoleLog(logPath, result.stdout, result.stderr);
return result;
+191
View File
@@ -0,0 +1,191 @@
import fs from 'node:fs';
import fsp from 'node:fs/promises';
import { createHash } from 'node:crypto';
import os from 'node:os';
import path from 'node:path';
import { findProjectRoot } from '../../utils/version.ts';
import { runCmdDetached } from '../../utils/exec.ts';
import { AppError } from '../../utils/errors.ts';
import type { DeviceInfo } from '../../utils/device.ts';
type IosSimulatorCameraHelperState = {
pid: number;
shmName: string;
videoPath: string;
logPath: string;
startedAt: string;
};
export type IosSimulatorCameraLaunch = {
env: NodeJS.ProcessEnv;
videoPath: string;
shmName: string;
helperPid: number;
};
const CAMERA_VENDOR_ROOT = path.join('third_party', 'serve-sim-camera');
const CAMERA_HELPER_RELATIVE_PATH = path.join(
CAMERA_VENDOR_ROOT,
'bin',
'camera-helper',
);
const CAMERA_INJECTOR_RELATIVE_PATH = path.join(
CAMERA_VENDOR_ROOT,
'bin',
'camera-injector.dylib',
);
export async function prepareIosSimulatorCameraVideo(params: {
device: DeviceInfo;
bundleId: string;
videoPath: string;
}): Promise<IosSimulatorCameraLaunch> {
assertIosSimulatorCameraSupported(params.device);
const videoPath = await resolveReadableVideoPath(params.videoPath);
const helperPath = resolveVendorExecutable(CAMERA_HELPER_RELATIVE_PATH);
const injectorPath = resolveVendorExecutable(CAMERA_INJECTOR_RELATIVE_PATH);
await stopIosSimulatorCameraVideo(params.device, params.bundleId);
const shmName = buildShmName(params.device.id, params.bundleId);
const logPath = helperLogPath(params.device, params.bundleId);
await fsp.mkdir(path.dirname(logPath), { recursive: true });
const logFd = fs.openSync(logPath, 'w');
let helperPid = 0;
try {
helperPid = runCmdDetached(
helperPath,
['--shm', shmName, '--source', 'video', '--arg', videoPath],
{
stdio: ['ignore', logFd, logFd],
},
);
} finally {
fs.closeSync(logFd);
}
await writeHelperState(params.device, params.bundleId, {
pid: helperPid,
shmName,
videoPath,
logPath,
startedAt: new Date().toISOString(),
});
return {
videoPath,
shmName,
helperPid,
env: {
SIMCTL_CHILD_DYLD_INSERT_LIBRARIES: injectorPath,
// Upstream serve-sim injector ABI. simctl strips the SIMCTL_CHILD_ prefix.
SIMCTL_CHILD_SIMCAM_SHM_NAME: shmName,
SIMCTL_CHILD_SIMCAM_MIRROR_MODE: 'auto',
},
};
}
export async function stopIosSimulatorCameraVideo(
device: DeviceInfo,
bundleId: string | undefined,
): Promise<void> {
if (device.platform !== 'ios' || device.kind !== 'simulator' || !bundleId) return;
const statePath = helperStatePath(device, bundleId);
const state = await readHelperState(statePath);
if (!state) return;
try {
if (state.pid > 0) {
process.kill(state.pid, 'SIGTERM');
}
} catch (error) {
if (!isMissingProcessError(error)) throw error;
} finally {
await fsp.rm(statePath, { force: true });
}
}
function assertIosSimulatorCameraSupported(device: DeviceInfo): void {
if (device.platform === 'ios' && device.kind === 'simulator') return;
throw new AppError(
'UNSUPPORTED_OPERATION',
'--camera-video is supported only for iOS simulators.',
{
platform: device.platform,
kind: device.kind,
},
);
}
async function resolveReadableVideoPath(value: string): Promise<string> {
const resolvedPath = path.resolve(value);
try {
const stat = await fsp.stat(resolvedPath);
if (stat.isFile()) return resolvedPath;
} catch {}
throw new AppError('INVALID_ARGS', `Camera video file does not exist: ${resolvedPath}`, {
hint: 'Pass a readable sample video path to --camera-video.',
});
}
function resolveVendorExecutable(relativePath: string): string {
const executablePath = path.join(findProjectRoot(), relativePath);
if (fs.existsSync(executablePath)) return executablePath;
throw new AppError('COMMAND_FAILED', 'Bundled iOS simulator camera helper is missing.', {
expectedPath: executablePath,
});
}
function buildShmName(deviceId: string, bundleId: string): string {
const hash = createHash('sha1')
.update(`${deviceId}:${bundleId}:${Date.now()}`)
.digest('hex')
.slice(0, 12);
return `/ad-camera-${hash}`;
}
function helperStatePath(device: DeviceInfo, bundleId: string): string {
const key = `${device.id}-${bundleId}`.replaceAll(/[^A-Za-z0-9._-]/g, '-');
return path.join(os.tmpdir(), 'agent-device-ios-camera', `${key}.json`);
}
function helperLogPath(device: DeviceInfo, bundleId: string): string {
const key = `${device.id}-${bundleId}`.replaceAll(/[^A-Za-z0-9._-]/g, '-');
return path.join(os.tmpdir(), 'agent-device-ios-camera', `${key}.log`);
}
async function writeHelperState(
device: DeviceInfo,
bundleId: string,
state: IosSimulatorCameraHelperState,
): Promise<void> {
const statePath = helperStatePath(device, bundleId);
await fsp.mkdir(path.dirname(statePath), { recursive: true });
await fsp.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
}
async function readHelperState(
statePath: string,
): Promise<IosSimulatorCameraHelperState | undefined> {
try {
const state = JSON.parse(
await fsp.readFile(statePath, 'utf8'),
) as Partial<IosSimulatorCameraHelperState>;
if (
typeof state.pid === 'number' &&
typeof state.shmName === 'string' &&
typeof state.videoPath === 'string' &&
typeof state.logPath === 'string' &&
typeof state.startedAt === 'string'
) {
return state as IosSimulatorCameraHelperState;
}
} catch {}
return undefined;
}
function isMissingProcessError(error: unknown): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: unknown }).code === 'ESRCH'
);
}
+18
View File
@@ -22,6 +22,24 @@ test('parseArgs recognizes command-specific flag combinations', async () => {
assert.equal(parsed.flags.relaunch, true);
},
},
{
label: 'open --camera-video',
argv: [
'open',
'com.example.app',
'--platform',
'ios',
'--camera-video',
'./fixtures/back.mp4',
],
strictFlags: true,
assertParsed: (parsed) => {
assert.equal(parsed.command, 'open');
assert.deepEqual(parsed.positionals, ['com.example.app']);
assert.equal(parsed.flags.platform, 'ios');
assert.equal(parsed.flags.cameraVideo, './fixtures/back.mp4');
},
},
{
label: 'open --platform ios --target tv',
argv: ['open', 'Settings', '--platform', 'ios', '--target', 'tv'],
+9 -1
View File
@@ -82,7 +82,15 @@ const CLI_COMMAND_OVERRIDES = {
'Boot device/simulator; optionally launch app or deep link URL (macOS also supports --surface app|frontmost-app|desktop|menubar)',
summary: 'Open an app, deep link or URL, save replays',
positionalArgs: ['appOrUrl?', 'url?'],
allowedFlags: ['activity', 'launchConsole', 'launchArgs', 'saveScript', 'relaunch', 'surface'],
allowedFlags: [
'activity',
'cameraVideo',
'launchConsole',
'launchArgs',
'saveScript',
'relaunch',
'surface',
],
},
close: {
positionalArgs: ['app?'],
+8
View File
@@ -80,6 +80,7 @@ export type CliFlags = RemoteConfigMetroOptions &
pauseMs?: number;
pattern?: SwipePattern;
activity?: string;
cameraVideo?: string;
launchConsole?: string;
launchArgs?: string[];
header?: string[];
@@ -532,6 +533,13 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [
usageLabel: '--launch-console <path>',
usageDescription: 'open: capture the initial iOS simulator launch console window to a file',
},
{
key: 'cameraVideo',
names: ['--camera-video'],
type: 'string',
usageLabel: '--camera-video <videoPath>',
usageDescription: 'open: iOS simulator video file injected as the app camera stream',
},
{
key: 'launchArgs',
names: ['--launch-args'],
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2026 Evan Bacon
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+20
View File
@@ -0,0 +1,20 @@
# serve-sim camera vendor
This directory vendors the iOS simulator camera helper and injector from
`serve-sim`.
- Upstream: https://github.com/EvanBacon/serve-sim
- Imported package: `serve-sim@0.1.34`
- License: Apache-2.0, copied in `LICENSE`
- Imported paths:
- `bin/camera-injector.dylib`
- `bin/camera-helper`
The imported binaries were renamed locally to avoid exposing upstream internal
artifact names in this codebase.
Local integration code lives outside this directory. Keep local modifications
to vendored artifacts minimal; when changing copied upstream artifacts,
document the change here and preserve Apache-2.0 attribution.
Current local modifications: none.
BIN
View File
Binary file not shown.
Binary file not shown.
+3
View File
@@ -120,6 +120,9 @@ await client.sessions.close();
For direct iOS simulator app launches, `client.apps.open({ app, platform: 'ios', launchConsole: './artifacts/app.console.log' })` captures launch-time
stdout/stderr. The option mirrors `open --launch-console` and is not valid for URL opens or non-simulator targets.
For iOS simulator camera tests, `client.apps.open({ app, platform: 'ios', cameraVideo: './fixtures/camera-feed.mp4' })` injects the video file as the
target app's camera stream for that launch. It relaunches the app process and is not valid for URL-only opens, physical devices, Android, macOS, or Linux.
## Android snapshot helper providers
Remote Android providers should import `agent-device/android-snapshot-helper` and inject their own
+2
View File
@@ -42,6 +42,7 @@ agent-device boot --platform android --device Pixel_9_Pro_XL --camera-back ./bac
agent-device shutdown --platform ios
agent-device shutdown --platform android --device Pixel_9_Pro_XL
agent-device open [app|url] [url]
agent-device open com.example.CameraApp --platform ios --camera-video ./camera-feed.mp4
agent-device open --platform macos --surface frontmost-app
agent-device open --platform macos --surface desktop
agent-device close [app]
@@ -69,6 +70,7 @@ agent-device app-switcher
- `open [app|url] [url]` already boots/activates the selected target when needed.
- `open <url>` deep links are supported on Android and iOS.
- `open <app> <url>` opens a deep link on iOS.
- `open <app> --camera-video <videoPath>` injects a sample video file as the iOS simulator camera stream for that app launch. It relaunches the target app process and is not valid for URL-only opens, physical devices, Android, macOS, or Linux.
- `open <app> --launch-console <path>` captures launch-time stdout/stderr for direct iOS simulator app launches. It is not valid for URL opens or
non-simulator targets.
- `open --platform macos --surface app|frontmost-app|desktop|menubar` selects the macOS session surface explicitly. `app` is the default when an app argument is provided.