From 7bc4193a1e2a2c679c78aecec836e3e9676ac74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 2 Feb 2026 21:33:45 +0100 Subject: [PATCH] feat: add recording --- .../references/missing-features.md | 9 -- skills/agent-device/references/recording.md | 22 ----- .../references/video-recording.md | 39 ++++++++ src/daemon.ts | 88 +++++++++++++++++++ src/utils/args.ts | 2 + src/utils/exec.ts | 59 +++++++++++++ 6 files changed, 188 insertions(+), 31 deletions(-) delete mode 100644 skills/agent-device/references/recording.md create mode 100644 skills/agent-device/references/video-recording.md diff --git a/skills/agent-device/references/missing-features.md b/skills/agent-device/references/missing-features.md index 114deecd1..8569bb29c 100644 --- a/skills/agent-device/references/missing-features.md +++ b/skills/agent-device/references/missing-features.md @@ -2,18 +2,9 @@ This is a prioritized, mobile-specific gap list based on agent-browser. -## P0 (core parity) - -- Waits: wait for text, wait for element (by ref), wait for app idle. -- Alerts: detect, accept, dismiss, get text. -- Navigation: back, home, app switcher (where supported). -- Scroll into view by text for iOS (Android already has scrollintoview). -- Snapshot scopes by ref (e.g., snapshot -s "label" or use node ref as scope). - ## P1 (debug and reliability) - Highlight ref (visual debug overlay for refs). -- Record start/stop for simulator and adb (CLI wrapper). - Trace/log capture toggle for XCTest/AX. - Retry strategy knobs (backoff, max retries) for flaky operations. diff --git a/skills/agent-device/references/recording.md b/skills/agent-device/references/recording.md deleted file mode 100644 index 1eee09171..000000000 --- a/skills/agent-device/references/recording.md +++ /dev/null @@ -1,22 +0,0 @@ -# Recording - -## iOS Simulator - -Use simctl recordVideo directly for now: - -```bash -xcrun simctl io booted recordVideo /tmp/recording.mov -``` - -Stop with Ctrl+C. - -## Android Emulator/Device - -Use adb screenrecord: - -```bash -adb -s shell screenrecord /sdcard/recording.mp4 -adb -s pull /sdcard/recording.mp4 ./recording.mp4 -``` - -Stop with Ctrl+C. diff --git a/skills/agent-device/references/video-recording.md b/skills/agent-device/references/video-recording.md new file mode 100644 index 000000000..1c839fec4 --- /dev/null +++ b/skills/agent-device/references/video-recording.md @@ -0,0 +1,39 @@ +# Video Recording + +Capture device automation sessions as video for debugging, documentation, or verification + +## iOS Simulator + +Use `agent-device record` commands (wrapper around simctl): + +```bash +# Start recording +agent-device record start ./recordings/ios.mov + +# Perform actions +agent-device open App +agent-device snapshot +agent-device click @e3 +agent-device close + +# Stop recording +agent-device record stop +``` + +## Android Emulator/Device + +Use `agent-device record` commands (wrapper around adb): + +```bash +# Start recording +agent-device record start ./recordings/android.mp4 + +# Perform actions +agent-device open App +agent-device snapshot +agent-device click @e3 +agent-device close + +# Stop recording +agent-device record stop +``` diff --git a/src/daemon.ts b/src/daemon.ts index 66c267f7f..d325eadd0 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -16,6 +16,7 @@ import { type RawSnapshotNode, } from './utils/snapshot.ts'; import { runIosRunnerCommand, stopIosRunnerSession } from './platforms/ios/runner-client.ts'; +import { runCmd, runCmdBackground } from './utils/exec.ts'; import { snapshotAndroid } from './platforms/android/index.ts'; type DaemonRequest = { @@ -38,6 +39,13 @@ type SessionState = { appName?: string; snapshot?: SnapshotState; actions: SessionAction[]; + recording?: { + platform: 'ios' | 'android'; + outPath: string; + remotePath?: string; + child: ReturnType; + wait: Promise; + }; }; type SessionAction = { @@ -492,6 +500,86 @@ async function handleRequest(req: DaemonRequest): Promise { return { ok: true, data }; } + if (command === 'record') { + const action = (req.positionals?.[0] ?? '').toLowerCase(); + if (!['start', 'stop'].includes(action)) { + return { ok: false, error: { code: 'INVALID_ARGS', message: 'record requires start|stop' } }; + } + const session = sessions.get(sessionName); + const device = session?.device ?? (await resolveTargetDevice(req.flags ?? {})); + if (!session) { + await ensureDeviceReady(device); + } + const activeSession = session ?? { + name: sessionName, + device, + createdAt: Date.now(), + actions: [], + }; + + if (action === 'start') { + if (activeSession.recording) { + return { ok: false, error: { code: 'INVALID_ARGS', message: 'recording already in progress' } }; + } + const outPath = req.positionals?.[1] ?? `./recording-${Date.now()}.mp4`; + const resolvedOut = path.resolve(outPath); + const outDir = path.dirname(resolvedOut); + if (!fs.existsSync(outDir)) { + fs.mkdirSync(outDir, { recursive: true }); + } + if (device.platform === 'ios') { + if (device.kind !== 'simulator') { + return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'record is only supported on iOS simulators in v1' } }; + } + const { child, wait } = runCmdBackground('xcrun', ['simctl', 'io', device.id, 'recordVideo', resolvedOut], { + allowFailure: true, + }); + activeSession.recording = { platform: 'ios', outPath: resolvedOut, child, wait }; + } else { + const remotePath = `/sdcard/agent-device-recording-${Date.now()}.mp4`; + const { child, wait } = runCmdBackground('adb', ['-s', device.id, 'shell', 'screenrecord', remotePath], { + allowFailure: true, + }); + activeSession.recording = { platform: 'android', outPath: resolvedOut, remotePath, child, wait }; + } + sessions.set(sessionName, activeSession); + recordAction(activeSession, { + command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { action: 'start' }, + }); + return { ok: true, data: { recording: 'started', outPath } }; + } + + if (!activeSession.recording) { + return { ok: false, error: { code: 'INVALID_ARGS', message: 'no active recording' } }; + } + const recording = activeSession.recording; + recording.child.kill('SIGINT'); + try { + await recording.wait; + } catch { + // ignore + } + if (recording.platform === 'android' && recording.remotePath) { + try { + await runCmd('adb', ['-s', device.id, 'pull', recording.remotePath, recording.outPath], { allowFailure: true }); + await runCmd('adb', ['-s', device.id, 'shell', 'rm', '-f', recording.remotePath], { allowFailure: true }); + } catch { + // ignore + } + } + activeSession.recording = undefined; + recordAction(activeSession, { + command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { action: 'stop', outPath: recording.outPath }, + }); + return { ok: true, data: { recording: 'stopped', outPath: recording.outPath } }; + } + if (command === 'click') { const session = sessions.get(sessionName); if (!session?.snapshot) { diff --git a/src/utils/args.ts b/src/utils/args.ts index ca228a7ac..470dc9ff0 100644 --- a/src/utils/args.ts +++ b/src/utils/args.ts @@ -184,6 +184,8 @@ Commands: scroll [amount] Scroll in direction (0-1 amount) scrollintoview Scroll until text appears (Android only) screenshot [--out path] Capture screenshot + record start [path] Start screen recording + record stop Stop screen recording session list List active sessions Flags: diff --git a/src/utils/exec.ts b/src/utils/exec.ts index cc8f9c0f7..f8935315f 100644 --- a/src/utils/exec.ts +++ b/src/utils/exec.ts @@ -21,6 +21,11 @@ export type ExecStreamOptions = ExecOptions & { onStderrChunk?: (chunk: string) => void; }; +export type ExecBackgroundResult = { + child: ReturnType; + wait: Promise; +}; + export async function runCmd( cmd: string, args: string[], @@ -223,6 +228,60 @@ export async function runCmdStreaming( }); } +export function runCmdBackground( + cmd: string, + args: string[], + options: ExecOptions = {}, +): ExecBackgroundResult { + const child = spawn(cmd, args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + + const wait = new Promise((resolve, reject) => { + child.on('error', (err) => { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + reject(new AppError('TOOL_MISSING', `${cmd} not found in PATH`, { cmd }, err)); + return; + } + reject(new AppError('COMMAND_FAILED', `Failed to run ${cmd}`, { cmd, args }, err)); + }); + child.on('close', (code) => { + const exitCode = code ?? 1; + if (exitCode !== 0 && !options.allowFailure) { + reject( + new AppError('COMMAND_FAILED', `${cmd} exited with code ${exitCode}`, { + cmd, + args, + stdout, + stderr, + exitCode, + }), + ); + return; + } + resolve({ stdout, stderr, exitCode }); + }); + }); + + return { child, wait }; +} + export function whichCmdSync(cmd: string): boolean { try { const { shell, args } = resolveWhichArgs(cmd);