feat: add recording

This commit is contained in:
Michał Pierzchała
2026-02-02 21:33:45 +01:00
parent 701a5ea31e
commit 7bc4193a1e
6 changed files with 188 additions and 31 deletions
@@ -2,18 +2,9 @@
This is a prioritized, mobile-specific gap list based on agent-browser.
## P0 (core parity)
- Waits: wait for text, wait for element (by ref), wait for app idle.
- Alerts: detect, accept, dismiss, get text.
- Navigation: back, home, app switcher (where supported).
- Scroll into view by text for iOS (Android already has scrollintoview).
- Snapshot scopes by ref (e.g., snapshot -s "label" or use node ref as scope).
## P1 (debug and reliability)
- Highlight ref (visual debug overlay for refs).
- Record start/stop for simulator and adb (CLI wrapper).
- Trace/log capture toggle for XCTest/AX.
- Retry strategy knobs (backoff, max retries) for flaky operations.
@@ -1,22 +0,0 @@
# Recording
## iOS Simulator
Use simctl recordVideo directly for now:
```bash
xcrun simctl io booted recordVideo /tmp/recording.mov
```
Stop with Ctrl+C.
## Android Emulator/Device
Use adb screenrecord:
```bash
adb -s <serial> shell screenrecord /sdcard/recording.mp4
adb -s <serial> pull /sdcard/recording.mp4 ./recording.mp4
```
Stop with Ctrl+C.
@@ -0,0 +1,39 @@
# Video Recording
Capture device automation sessions as video for debugging, documentation, or verification
## iOS Simulator
Use `agent-device record` commands (wrapper around simctl):
```bash
# Start recording
agent-device record start ./recordings/ios.mov
# Perform actions
agent-device open App
agent-device snapshot
agent-device click @e3
agent-device close
# Stop recording
agent-device record stop
```
## Android Emulator/Device
Use `agent-device record` commands (wrapper around adb):
```bash
# Start recording
agent-device record start ./recordings/android.mp4
# Perform actions
agent-device open App
agent-device snapshot
agent-device click @e3
agent-device close
# Stop recording
agent-device record stop
```
+88
View File
@@ -16,6 +16,7 @@ import {
type RawSnapshotNode,
} from './utils/snapshot.ts';
import { runIosRunnerCommand, stopIosRunnerSession } from './platforms/ios/runner-client.ts';
import { runCmd, runCmdBackground } from './utils/exec.ts';
import { snapshotAndroid } from './platforms/android/index.ts';
type DaemonRequest = {
@@ -38,6 +39,13 @@ type SessionState = {
appName?: string;
snapshot?: SnapshotState;
actions: SessionAction[];
recording?: {
platform: 'ios' | 'android';
outPath: string;
remotePath?: string;
child: ReturnType<typeof import('node:child_process').spawn>;
wait: Promise<import('./utils/exec.ts').ExecResult>;
};
};
type SessionAction = {
@@ -492,6 +500,86 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
return { ok: true, data };
}
if (command === 'record') {
const action = (req.positionals?.[0] ?? '').toLowerCase();
if (!['start', 'stop'].includes(action)) {
return { ok: false, error: { code: 'INVALID_ARGS', message: 'record requires start|stop' } };
}
const session = sessions.get(sessionName);
const device = session?.device ?? (await resolveTargetDevice(req.flags ?? {}));
if (!session) {
await ensureDeviceReady(device);
}
const activeSession = session ?? {
name: sessionName,
device,
createdAt: Date.now(),
actions: [],
};
if (action === 'start') {
if (activeSession.recording) {
return { ok: false, error: { code: 'INVALID_ARGS', message: 'recording already in progress' } };
}
const outPath = req.positionals?.[1] ?? `./recording-${Date.now()}.mp4`;
const resolvedOut = path.resolve(outPath);
const outDir = path.dirname(resolvedOut);
if (!fs.existsSync(outDir)) {
fs.mkdirSync(outDir, { recursive: true });
}
if (device.platform === 'ios') {
if (device.kind !== 'simulator') {
return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'record is only supported on iOS simulators in v1' } };
}
const { child, wait } = runCmdBackground('xcrun', ['simctl', 'io', device.id, 'recordVideo', resolvedOut], {
allowFailure: true,
});
activeSession.recording = { platform: 'ios', outPath: resolvedOut, child, wait };
} else {
const remotePath = `/sdcard/agent-device-recording-${Date.now()}.mp4`;
const { child, wait } = runCmdBackground('adb', ['-s', device.id, 'shell', 'screenrecord', remotePath], {
allowFailure: true,
});
activeSession.recording = { platform: 'android', outPath: resolvedOut, remotePath, child, wait };
}
sessions.set(sessionName, activeSession);
recordAction(activeSession, {
command,
positionals: req.positionals ?? [],
flags: req.flags ?? {},
result: { action: 'start' },
});
return { ok: true, data: { recording: 'started', outPath } };
}
if (!activeSession.recording) {
return { ok: false, error: { code: 'INVALID_ARGS', message: 'no active recording' } };
}
const recording = activeSession.recording;
recording.child.kill('SIGINT');
try {
await recording.wait;
} catch {
// ignore
}
if (recording.platform === 'android' && recording.remotePath) {
try {
await runCmd('adb', ['-s', device.id, 'pull', recording.remotePath, recording.outPath], { allowFailure: true });
await runCmd('adb', ['-s', device.id, 'shell', 'rm', '-f', recording.remotePath], { allowFailure: true });
} catch {
// ignore
}
}
activeSession.recording = undefined;
recordAction(activeSession, {
command,
positionals: req.positionals ?? [],
flags: req.flags ?? {},
result: { action: 'stop', outPath: recording.outPath },
});
return { ok: true, data: { recording: 'stopped', outPath: recording.outPath } };
}
if (command === 'click') {
const session = sessions.get(sessionName);
if (!session?.snapshot) {
+2
View File
@@ -184,6 +184,8 @@ Commands:
scroll <direction> [amount] Scroll in direction (0-1 amount)
scrollintoview <text> Scroll until text appears (Android only)
screenshot [--out path] Capture screenshot
record start [path] Start screen recording
record stop Stop screen recording
session list List active sessions
Flags:
+59
View File
@@ -21,6 +21,11 @@ export type ExecStreamOptions = ExecOptions & {
onStderrChunk?: (chunk: string) => void;
};
export type ExecBackgroundResult = {
child: ReturnType<typeof spawn>;
wait: Promise<ExecResult>;
};
export async function runCmd(
cmd: string,
args: string[],
@@ -223,6 +228,60 @@ export async function runCmdStreaming(
});
}
export function runCmdBackground(
cmd: string,
args: string[],
options: ExecOptions = {},
): ExecBackgroundResult {
const child = spawn(cmd, args, {
cwd: options.cwd,
env: options.env,
stdio: ['ignore', 'pipe', 'pipe'],
});
let stdout = '';
let stderr = '';
child.stdout.setEncoding('utf8');
child.stderr.setEncoding('utf8');
child.stdout.on('data', (chunk) => {
stdout += chunk;
});
child.stderr.on('data', (chunk) => {
stderr += chunk;
});
const wait = new Promise<ExecResult>((resolve, reject) => {
child.on('error', (err) => {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
reject(new AppError('TOOL_MISSING', `${cmd} not found in PATH`, { cmd }, err));
return;
}
reject(new AppError('COMMAND_FAILED', `Failed to run ${cmd}`, { cmd, args }, err));
});
child.on('close', (code) => {
const exitCode = code ?? 1;
if (exitCode !== 0 && !options.allowFailure) {
reject(
new AppError('COMMAND_FAILED', `${cmd} exited with code ${exitCode}`, {
cmd,
args,
stdout,
stderr,
exitCode,
}),
);
return;
}
resolve({ stdout, stderr, exitCode });
});
});
return { child, wait };
}
export function whichCmdSync(cmd: string): boolean {
try {
const { shell, args } = resolveWhichArgs(cmd);