From ac50381270a1084c3e99bcfe5acf8d1be5c817b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 27 Mar 2026 15:27:56 +0100 Subject: [PATCH] docs: clarify iOS runner protocol contract (#300) --- ios-runner/README.md | 19 ++++- ios-runner/RUNNER_PROTOCOL.md | 73 ++++++++++++++++ .../ios/__tests__/runner-client.test.ts | 85 +++++++++++++++++++ 3 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 ios-runner/RUNNER_PROTOCOL.md diff --git a/ios-runner/README.md b/ios-runner/README.md index dd02bddaf..3e6dc7bd1 100644 --- a/ios-runner/README.md +++ b/ios-runner/README.md @@ -1,16 +1,25 @@ # agent-device iOS Runner -This folder is reserved for the lightweight XCUITest runner used to provide element-level automation on iOS. +This folder contains the lightweight XCUITest runner used to provide element-level automation for Apple-family targets. ## Intent + - Provide a minimal XCTest target that exposes UI automation over a small HTTP server. - Allow local builds via `xcodebuild` and caching for faster subsequent runs. - Support simulator prebuilds where compatible. ## Status -Planned for the automation layer. See `docs/ios-automation.md` and `docs/ios-runner-protocol.md`. + +Current internal runner for iOS, tvOS, and macOS desktop automation. + +Protocol and maintenance references: + +- Protocol overview: [`RUNNER_PROTOCOL.md`](RUNNER_PROTOCOL.md) +- TypeScript client: [`../src/platforms/ios/runner-client.ts`](../src/platforms/ios/runner-client.ts) +- Swift wire models: [`AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift`](AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift) ## UITest Runner File Map + `AgentDeviceRunnerUITests/RunnerTests` is split into focused files to reduce context size for contributors and LLM agents. - `RunnerTests.swift`: shared state/constants, `setUp()`, and `testCommand()` entry flow. @@ -23,3 +32,9 @@ Planned for the automation layer. See `docs/ios-automation.md` and `docs/ios-run - `RunnerTests+Snapshot.swift`: fast/raw snapshot builders and include/filter helpers. - `RunnerTests+SystemModal.swift`: SpringBoard/system modal detection and modal snapshot shaping. - `RunnerTests+ScreenRecorder.swift`: nested `ScreenRecorder` implementation. + +## Protocol Notes + +- The daemon posts JSON commands to `POST /command` on the runner's local HTTP listener. +- The runner responds with a JSON envelope shaped as `{ ok, data?, error? }`. +- The protocol is internal to `agent-device`; when adding or renaming commands, update both wire models and the protocol tests/docs in the same change. diff --git a/ios-runner/RUNNER_PROTOCOL.md b/ios-runner/RUNNER_PROTOCOL.md new file mode 100644 index 000000000..e5884c70e --- /dev/null +++ b/ios-runner/RUNNER_PROTOCOL.md @@ -0,0 +1,73 @@ +# iOS Runner Protocol + +The Apple runner speaks a small internal HTTP+JSON protocol between the TypeScript daemon and the XCUITest host. This protocol is a maintainer document, not part of the public user docs, but it should stay explicit so the TypeScript and Swift sides do not drift. + +## Transport + +- Endpoint: `POST /command` +- Content type: `application/json` +- Request body: one JSON command object +- Response body: one JSON envelope + +The daemon probes `http://127.0.0.1:/command` for simulator and desktop flows, and can use a tunneled device address for physical iOS/tvOS devices before falling back to localhost. + +## Request Shape + +Every request includes a `command` field. Additional fields depend on the command family. + +Examples: + +```json +{ "command": "tap", "x": 120, "y": 240 } +``` + +```json +{ + "command": "snapshot", + "interactiveOnly": true, + "compact": true, + "depth": 2, + "scope": "app", + "raw": false +} +``` + +```json +{ "command": "recordStart", "outPath": "/tmp/demo.mp4", "fps": 30 } +``` + +The current command names are defined in: + +- [`../src/platforms/ios/runner-client.ts`](../src/platforms/ios/runner-client.ts) +- [`AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift`](AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift) + +## Response Shape + +Successful and failed responses use the same top-level envelope: + +```json +{ + "ok": true, + "data": { + "message": "ok" + } +} +``` + +```json +{ + "ok": false, + "error": { + "code": "UNSUPPORTED_OPERATION", + "message": "Unable to dismiss the iOS keyboard without a native dismiss gesture or control" + } +} +``` + +`data` is command-specific. Common fields include snapshot nodes, text lookup results, gesture timing, visibility metadata, and screenshot or recording output details. + +## Maintenance Rules + +- Treat the TypeScript and Swift wire models as a single contract. +- When adding, removing, or renaming a command, update the protocol fixtures/tests in the same change. +- Keep this file focused on the actual wire shape rather than implementation details of command execution. diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index 485c1bdba..3b563091c 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import type { DeviceInfo } from '../../../utils/device.ts'; import { AppError } from '../../../utils/errors.ts'; +import type { RunnerCommand } from '../runner-client.ts'; import { assertSafeDerivedCleanup, isRetryableRunnerError, @@ -69,6 +70,50 @@ const macOsDevice: DeviceInfo = { booted: true, }; +const runnerProtocolCommandFixtures: Record = { + tap: { command: 'tap', x: 120, y: 240 }, + mouseClick: { command: 'mouseClick', x: 120, y: 240, button: 'secondary' }, + tapSeries: { command: 'tapSeries', x: 120, y: 240, count: 2, intervalMs: 80 }, + longPress: { command: 'longPress', x: 120, y: 240, durationMs: 750 }, + interactionFrame: { command: 'interactionFrame' }, + drag: { command: 'drag', x: 120, y: 240, x2: 300, y2: 420, durationMs: 400 }, + dragSeries: { + command: 'dragSeries', + x: 120, + y: 240, + x2: 300, + y2: 420, + count: 2, + pauseMs: 100, + pattern: 'ping-pong', + }, + type: { command: 'type', text: 'hello', delayMs: 20, clearFirst: true }, + swipe: { command: 'swipe', direction: 'down', durationMs: 250 }, + findText: { command: 'findText', text: 'Settings' }, + readText: { command: 'readText' }, + snapshot: { + command: 'snapshot', + interactiveOnly: true, + compact: true, + depth: 2, + scope: 'app', + raw: false, + }, + screenshot: { command: 'screenshot', outPath: '/tmp/runner-screenshot.png' }, + back: { command: 'back' }, + backInApp: { command: 'backInApp' }, + backSystem: { command: 'backSystem' }, + home: { command: 'home' }, + appSwitcher: { command: 'appSwitcher' }, + keyboardDismiss: { command: 'keyboardDismiss' }, + alert: { command: 'alert', action: 'accept' }, + pinch: { command: 'pinch', scale: 0.5 }, + recordStart: { command: 'recordStart', outPath: '/tmp/runner-recording.mp4', fps: 30 }, + recordStop: { command: 'recordStop' }, + uptime: { command: 'uptime' }, + shutdown: { command: 'shutdown' }, +}; + async function makeTmpDir(t: test.TestContext): Promise { const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'agent-device-xctestrun-')); t.after(async () => { @@ -81,6 +126,46 @@ test('resolveRunnerDestination uses simulator destination for simulators', () => assert.equal(resolveRunnerDestination(iosSimulator), 'platform=iOS Simulator,id=sim-1'); }); +test('runner protocol fixtures cover every runner command with JSON-safe samples', () => { + const commands = Object.keys(runnerProtocolCommandFixtures).sort(); + assert.deepEqual(commands, [ + 'alert', + 'appSwitcher', + 'back', + 'backInApp', + 'backSystem', + 'drag', + 'dragSeries', + 'findText', + 'home', + 'interactionFrame', + 'keyboardDismiss', + 'longPress', + 'mouseClick', + 'pinch', + 'readText', + 'recordStart', + 'recordStop', + 'screenshot', + 'shutdown', + 'snapshot', + 'swipe', + 'tap', + 'tapSeries', + 'type', + 'uptime', + ]); + + const roundTrip = JSON.parse(JSON.stringify(runnerProtocolCommandFixtures)) as Record< + string, + Record + >; + assert.equal(roundTrip.tap.command, 'tap'); + assert.equal(roundTrip.mouseClick.button, 'secondary'); + assert.equal(roundTrip.snapshot.scope, 'app'); + assert.equal(roundTrip.recordStart.fps, 30); +}); + test('resolveRunnerDestination uses device destination for physical devices', () => { assert.equal(resolveRunnerDestination(iosDevice), 'platform=iOS,id=00008110-000E12341234002E'); });