2026-02-28 13:38:59 +01:00
|
|
|
// MARK: - Wire Models
|
|
|
|
|
|
|
|
|
|
enum CommandType: String, Codable {
|
|
|
|
|
case tap
|
2026-03-23 16:01:52 +01:00
|
|
|
case mouseClick
|
2026-02-28 13:38:59 +01:00
|
|
|
case longPress
|
|
|
|
|
case drag
|
2026-05-12 15:32:21 +07:00
|
|
|
case remotePress
|
2026-02-28 13:38:59 +01:00
|
|
|
case type
|
|
|
|
|
case swipe
|
2026-06-11 12:03:53 +02:00
|
|
|
case scroll
|
2026-06-25 07:38:21 +02:00
|
|
|
case desktopScroll
|
2026-02-28 13:38:59 +01:00
|
|
|
case findText
|
2026-05-19 11:36:48 +02:00
|
|
|
case querySelector
|
2026-03-25 17:13:31 +01:00
|
|
|
case readText
|
2026-02-28 13:38:59 +01:00
|
|
|
case snapshot
|
|
|
|
|
case screenshot
|
|
|
|
|
case back
|
2026-03-27 10:18:00 +01:00
|
|
|
case backInApp
|
|
|
|
|
case backSystem
|
2026-02-28 13:38:59 +01:00
|
|
|
case home
|
2026-04-01 20:34:56 +02:00
|
|
|
case rotate
|
2026-02-28 13:38:59 +01:00
|
|
|
case appSwitcher
|
2026-03-27 11:49:30 +01:00
|
|
|
case keyboardDismiss
|
2026-05-27 21:45:51 +02:00
|
|
|
case keyboardReturn
|
2026-02-28 13:38:59 +01:00
|
|
|
case alert
|
|
|
|
|
case pinch
|
2026-06-11 12:50:30 +02:00
|
|
|
case sequence
|
2026-05-22 18:01:58 +02:00
|
|
|
case rotateGesture
|
|
|
|
|
case transformGesture
|
2026-02-28 13:38:59 +01:00
|
|
|
case recordStart
|
|
|
|
|
case recordStop
|
2026-06-01 20:40:09 -05:00
|
|
|
case status
|
2026-03-24 13:38:41 +01:00
|
|
|
case uptime
|
2026-02-28 13:38:59 +01:00
|
|
|
case shutdown
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-01 18:11:04 +02:00
|
|
|
/// Runner command traits — see CONTEXT.md ("Runner command traits").
|
|
|
|
|
///
|
|
|
|
|
/// Single source of truth for how the runner classifies a command across three
|
|
|
|
|
/// independent axes, replacing the three hand-maintained switches that used to live
|
|
|
|
|
/// in RunnerTests+Lifecycle.swift (isInteractionCommand / isReadOnlyCommand /
|
|
|
|
|
/// isRunnerLifecycleCommand). The classification is load-bearing for ADR-0002 session
|
|
|
|
|
/// invalidation: `readOnly` gates the retry that nulls currentApp/currentBundleId.
|
|
|
|
|
struct CommandTraits {
|
|
|
|
|
/// Whether the command needs the foreground-guard + stabilization preflight before running.
|
|
|
|
|
let isInteraction: Bool
|
|
|
|
|
/// Whether the command is eligible for the session-invalidating retry.
|
|
|
|
|
/// `.conditional` is resolved against the request (alert is read-only only for its `get` action).
|
|
|
|
|
let readOnly: ReadOnly
|
|
|
|
|
/// Whether the command skips the app-activation preflight entirely.
|
|
|
|
|
let isLifecycle: Bool
|
|
|
|
|
|
|
|
|
|
enum ReadOnly {
|
|
|
|
|
case always
|
|
|
|
|
case never
|
|
|
|
|
/// Alert-only today. Resolved in `isReadOnlyCommand` with alert's rule (read-only for the
|
|
|
|
|
/// `get` action, mutating otherwise). A new `.conditional` command would inherit that rule
|
|
|
|
|
/// until the resolver is generalized — give it explicit handling there if its semantics differ.
|
|
|
|
|
case conditional
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extension CommandType {
|
|
|
|
|
/// The classification for this command. Exhaustive by construction: a new CommandType
|
|
|
|
|
/// cannot compile without being classified here, so commands can no longer silently drift
|
|
|
|
|
/// out of classification the way the parallel switches allowed.
|
|
|
|
|
var traits: CommandTraits {
|
|
|
|
|
switch self {
|
|
|
|
|
// Interaction commands: require the foreground-guard + stabilization preflight.
|
refactor(ios): consolidate series batching onto the sequence runner command (#768)
* refactor(ios): consolidate series batching onto the sequence runner command
Closes #767
Routes every Apple multi-press variant (plain, double-tap, hold, jitter)
and swipe series through budget-chunked sequence requests, retiring the
daemon-side tapSeries and dragSeries senders:
- Add a doubleTap step kind to the sequence allowlist on both ends,
mirroring the retired tapSeries doubleTapAt branch.
- The single doubleTap interactor sends a one-step sequence and parses
the result, surfacing step failures as errors.
- Swipe series unroll ping-pong daemon-side into per-step endpoints;
the runner's coordinate-drag path ignores durationMs exactly as the
daemon-sent (non-synthesized) dragSeries did.
- Extract runIosSequenceChunks so press and swipe share the chunking,
aggregation, and global step-index rebasing.
- Keep tapSeries/dragSeries runner handlers for wire compatibility with
older daemons, annotated like interactionFrame; remove both from the
preflight-skip allowlist (daemon never sends them) and update ADR
0005 / protocol-optimizations docs.
This also closes the latent watchdog exposure where press --count N
--interval-ms M routed to tapSeries and executed all pauses inside one
30s-watchdog main-thread block with no chunking.
Behavior note: plain tap series now use the synthesized HID tap path on
iOS non-tv (with runner-side tapAt fallback), matching the individual
tap command instead of the retired tapSeries' XCUICoordinate taps.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
* refactor(ios): drop dead series wire surface from the daemon
- Remove chunkRunnerSequenceSteps: superseded by the budget-aware
chunker; no production callers remained.
- Remove tapSeries/dragSeries from the RunnerCommand union along with
their orphaned fields (count, intervalMs, doubleTap, pauseMs,
pattern) and protocol fixtures: this type is the send surface of the
current daemon, which no longer sends either command. The Swift
runner keeps serving both for wire compatibility with older daemons.
- Retarget the ready-mutation preflight test from tapSeries to
sequence.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
* refactor(ios): remove retired series and frame wire commands entirely
Drops the runner-side wire compatibility for tapSeries, dragSeries, and
interactionFrame now that no daemon path sends them (series fuse into
sequence since this branch; interactionFrame was fused into scroll in
#760):
- Swift: delete the three handler cases, performDragSeries, runSeries
(no remaining callers), the CommandType enum cases, journal-retention
and traits entries, and the Command fields (count, intervalMs,
doubleTap, pauseMs, pattern) that existed only for them. The
never-sent synthesized dragSeries branch goes with it.
- TS: drop interactionFrame from the RunnerCommand union and
isReadOnlyRunnerCommand, and its protocol fixture.
- Update stale perf scenario labels referencing the retired commands.
Verified dead before removal: no dynamic command construction anywhere
(runner-command-recovery only echoes in-flight command ids), no
raw-string references in Swift, no docs references. Helpers shared with
live paths (synthesizedDragAt, doubleTapAt, keyboardAvoidingDragPoints,
sleepFor) all retain callers.
Compat: an old daemon paired with a runner built from these sources
gets a CommandType decode rejection; the source-fingerprint check
rebuilds a matching runner on the next session.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
---------
Co-authored-by: Claude <noreply@anthropic.com>
2026-06-11 14:08:30 +02:00
|
|
|
// keyboardReturn is the sibling of keyboardDismiss (missing from the historical switch —
|
|
|
|
|
// drift the table now prevents). .scroll is the fused frame-resolve + drag scroll; same
|
2026-06-25 07:38:21 +02:00
|
|
|
// classification as .drag. .desktopScroll is the macOS frame-resolve + wheel event sibling.
|
|
|
|
|
// .sequence is the fused multi-step gesture batch.
|
|
|
|
|
case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll,
|
2026-06-01 18:11:04 +02:00
|
|
|
.back, .backInApp, .backSystem, .rotate, .appSwitcher,
|
2026-06-11 12:50:30 +02:00
|
|
|
.keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture:
|
2026-06-01 18:11:04 +02:00
|
|
|
return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
|
|
|
|
|
|
|
|
|
|
// Read-only reads: eligible for the session-invalidating retry.
|
refactor(ios): consolidate series batching onto the sequence runner command (#768)
* refactor(ios): consolidate series batching onto the sequence runner command
Closes #767
Routes every Apple multi-press variant (plain, double-tap, hold, jitter)
and swipe series through budget-chunked sequence requests, retiring the
daemon-side tapSeries and dragSeries senders:
- Add a doubleTap step kind to the sequence allowlist on both ends,
mirroring the retired tapSeries doubleTapAt branch.
- The single doubleTap interactor sends a one-step sequence and parses
the result, surfacing step failures as errors.
- Swipe series unroll ping-pong daemon-side into per-step endpoints;
the runner's coordinate-drag path ignores durationMs exactly as the
daemon-sent (non-synthesized) dragSeries did.
- Extract runIosSequenceChunks so press and swipe share the chunking,
aggregation, and global step-index rebasing.
- Keep tapSeries/dragSeries runner handlers for wire compatibility with
older daemons, annotated like interactionFrame; remove both from the
preflight-skip allowlist (daemon never sends them) and update ADR
0005 / protocol-optimizations docs.
This also closes the latent watchdog exposure where press --count N
--interval-ms M routed to tapSeries and executed all pauses inside one
30s-watchdog main-thread block with no chunking.
Behavior note: plain tap series now use the synthesized HID tap path on
iOS non-tv (with runner-side tapAt fallback), matching the individual
tap command instead of the retired tapSeries' XCUICoordinate taps.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
* refactor(ios): drop dead series wire surface from the daemon
- Remove chunkRunnerSequenceSteps: superseded by the budget-aware
chunker; no production callers remained.
- Remove tapSeries/dragSeries from the RunnerCommand union along with
their orphaned fields (count, intervalMs, doubleTap, pauseMs,
pattern) and protocol fixtures: this type is the send surface of the
current daemon, which no longer sends either command. The Swift
runner keeps serving both for wire compatibility with older daemons.
- Retarget the ready-mutation preflight test from tapSeries to
sequence.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
* refactor(ios): remove retired series and frame wire commands entirely
Drops the runner-side wire compatibility for tapSeries, dragSeries, and
interactionFrame now that no daemon path sends them (series fuse into
sequence since this branch; interactionFrame was fused into scroll in
#760):
- Swift: delete the three handler cases, performDragSeries, runSeries
(no remaining callers), the CommandType enum cases, journal-retention
and traits entries, and the Command fields (count, intervalMs,
doubleTap, pauseMs, pattern) that existed only for them. The
never-sent synthesized dragSeries branch goes with it.
- TS: drop interactionFrame from the RunnerCommand union and
isReadOnlyRunnerCommand, and its protocol fixture.
- Update stale perf scenario labels referencing the retired commands.
Verified dead before removal: no dynamic command construction anywhere
(runner-command-recovery only echoes in-flight command ids), no
raw-string references in Swift, no docs references. Helpers shared with
live paths (synthesizedDragAt, doubleTapAt, keyboardAvoidingDragPoints,
sleepFor) all retain callers.
Compat: an old daemon paired with a runner built from these sources
gets a CommandType decode rejection; the source-fingerprint check
rebuilds a matching runner on the next session.
https://claude.ai/code/session_01VokBZWESTDgcnbYwS4DkJo
---------
Co-authored-by: Claude <noreply@anthropic.com>
2026-06-11 14:08:30 +02:00
|
|
|
case .findText, .readText, .snapshot:
|
2026-06-01 18:11:04 +02:00
|
|
|
return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
|
|
|
|
|
|
|
|
|
|
// Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
|
|
|
|
|
case .screenshot:
|
|
|
|
|
return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
|
|
|
|
|
|
|
|
|
|
// Alert is read-only only for its `get` action (resolved by isReadOnlyCommand).
|
|
|
|
|
case .alert:
|
|
|
|
|
return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
|
|
|
|
|
|
|
|
|
|
// Runner-lifecycle commands: skip the app-activation preflight.
|
|
|
|
|
case .recordStop, .uptime, .shutdown:
|
|
|
|
|
return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
|
|
|
|
|
|
2026-06-01 20:40:09 -05:00
|
|
|
case .status:
|
|
|
|
|
return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
|
|
|
|
|
|
2026-06-01 18:11:04 +02:00
|
|
|
// Normal preflight, not retried.
|
|
|
|
|
// NOTE: mouseClick stays non-interaction for now — it is macOS-only and the foreground
|
|
|
|
|
// guard interacts with bespoke macOS activation, so classifying it needs a macOS smoke
|
|
|
|
|
// check first (tracked as a follow-up). Also preserved: querySelector is NOT read-only;
|
|
|
|
|
// recordStart is NOT a lifecycle command; home/alert remain non-interaction by design.
|
|
|
|
|
case .mouseClick, .querySelector, .home, .recordStart:
|
|
|
|
|
return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-28 13:38:59 +01:00
|
|
|
struct Command: Codable {
|
|
|
|
|
let command: CommandType
|
2026-06-01 20:40:09 -05:00
|
|
|
let commandId: String?
|
|
|
|
|
let statusCommandId: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let appBundleId: String?
|
|
|
|
|
let text: String?
|
2026-05-19 11:36:48 +02:00
|
|
|
let selectorKey: String?
|
|
|
|
|
let selectorValue: String?
|
2026-05-27 21:45:51 +02:00
|
|
|
let allowNonHittableCoordinateFallback: Bool?
|
2026-03-27 10:43:52 +01:00
|
|
|
let delayMs: Int?
|
2026-05-19 11:36:48 +02:00
|
|
|
let textEntryMode: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let clearFirst: Bool?
|
|
|
|
|
let action: String?
|
|
|
|
|
let x: Double?
|
|
|
|
|
let y: Double?
|
2026-03-23 16:01:52 +01:00
|
|
|
let button: String?
|
2026-05-12 15:32:21 +07:00
|
|
|
let remoteButton: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let x2: Double?
|
|
|
|
|
let y2: Double?
|
2026-05-22 18:01:58 +02:00
|
|
|
let dx: Double?
|
|
|
|
|
let dy: Double?
|
2026-02-28 13:38:59 +01:00
|
|
|
let durationMs: Double?
|
2026-03-27 12:06:57 +01:00
|
|
|
let direction: String?
|
2026-06-11 12:03:53 +02:00
|
|
|
let amount: Double?
|
|
|
|
|
let pixels: Double?
|
2026-04-01 20:34:56 +02:00
|
|
|
let orientation: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let scale: Double?
|
2026-05-22 18:01:58 +02:00
|
|
|
let degrees: Double?
|
|
|
|
|
let velocity: Double?
|
2026-02-28 13:38:59 +01:00
|
|
|
let outPath: String?
|
|
|
|
|
let fps: Int?
|
2026-06-19 23:44:38 +07:00
|
|
|
let maxSize: Int?
|
2026-02-28 13:38:59 +01:00
|
|
|
let interactiveOnly: Bool?
|
|
|
|
|
let depth: Int?
|
|
|
|
|
let scope: String?
|
|
|
|
|
let raw: Bool?
|
2026-03-28 19:06:52 +01:00
|
|
|
let fullscreen: Bool?
|
2026-06-03 04:59:13 -07:00
|
|
|
let synthesized: Bool?
|
2026-06-11 12:50:30 +02:00
|
|
|
let steps: [SequenceStep]?
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// One allowlisted coordinate gesture step inside a fused `sequence` command.
|
|
|
|
|
/// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
|
|
|
|
|
/// INVALID_ARGS for an unknown kind instead of a generic decode failure.
|
|
|
|
|
struct SequenceStep: Codable {
|
|
|
|
|
let kind: String
|
|
|
|
|
let x: Double?
|
|
|
|
|
let y: Double?
|
|
|
|
|
let x2: Double?
|
|
|
|
|
let y2: Double?
|
|
|
|
|
let durationMs: Double?
|
|
|
|
|
let pauseMs: Double?
|
2026-06-13 09:54:53 +02:00
|
|
|
/// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the
|
|
|
|
|
/// drag-based XCUICoordinate path, matching the individual command behavior.
|
2026-06-11 12:50:30 +02:00
|
|
|
let synthesized: Bool?
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Per-step result for a `sequence` response. `ok:false` carries the failing step's
|
|
|
|
|
/// errorCode/errorMessage; execution stops at the first failed step.
|
|
|
|
|
struct SequenceStepResult: Codable {
|
|
|
|
|
let ok: Bool
|
|
|
|
|
let kind: String
|
|
|
|
|
let errorCode: String?
|
|
|
|
|
let errorMessage: String?
|
|
|
|
|
let gestureStartUptimeMs: Double?
|
|
|
|
|
let gestureEndUptimeMs: Double?
|
2026-02-28 13:38:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct Response: Codable {
|
|
|
|
|
let ok: Bool
|
|
|
|
|
let data: DataPayload?
|
|
|
|
|
let error: ErrorPayload?
|
|
|
|
|
|
|
|
|
|
init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
|
|
|
|
|
self.ok = ok
|
|
|
|
|
self.data = data
|
|
|
|
|
self.error = error
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-06-11 14:23:55 +02:00
|
|
|
extension Response {
|
|
|
|
|
// The daemon pairs this gesture-clock anchor with its own receipt time to map
|
|
|
|
|
// gesture uptimes onto wall-clock for the recording touch overlay. Error responses
|
|
|
|
|
// carry no anchor so the daemon falls back instead of pairing a stale value.
|
|
|
|
|
func stampingCurrentUptimeMs(_ value: Double) -> Response {
|
|
|
|
|
guard ok else { return self }
|
|
|
|
|
var payload = data ?? DataPayload()
|
|
|
|
|
payload.currentUptimeMs = value
|
|
|
|
|
return Response(ok: ok, data: payload, error: error)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-28 13:38:59 +01:00
|
|
|
struct DataPayload: Codable {
|
|
|
|
|
let message: String?
|
2026-03-25 17:13:31 +01:00
|
|
|
let text: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let found: Bool?
|
|
|
|
|
let items: [String]?
|
|
|
|
|
let nodes: [SnapshotNode]?
|
|
|
|
|
let truncated: Bool?
|
refactor(ios): snapshot capture plans with a structured quality verdict (#783)
* refactor(ios): snapshot capture plans with a structured quality verdict
Implements ADR 0004's explicit-strategies decision as architecture
(candidates 1+2 of the snapshot pipeline review):
- Snapshot backend seam: three adapters (recursive tree, query sweep,
private AX) behind one captureWithBackend dispatch. Each strategy
declares its chain as data (regular: tree→queries→private-ax,
compact: queries→private-ax, raw: tree→private-ax) and one plan
runner walks it under a 20s umbrella budget so chained recovery can
never stack past the 30s main-thread watchdog. Terminal policy is
per-plan: raw rethrows AX failures (diagnostics preserve errors),
interactive fails closed with runnerFatal invalidation.
- Single quality classifier: one sparsePayloadReason predicate (with
reason codes), one collapsed-leaf detector, replacing the three
divergent sparse detectors (Swift structural, daemon count==1, CLI
count<=3) that each patched a different failure shape.
- Structured snapshot quality verdict on the wire (state, backend,
reason, reasonCode, effectiveDepth, collapsedLeafIndexes): the daemon
and CLI render warnings from it instead of re-deriving degradation
from node shapes; budget starvation is no longer blamed on the app's
accessibility. Legacy runner messages and daemon-side detectors stay
behind a verdict-absent gate for mixed-version compat.
- The verdict surfaces in --json (snapshotQuality) for agents; the
generic sparse CLI hint is suppressed when a verdict explains it.
Threading the verdict exposed two more hand-copy field drops
(captureInteractionOutcomeAwareSnapshot, serializeSnapshotResult,
client response mapping) - now carried alongside warnings everywhere.
Verified live: Settings healthy (tree, no warnings), Settings compact
under load (recovered/private-ax/budget), production login (sparse
best-effort with honest warning), collapse fixture (healthy +
collapsedLeafIndexes -> @ref warning), Bluesky Home (recovered/
private-ax, 24 nodes in 2s). Full unit suite 2327 passed, fallow clean,
runner builds.
* fix(ios): correct recovered-snapshot viewport and private-AX scope semantics
Review follow-ups on the capture-plan refactor:
- The query-sweep synthetic root doubles as the daemon's viewport
(find.ts prefers on-screen matches inside nodes[0].rect), but it was
built from candidate bounds, so off-screen controls below the screen
could inflate it and win duplicate-label resolution. The root now
uses the real finite viewport, falling back to candidate bounds only
when viewport capture failed.
- The private-AX backend applied --scope as a per-node text filter,
hiding the matched container's children — diverging from regular
snapshot scope semantics and contradicting the depth-cap hint that
recommends scoped re-runs. Scope now selects the matched subtree:
descendants inherit the match and only the normal option filters
apply to them (in-bundle test covers a non-matching descendant).
Verified live on Bluesky Home: scope homeScreen returns the 52-node
subtree including non-matching descendants; compact root rect equals
the screen (0,0,402,874).
* fix(ios): fail closed on interactive AX failure, stamp fatal verdict, validate parser
Three review findings on the capture-plan terminal path:
- P1: the fail-closed guard required `best == nil`, but the query-sweep
tier always returns a synthetic-root sparse payload that sets `best` —
so an interactive recursive-tree AX serialization failure that no
backend recovered returned a sparse snapshot instead of invalidating
the cached target. Reaching the terminal already means no backend
produced a usable tree, so the sparse `best` must not suppress the
fail-closed path. Extracted the decision into a pure, unit-tested
`resolveSnapshotPlanTerminal` (closes the terminal-ordering testability
gap the architecture review flagged).
- P2: `snapshotAccessibilityUnavailable` returned a payload with no
`snapshotQuality`, leaving one planned sparse result on the
legacy-message path. It now carries a sparse/ax-rejected verdict like
every other planned snapshot, so downstream sparse handling keys off
the verdict.
- P2: `readSnapshotQualityVerdict` cast any string state/backend into the
union, so a malformed object suppressed the legacy node-shape
detectors. State and backend are now validated against their unions
(unknown → verdict-absent → legacy detectors run); an unknown
reasonCode is dropped rather than rejecting the whole verdict, so a
forward-version runner still yields a usable verdict.
Unit-covered: Swift resolveSnapshotPlanTerminal matrix + fatal-verdict
assertion; TS parser accept/reject/forward-compat. Full suite 249 files
/ 2449 tests, fallow, lint, runner build green.
2026-06-12 12:25:40 +02:00
|
|
|
let snapshotQuality: SnapshotQuality?
|
2026-03-24 13:38:41 +01:00
|
|
|
let gestureStartUptimeMs: Double?
|
|
|
|
|
let gestureEndUptimeMs: Double?
|
|
|
|
|
let x: Double?
|
|
|
|
|
let y: Double?
|
|
|
|
|
let x2: Double?
|
|
|
|
|
let y2: Double?
|
|
|
|
|
let referenceWidth: Double?
|
|
|
|
|
let referenceHeight: Double?
|
2026-06-11 14:23:55 +02:00
|
|
|
var currentUptimeMs: Double?
|
2026-06-01 20:40:09 -05:00
|
|
|
let commandId: String?
|
|
|
|
|
let lifecycleState: String?
|
|
|
|
|
let lifecycleCommand: String?
|
|
|
|
|
let lifecycleResponseOk: Bool?
|
|
|
|
|
let lifecycleResponseJson: String?
|
|
|
|
|
let lifecycleErrorCode: String?
|
|
|
|
|
let lifecycleErrorMessage: String?
|
|
|
|
|
let lifecycleErrorHint: String?
|
2026-03-27 11:49:30 +01:00
|
|
|
let visible: Bool?
|
|
|
|
|
let wasVisible: Bool?
|
|
|
|
|
let dismissed: Bool?
|
2026-04-01 20:34:56 +02:00
|
|
|
let orientation: String?
|
2026-06-03 04:59:13 -07:00
|
|
|
let gestureFallback: String?
|
|
|
|
|
let gestureFallbackMessage: String?
|
|
|
|
|
let gestureFallbackHint: String?
|
2026-06-07 11:17:27 +02:00
|
|
|
let runnerFatal: Bool?
|
|
|
|
|
let runnerFatalReason: String?
|
2026-06-11 12:50:30 +02:00
|
|
|
let completedSteps: Int?
|
|
|
|
|
let failedStepIndex: Int?
|
|
|
|
|
let sequenceResults: [SequenceStepResult]?
|
2026-02-28 13:38:59 +01:00
|
|
|
|
|
|
|
|
init(
|
|
|
|
|
message: String? = nil,
|
2026-03-25 17:13:31 +01:00
|
|
|
text: String? = nil,
|
2026-02-28 13:38:59 +01:00
|
|
|
found: Bool? = nil,
|
|
|
|
|
items: [String]? = nil,
|
|
|
|
|
nodes: [SnapshotNode]? = nil,
|
2026-03-24 13:38:41 +01:00
|
|
|
truncated: Bool? = nil,
|
refactor(ios): snapshot capture plans with a structured quality verdict (#783)
* refactor(ios): snapshot capture plans with a structured quality verdict
Implements ADR 0004's explicit-strategies decision as architecture
(candidates 1+2 of the snapshot pipeline review):
- Snapshot backend seam: three adapters (recursive tree, query sweep,
private AX) behind one captureWithBackend dispatch. Each strategy
declares its chain as data (regular: tree→queries→private-ax,
compact: queries→private-ax, raw: tree→private-ax) and one plan
runner walks it under a 20s umbrella budget so chained recovery can
never stack past the 30s main-thread watchdog. Terminal policy is
per-plan: raw rethrows AX failures (diagnostics preserve errors),
interactive fails closed with runnerFatal invalidation.
- Single quality classifier: one sparsePayloadReason predicate (with
reason codes), one collapsed-leaf detector, replacing the three
divergent sparse detectors (Swift structural, daemon count==1, CLI
count<=3) that each patched a different failure shape.
- Structured snapshot quality verdict on the wire (state, backend,
reason, reasonCode, effectiveDepth, collapsedLeafIndexes): the daemon
and CLI render warnings from it instead of re-deriving degradation
from node shapes; budget starvation is no longer blamed on the app's
accessibility. Legacy runner messages and daemon-side detectors stay
behind a verdict-absent gate for mixed-version compat.
- The verdict surfaces in --json (snapshotQuality) for agents; the
generic sparse CLI hint is suppressed when a verdict explains it.
Threading the verdict exposed two more hand-copy field drops
(captureInteractionOutcomeAwareSnapshot, serializeSnapshotResult,
client response mapping) - now carried alongside warnings everywhere.
Verified live: Settings healthy (tree, no warnings), Settings compact
under load (recovered/private-ax/budget), production login (sparse
best-effort with honest warning), collapse fixture (healthy +
collapsedLeafIndexes -> @ref warning), Bluesky Home (recovered/
private-ax, 24 nodes in 2s). Full unit suite 2327 passed, fallow clean,
runner builds.
* fix(ios): correct recovered-snapshot viewport and private-AX scope semantics
Review follow-ups on the capture-plan refactor:
- The query-sweep synthetic root doubles as the daemon's viewport
(find.ts prefers on-screen matches inside nodes[0].rect), but it was
built from candidate bounds, so off-screen controls below the screen
could inflate it and win duplicate-label resolution. The root now
uses the real finite viewport, falling back to candidate bounds only
when viewport capture failed.
- The private-AX backend applied --scope as a per-node text filter,
hiding the matched container's children — diverging from regular
snapshot scope semantics and contradicting the depth-cap hint that
recommends scoped re-runs. Scope now selects the matched subtree:
descendants inherit the match and only the normal option filters
apply to them (in-bundle test covers a non-matching descendant).
Verified live on Bluesky Home: scope homeScreen returns the 52-node
subtree including non-matching descendants; compact root rect equals
the screen (0,0,402,874).
* fix(ios): fail closed on interactive AX failure, stamp fatal verdict, validate parser
Three review findings on the capture-plan terminal path:
- P1: the fail-closed guard required `best == nil`, but the query-sweep
tier always returns a synthetic-root sparse payload that sets `best` —
so an interactive recursive-tree AX serialization failure that no
backend recovered returned a sparse snapshot instead of invalidating
the cached target. Reaching the terminal already means no backend
produced a usable tree, so the sparse `best` must not suppress the
fail-closed path. Extracted the decision into a pure, unit-tested
`resolveSnapshotPlanTerminal` (closes the terminal-ordering testability
gap the architecture review flagged).
- P2: `snapshotAccessibilityUnavailable` returned a payload with no
`snapshotQuality`, leaving one planned sparse result on the
legacy-message path. It now carries a sparse/ax-rejected verdict like
every other planned snapshot, so downstream sparse handling keys off
the verdict.
- P2: `readSnapshotQualityVerdict` cast any string state/backend into the
union, so a malformed object suppressed the legacy node-shape
detectors. State and backend are now validated against their unions
(unknown → verdict-absent → legacy detectors run); an unknown
reasonCode is dropped rather than rejecting the whole verdict, so a
forward-version runner still yields a usable verdict.
Unit-covered: Swift resolveSnapshotPlanTerminal matrix + fatal-verdict
assertion; TS parser accept/reject/forward-compat. Full suite 249 files
/ 2449 tests, fallow, lint, runner build green.
2026-06-12 12:25:40 +02:00
|
|
|
snapshotQuality: SnapshotQuality? = nil,
|
2026-03-24 13:38:41 +01:00
|
|
|
gestureStartUptimeMs: Double? = nil,
|
|
|
|
|
gestureEndUptimeMs: Double? = nil,
|
|
|
|
|
x: Double? = nil,
|
|
|
|
|
y: Double? = nil,
|
|
|
|
|
x2: Double? = nil,
|
|
|
|
|
y2: Double? = nil,
|
|
|
|
|
referenceWidth: Double? = nil,
|
|
|
|
|
referenceHeight: Double? = nil,
|
2026-03-27 11:49:30 +01:00
|
|
|
currentUptimeMs: Double? = nil,
|
2026-06-01 20:40:09 -05:00
|
|
|
commandId: String? = nil,
|
|
|
|
|
lifecycleState: String? = nil,
|
|
|
|
|
lifecycleCommand: String? = nil,
|
|
|
|
|
lifecycleResponseOk: Bool? = nil,
|
|
|
|
|
lifecycleResponseJson: String? = nil,
|
|
|
|
|
lifecycleErrorCode: String? = nil,
|
|
|
|
|
lifecycleErrorMessage: String? = nil,
|
|
|
|
|
lifecycleErrorHint: String? = nil,
|
2026-03-27 11:49:30 +01:00
|
|
|
visible: Bool? = nil,
|
|
|
|
|
wasVisible: Bool? = nil,
|
2026-04-01 20:34:56 +02:00
|
|
|
dismissed: Bool? = nil,
|
2026-06-03 04:59:13 -07:00
|
|
|
orientation: String? = nil,
|
|
|
|
|
gestureFallback: String? = nil,
|
|
|
|
|
gestureFallbackMessage: String? = nil,
|
2026-06-07 11:17:27 +02:00
|
|
|
gestureFallbackHint: String? = nil,
|
|
|
|
|
runnerFatal: Bool? = nil,
|
2026-06-11 12:50:30 +02:00
|
|
|
runnerFatalReason: String? = nil,
|
|
|
|
|
completedSteps: Int? = nil,
|
|
|
|
|
failedStepIndex: Int? = nil,
|
|
|
|
|
sequenceResults: [SequenceStepResult]? = nil
|
2026-02-28 13:38:59 +01:00
|
|
|
) {
|
|
|
|
|
self.message = message
|
2026-03-25 17:13:31 +01:00
|
|
|
self.text = text
|
2026-02-28 13:38:59 +01:00
|
|
|
self.found = found
|
|
|
|
|
self.items = items
|
|
|
|
|
self.nodes = nodes
|
|
|
|
|
self.truncated = truncated
|
refactor(ios): snapshot capture plans with a structured quality verdict (#783)
* refactor(ios): snapshot capture plans with a structured quality verdict
Implements ADR 0004's explicit-strategies decision as architecture
(candidates 1+2 of the snapshot pipeline review):
- Snapshot backend seam: three adapters (recursive tree, query sweep,
private AX) behind one captureWithBackend dispatch. Each strategy
declares its chain as data (regular: tree→queries→private-ax,
compact: queries→private-ax, raw: tree→private-ax) and one plan
runner walks it under a 20s umbrella budget so chained recovery can
never stack past the 30s main-thread watchdog. Terminal policy is
per-plan: raw rethrows AX failures (diagnostics preserve errors),
interactive fails closed with runnerFatal invalidation.
- Single quality classifier: one sparsePayloadReason predicate (with
reason codes), one collapsed-leaf detector, replacing the three
divergent sparse detectors (Swift structural, daemon count==1, CLI
count<=3) that each patched a different failure shape.
- Structured snapshot quality verdict on the wire (state, backend,
reason, reasonCode, effectiveDepth, collapsedLeafIndexes): the daemon
and CLI render warnings from it instead of re-deriving degradation
from node shapes; budget starvation is no longer blamed on the app's
accessibility. Legacy runner messages and daemon-side detectors stay
behind a verdict-absent gate for mixed-version compat.
- The verdict surfaces in --json (snapshotQuality) for agents; the
generic sparse CLI hint is suppressed when a verdict explains it.
Threading the verdict exposed two more hand-copy field drops
(captureInteractionOutcomeAwareSnapshot, serializeSnapshotResult,
client response mapping) - now carried alongside warnings everywhere.
Verified live: Settings healthy (tree, no warnings), Settings compact
under load (recovered/private-ax/budget), production login (sparse
best-effort with honest warning), collapse fixture (healthy +
collapsedLeafIndexes -> @ref warning), Bluesky Home (recovered/
private-ax, 24 nodes in 2s). Full unit suite 2327 passed, fallow clean,
runner builds.
* fix(ios): correct recovered-snapshot viewport and private-AX scope semantics
Review follow-ups on the capture-plan refactor:
- The query-sweep synthetic root doubles as the daemon's viewport
(find.ts prefers on-screen matches inside nodes[0].rect), but it was
built from candidate bounds, so off-screen controls below the screen
could inflate it and win duplicate-label resolution. The root now
uses the real finite viewport, falling back to candidate bounds only
when viewport capture failed.
- The private-AX backend applied --scope as a per-node text filter,
hiding the matched container's children — diverging from regular
snapshot scope semantics and contradicting the depth-cap hint that
recommends scoped re-runs. Scope now selects the matched subtree:
descendants inherit the match and only the normal option filters
apply to them (in-bundle test covers a non-matching descendant).
Verified live on Bluesky Home: scope homeScreen returns the 52-node
subtree including non-matching descendants; compact root rect equals
the screen (0,0,402,874).
* fix(ios): fail closed on interactive AX failure, stamp fatal verdict, validate parser
Three review findings on the capture-plan terminal path:
- P1: the fail-closed guard required `best == nil`, but the query-sweep
tier always returns a synthetic-root sparse payload that sets `best` —
so an interactive recursive-tree AX serialization failure that no
backend recovered returned a sparse snapshot instead of invalidating
the cached target. Reaching the terminal already means no backend
produced a usable tree, so the sparse `best` must not suppress the
fail-closed path. Extracted the decision into a pure, unit-tested
`resolveSnapshotPlanTerminal` (closes the terminal-ordering testability
gap the architecture review flagged).
- P2: `snapshotAccessibilityUnavailable` returned a payload with no
`snapshotQuality`, leaving one planned sparse result on the
legacy-message path. It now carries a sparse/ax-rejected verdict like
every other planned snapshot, so downstream sparse handling keys off
the verdict.
- P2: `readSnapshotQualityVerdict` cast any string state/backend into the
union, so a malformed object suppressed the legacy node-shape
detectors. State and backend are now validated against their unions
(unknown → verdict-absent → legacy detectors run); an unknown
reasonCode is dropped rather than rejecting the whole verdict, so a
forward-version runner still yields a usable verdict.
Unit-covered: Swift resolveSnapshotPlanTerminal matrix + fatal-verdict
assertion; TS parser accept/reject/forward-compat. Full suite 249 files
/ 2449 tests, fallow, lint, runner build green.
2026-06-12 12:25:40 +02:00
|
|
|
self.snapshotQuality = snapshotQuality
|
2026-03-24 13:38:41 +01:00
|
|
|
self.gestureStartUptimeMs = gestureStartUptimeMs
|
|
|
|
|
self.gestureEndUptimeMs = gestureEndUptimeMs
|
|
|
|
|
self.x = x
|
|
|
|
|
self.y = y
|
|
|
|
|
self.x2 = x2
|
|
|
|
|
self.y2 = y2
|
|
|
|
|
self.referenceWidth = referenceWidth
|
|
|
|
|
self.referenceHeight = referenceHeight
|
|
|
|
|
self.currentUptimeMs = currentUptimeMs
|
2026-06-01 20:40:09 -05:00
|
|
|
self.commandId = commandId
|
|
|
|
|
self.lifecycleState = lifecycleState
|
|
|
|
|
self.lifecycleCommand = lifecycleCommand
|
|
|
|
|
self.lifecycleResponseOk = lifecycleResponseOk
|
|
|
|
|
self.lifecycleResponseJson = lifecycleResponseJson
|
|
|
|
|
self.lifecycleErrorCode = lifecycleErrorCode
|
|
|
|
|
self.lifecycleErrorMessage = lifecycleErrorMessage
|
|
|
|
|
self.lifecycleErrorHint = lifecycleErrorHint
|
2026-03-27 11:49:30 +01:00
|
|
|
self.visible = visible
|
|
|
|
|
self.wasVisible = wasVisible
|
|
|
|
|
self.dismissed = dismissed
|
2026-04-01 20:34:56 +02:00
|
|
|
self.orientation = orientation
|
2026-06-03 04:59:13 -07:00
|
|
|
self.gestureFallback = gestureFallback
|
|
|
|
|
self.gestureFallbackMessage = gestureFallbackMessage
|
|
|
|
|
self.gestureFallbackHint = gestureFallbackHint
|
2026-06-07 11:17:27 +02:00
|
|
|
self.runnerFatal = runnerFatal
|
|
|
|
|
self.runnerFatalReason = runnerFatalReason
|
2026-06-11 12:50:30 +02:00
|
|
|
self.completedSteps = completedSteps
|
|
|
|
|
self.failedStepIndex = failedStepIndex
|
|
|
|
|
self.sequenceResults = sequenceResults
|
2026-02-28 13:38:59 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct ErrorPayload: Codable {
|
2026-03-27 11:49:30 +01:00
|
|
|
let code: String?
|
2026-02-28 13:38:59 +01:00
|
|
|
let message: String
|
2026-06-01 15:10:53 +02:00
|
|
|
let hint: String?
|
2026-03-27 11:49:30 +01:00
|
|
|
|
2026-06-01 15:10:53 +02:00
|
|
|
init(code: String? = nil, message: String, hint: String? = nil) {
|
2026-03-27 11:49:30 +01:00
|
|
|
self.code = code
|
|
|
|
|
self.message = message
|
2026-06-01 15:10:53 +02:00
|
|
|
self.hint = hint
|
2026-03-27 11:49:30 +01:00
|
|
|
}
|
2026-02-28 13:38:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct SnapshotRect: Codable {
|
|
|
|
|
let x: Double
|
|
|
|
|
let y: Double
|
|
|
|
|
let width: Double
|
|
|
|
|
let height: Double
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct SnapshotNode: Codable {
|
|
|
|
|
let index: Int
|
|
|
|
|
let type: String
|
|
|
|
|
let label: String?
|
|
|
|
|
let identifier: String?
|
|
|
|
|
let value: String?
|
|
|
|
|
let rect: SnapshotRect
|
|
|
|
|
let enabled: Bool
|
2026-05-12 15:32:21 +07:00
|
|
|
let focused: Bool?
|
2026-05-19 11:36:48 +02:00
|
|
|
let selected: Bool?
|
2026-02-28 13:38:59 +01:00
|
|
|
let hittable: Bool
|
|
|
|
|
let depth: Int
|
2026-04-01 19:00:02 +02:00
|
|
|
let parentIndex: Int?
|
|
|
|
|
let hiddenContentAbove: Bool?
|
|
|
|
|
let hiddenContentBelow: Bool?
|
2026-02-28 13:38:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct SnapshotOptions {
|
|
|
|
|
let interactiveOnly: Bool
|
|
|
|
|
let depth: Int?
|
|
|
|
|
let scope: String?
|
|
|
|
|
let raw: Bool
|
|
|
|
|
}
|