feat(ios): route Simulator snapshots through AX bridge (#2279)

* feat(ios): route simulator snapshots through AX bridge

* fix(ios): preserve snapshot fallback lineage

* fix(ios): keep regular depth in presentation

* perf(ios): reuse process-verified snapshot targets

* fix(ios): refuse snapshots beneath another foreground owner

* test(ios): bound native setup and isolate runner reset

* test(ios): synchronize helper crashes with request dispatch

* test(ios): exercise foreground guards through native capture

* chore(gates): run native snapshot ownership regression on iOS CI
This commit is contained in:
Michał Pierzchała
2026-09-05 23:05:58 +02:00
committed by GitHub
parent f1d4efe059
commit cf83afb9c9
34 changed files with 1280 additions and 73 deletions
+1
View File
@@ -142,6 +142,7 @@ jobs:
exit 0
fi
pnpm build
pnpm exec vitest run packages/platform-apple/src/snapshot-source/native-runtime.test.ts
pnpm check:package -- --verify-snapshot-bridge-preparation
- name: Run targeted iOS runner XCTest regressions
+1 -1
View File
@@ -141,7 +141,7 @@ The same session and evidence model works at every step: the agent explores the
## How it works
`agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux.
`agent-device` keeps device state in sessions. It uses a local accessibility bridge for iOS Simulator snapshots and XCTest for iOS interactions, physical iOS, and tvOS; ADB and the snapshot helper on Android; HDC and ArkUI `uitest` on HarmonyOS; Vega CLI/VDA on the Vega Virtual Device; a local helper on macOS; and AT-SPI on Linux.
Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
+10
View File
@@ -37,3 +37,13 @@ Objective-C runtime, invokes dynamically discovered selectors, and contains
Objective-C shim for those operations, adding another native boundary. Keeping
the guest in Objective-C also allows direct lazy compilation with `clang`
without an Xcode project or Swift module for private headers.
## Foreground ownership
The reader checks AXRuntime's primary foreground application before and after
acquisition. If the target is covered by system UI, ownership is unavailable,
or the owner changes during capture, it returns a typed failure without the
app tree. The existing route then uses XCTest, which owns system-modal
resolution. Secondary owners such as the return-to-app status-bar control do
not replace the native primary owner. The route's generation circuit remains
disabled after fallback until that app relaunches.
@@ -61,6 +61,11 @@ typedef bool (*AutomationEnabledFn)(void);
+ (nullable XCAccessibilityElement *)elementWithProcessIdentifier:(pid_t)pid;
@end
@protocol AXForegroundElement <NSObject>
+ (nullable id<AXForegroundElement>)primaryApp;
- (pid_t)pid;
@end
static NSNumber *finiteNumber(double value)
{
return isfinite(value) ? @(value) : nil;
@@ -168,6 +173,14 @@ static void finishRequestWatchdog(dispatch_source_t watchdog, SnapshotWatchdogSt
return _automationEnabled != NULL && _automationEnabled();
}
- (BOOL)isPrimaryForegroundProcess:(pid_t)pid
{
Class<AXForegroundElement> elementClass = (Class<AXForegroundElement>)objc_lookUpClass("AXElement");
if (![elementClass respondsToSelector:@selector(primaryApp)]) return NO;
id<AXForegroundElement> application = [elementClass primaryApp];
return [application respondsToSelector:@selector(pid)] && [application pid] == pid;
}
- (nullable id)jsonValue:(id)value name:(NSString *)name
{
if (!value || value == [NSNull null]) return nil;
@@ -299,7 +312,17 @@ static void finishRequestWatchdog(dispatch_source_t watchdog, SnapshotWatchdogSt
NSError *runtimeError = nil;
id snapshot = nil;
@try {
if (![self isPrimaryForegroundProcess:pid]) {
if (error) *error = failureResponse(requestId, @"unsupported", @"foreground-owner-unverified", @"target app is not the primary foreground accessibility owner");
finishRequestWatchdog(watchdog, watchdogState);
return nil;
}
snapshot = [_framework userTestingSnapshotForElement:(__bridge id)raw options:options error:&runtimeError];
if (![self isPrimaryForegroundProcess:pid]) {
if (error) *error = failureResponse(requestId, @"unsupported", @"foreground-owner-changed", @"foreground accessibility ownership changed during acquisition");
finishRequestWatchdog(watchdog, watchdogState);
return nil;
}
} @catch (NSException *exception) {
if (error) *error = failureResponse(requestId, @"reader_unavailable", @"private-api-exception", exception.reason ?: @"AX snapshot raised an exception");
finishRequestWatchdog(watchdog, watchdogState);
+19 -15
View File
@@ -2,17 +2,18 @@
## Status
Accepted. Amended after iOS snapshot capture was simplified to two public modes:
regular interactive snapshots and raw diagnostic snapshots.
Accepted. Amended after local iOS Simulator acquisition moved to the host AX bridge while the
public surface remained two modes: regular interactive snapshots and raw diagnostic snapshots.
The runner owns capture-plan acquisition and backend fallback. Host-side iOS validation, semantic
presentation, and publication are owned by `@agent-device/capture-kit`; structured snapshot quality
verdicts make degraded or recovered output observable end to end.
The Apple platform runtime owns acquisition routing and its generation-scoped XCTest fallback.
Host-side iOS validation, semantic presentation, and publication are owned by
`@agent-device/capture-kit`; structured snapshot quality verdicts and fallback warnings make
degraded or recovered output observable end to end.
## Context
Agent Device exposes iOS UI state through snapshots produced by the long-lived XCTest runner. The
runner has two durable snapshot needs:
Agent Device exposes iOS UI state through host AX acquisition on local Simulators and the long-lived
XCTest runner everywhere else. The snapshot surface has two durable needs:
- agent-facing regular context, where the important contract is the effective user-visible UI,
fixed controls such as tab bars, and scroll-hidden hints for content outside visible scroll
@@ -35,8 +36,13 @@ predictable.
## Decision
Keep XCTest as the default iOS automation runner and split iOS snapshot capture into explicit
strategies:
Keep XCTest as the iOS automation runner. Route eligible local iOS Simulator snapshots through the
host AX bridge, present them once through the shared TypeScript engine, and use one typed XCTest
fallback when bridge acquisition or presentation fails. Disable the bridge for that app generation
after fallback; a new app generation re-enables it. Physical devices, providers, custom-action
captures, and interactions remain on their existing owners.
Keep the two public snapshot strategies explicit:
- **Regular visible strategy**: use recursive XCTest snapshots, emit the effective user-visible
tree plus visible ancestors and scroll-hidden hints, and fall back through the capture plan when
@@ -51,12 +57,10 @@ strategies:
carry the response, fail explicitly instead of silently truncating the tree at a hard node count.
If XCTest reports a real AX serialization failure, preserve that error instead of pretending the
UI is empty.
- **Future AX-service strategy**: treat Bluesky-class failures as evidence that XCTest is
not a complete semantic snapshot backend. A robust semantic fix should add a host-side simulator
accessibility backend, similar in role to existing simulator accessibility inspection tools,
and acquire its output as `RawAXNode` values. Every backend crosses the same
`SnapshotPresentation` construction boundary before producing wire-facing `PresentedNode` values.
That backend can be simulator-only; physical devices should use an equivalent non-XCTest semantic
- **Host AX strategy**: acquire local Simulator trees as raw facts through the bounded host bridge.
Every result crosses the same presentation boundary before publication. XCTest fallback carries
explicit source residue, and comparisons require matching producer, intent, app generation,
presentation key, and residue. Physical devices should use an equivalent non-XCTest semantic
backend only if Apple exposes a supported channel.
The daemon should make degraded output observable. If an iOS interactive snapshot contains only the
@@ -90,6 +90,16 @@ export function areIosSnapshotComparisonIdentitiesEqual(
);
}
export function iosSnapshotComparisonIdentityKey(identity: IosSnapshotComparisonIdentity): string {
return JSON.stringify({
producer: identity.producer,
intent: identity.intent,
lineage: identity.lineage,
presentationKey: identity.presentationKey,
residue: identity.residue.map(residueIdentity).sort(),
});
}
export function buildIosSnapshotComparisonIdentity(
input: IosSnapshotInput,
request: IosSnapshotRequest,
@@ -180,6 +190,8 @@ function residueIdentity(residue: IosAcquisitionResidue): string {
expected: residue.expected,
observed: residue.observed,
});
case 'unknown-generation':
return JSON.stringify({ kind: residue.kind, captureId: residue.captureId });
case 'unavailable-fact':
return JSON.stringify({ kind: residue.kind, fact: residue.fact });
case 'fallback-source':
+6 -2
View File
@@ -9,8 +9,9 @@ import type { SessionSurface } from './session-surface.ts';
import type { BackendSnapshotResult } from './snapshot-types.ts';
import type { RunnerLogicalLeaseContext } from './runner-lease-context.ts';
import type {
IosProviderAcquisitionProducer,
IosAcquisitionProducer,
IosSnapshotAcquisitionFacts,
IosSnapshotComparisonIdentity,
} from './ios-snapshot.ts';
import type {
RawSnapshotNode,
@@ -182,6 +183,8 @@ export type SnapshotOptions = BaseSnapshotOptions & {
includeRects?: boolean;
includeHiddenContentHints?: boolean;
surface?: SessionSurface;
/** Internal capture purpose; action outcomes always require the full tree. */
acquisitionIntent?: 'full' | 'surface-observation';
};
/**
@@ -251,11 +254,12 @@ export type KeyboardEnterResult =
*/
export type SnapshotResult = Omit<BackendSnapshotResult, 'backend' | 'nodes'> & {
nodes?: RawSnapshotNode[];
comparisonIdentity?: IosSnapshotComparisonIdentity;
} & SnapshotProvenance;
export type SnapshotRuntimeAcquiredResult = Readonly<{
stage: 'acquired';
acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosProviderAcquisitionProducer }>;
acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosAcquisitionProducer }>;
}>;
export type SnapshotRuntimeResult = SnapshotResult | SnapshotRuntimeAcquiredResult;
+4
View File
@@ -142,6 +142,10 @@ export type IosAcquisitionResidue =
expected?: IosSnapshotGeneration;
observed?: IosSnapshotGeneration;
}>
| Readonly<{
kind: 'unknown-generation';
captureId: string;
}>
| Readonly<{
kind: 'unavailable-fact';
fact: IosSnapshotFact;
+6 -1
View File
@@ -171,7 +171,10 @@ export type SnapshotNode = RawSnapshotNode & {
* snapshot-provenance.test.ts).
*/
export type SnapshotProvenance =
| { backend: 'xctest'; producer: 'apple-runner' | 'appium-source' | 'limrun-ios-tree' }
| {
backend: 'xctest';
producer: 'apple-runner' | 'simulator-ax-bridge' | 'appium-source' | 'limrun-ios-tree';
}
| { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' }
| { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' }
| { backend: 'macos-helper'; producer: 'macos-helper' }
@@ -247,6 +250,8 @@ export type SnapshotState = {
snapshotQuality?: SnapshotQualityVerdict;
comparisonSafe?: boolean;
presentationKey?: string;
/** Opaque equality key for iOS acquisition and presentation lineage. */
comparisonKey?: string;
/**
* Android: the capture is an occluding system surface (notification shade, quick settings)
* rather than app content. Consumers that surface this tree to the agent must disclose the
@@ -14,11 +14,13 @@ import type {
PlatformRuntimeOperations,
} from '@agent-device/contracts/platform-runtime-operations';
import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device';
import type { AppleSnapshotRoute } from './snapshot-route.ts';
/** Apple-owned selection between app snapshots and explicit macOS surface snapshots. */
export function bindAppleSnapshotRuntime(
host: PlatformRuntimeHost,
request: Readonly<{ device: DeviceInfo; signal: AbortSignal }>,
route?: AppleSnapshotRoute,
): SnapshotRuntimeOperation {
const appSnapshot = bindLocalSnapshotInteractor({
device: request.device,
@@ -37,7 +39,14 @@ export function bindAppleSnapshotRuntime(
captureSnapshotSignal(request.signal, input),
);
}
return await appSnapshot.captureSnapshot(input);
if (!route) return await appSnapshot.captureSnapshot(input);
const signal = captureSnapshotSignal(request.signal, input);
return await route.capture(
request.device,
input,
signal,
async (fallbackInput) => await appSnapshot.captureSnapshot(fallbackInput),
);
};
return Object.freeze({
captureSnapshot,
+13 -5
View File
@@ -70,6 +70,7 @@ import {
bindAppleFindTextRuntime,
bindAppleSnapshotRuntime,
} from './runtime-snapshot.ts';
import { createAppleSnapshotRoute } from './snapshot-route.ts';
const owner = localRuntimeOwner('apple');
const available = Object.freeze({ available: true } as const);
@@ -268,6 +269,7 @@ function appleFocusFact(device: DeviceInfo): RuntimeOperationFact {
export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformRuntimeOwner {
const appLogs = createAppleAppLogRuntime(host);
const snapshotRoute = createAppleSnapshotRoute(host);
const inspectFacts = async (device: DeviceInfo) => {
const logs = await appLogs.inspectFacts(device);
const deployment = appleAppDeploymentFacts(device);
@@ -375,10 +377,14 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR
}),
),
...whenAdmitted(facts.operations.captureSnapshot, () =>
bindAppleSnapshotRuntime(host, {
device: request.device,
signal: request.scope.signal,
}),
bindAppleSnapshotRuntime(
host,
{
device: request.device,
signal: request.scope.signal,
},
snapshotRoute,
),
),
...whenAdmitted(facts.operations.captureScreenshot, () =>
bindLocalScreenshotInteractor({
@@ -492,7 +498,9 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR
[Symbol.asyncDispose]: async () => await logs[Symbol.asyncDispose](),
}) satisfies DeviceBinding<PlatformRuntimeOperations>;
},
shutdown: async () => await appLogs.shutdown(),
shutdown: async () => {
await Promise.all([appLogs.shutdown(), snapshotRoute.shutdown()]);
},
});
}
@@ -0,0 +1,14 @@
import { runAppleToolCommand } from './core/tool-provider.ts';
export async function readSnapshotTargetProcessStartTime(
pid: number,
options: { signal?: AbortSignal; timeoutMs: number },
): Promise<string | null> {
const result = await runAppleToolCommand('ps', ['-p', String(pid), '-o', 'lstart='], {
allowFailure: true,
signal: options.signal,
timeoutMs: options.timeoutMs,
});
if (result.exitCode !== 0) return null;
return result.stdout.trim() || null;
}
@@ -0,0 +1,240 @@
import { expect, test, vi } from 'vitest';
import type { DeviceInfo } from '@agent-device/kernel/device';
import { areIosSnapshotComparisonIdentitiesEqual } from '@agent-device/capture-kit/ios-snapshot-planning';
import { platformRuntimeHostFixture } from './runtime.fixtures.ts';
import { createAppleSnapshotRoute } from './snapshot-route.ts';
import type { SimulatorSnapshotSource, SnapshotSourceOutcome } from './snapshot-source-facade.ts';
const ios = {
platform: 'apple',
appleOs: 'ios',
id: 'ios-1',
name: 'iPhone',
kind: 'simulator',
target: 'mobile',
booted: true,
} as const satisfies DeviceInfo;
const target = {
udid: ios.id,
runtime: 'iOS 26.0',
pid: 42,
generation: '42:launch-a',
targetId: `${ios.id}:com.example.app`,
processStartTime: 'target-start',
} as const;
const input = { options: { appBundleId: 'com.example.app' } } as const;
test('eligible simulator capture publishes bridge acquisition without touching XCTest', async () => {
const acquired = bridgeAcquisition();
const source = sourceReturning(acquired);
const presentIosAcquisition = vi.fn(async () => ({
backend: 'xctest' as const,
producer: 'simulator-ax-bridge' as const,
nodes: [{ index: 0, type: 'Application' }],
}));
const fallback = vi.fn(async () => runnerResult());
const route = createAppleSnapshotRoute(
{
...platformRuntimeHostFixture(),
snapshot: { captureSurface: vi.fn(), presentIosAcquisition },
},
{ source, resolveTarget: vi.fn(async () => target) },
);
await expect(route.capture(ios, input, signal(), fallback)).resolves.toMatchObject({
producer: 'simulator-ax-bridge',
});
expect(presentIosAcquisition).toHaveBeenCalledWith(acquired, input.options);
expect(fallback).not.toHaveBeenCalled();
});
test('typed bridge failure falls back once and disables retries for that app generation', async () => {
const source = sourceReturning({
stage: 'failed',
failure: { kind: 'transport-failure', code: 'bridge-disconnected' },
});
const fallback = vi.fn(async () => runnerResult());
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source,
resolveTarget: vi.fn(async () => target),
});
const first = await route.capture(ios, input, signal(), fallback);
const second = await route.capture(ios, input, signal(), fallback);
expect(source.acquire).toHaveBeenCalledOnce();
expect(fallback).toHaveBeenCalledTimes(2);
expect(first.warnings).toEqual([
'Simulator AX snapshot unavailable (bridge-disconnected); used XCTest for this app generation.',
]);
expect(first.comparisonIdentity).toMatchObject({
producer: 'apple-runner',
lineage: { generation: target.generation },
residue: [{ kind: 'fallback-source', producer: 'apple-runner' }],
});
expect(second.comparisonIdentity).toMatchObject({
producer: 'apple-runner',
lineage: { generation: target.generation },
});
});
test('a new app generation re-enables the bridge', async () => {
const source = sourceReturning({
stage: 'failed',
failure: { kind: 'transport-failure', code: 'bridge-disconnected' },
});
const resolveTarget = vi
.fn()
.mockResolvedValueOnce(target)
.mockResolvedValueOnce(target)
.mockResolvedValueOnce({ ...target, pid: 84, generation: '84:launch-b' });
const fallback = vi.fn(async () => runnerResult());
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source,
resolveTarget,
});
await route.capture(ios, input, signal(), fallback);
await route.capture(ios, input, signal(), fallback);
await route.capture(ios, input, signal(), fallback);
expect(source.acquire).toHaveBeenCalledTimes(2);
});
test('stale bridge acquisition resolves the current generation before XCTest fallback', async () => {
const currentTarget = { ...target, pid: 84, generation: '84:launch-b' };
const source = sourceReturning({
stage: 'failed',
failure: { kind: 'stale-target', code: 'target-generation-changed' },
});
const resolveTarget = vi.fn().mockResolvedValueOnce(target).mockResolvedValueOnce(currentTarget);
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source,
resolveTarget,
});
const result = await route.capture(ios, input, signal(), async () => runnerResult());
expect(resolveTarget).toHaveBeenCalledTimes(2);
expect(resolveTarget).toHaveBeenLastCalledWith(
ios,
input.options.appBundleId,
expect.any(AbortSignal),
'refresh',
);
expect(result.comparisonIdentity?.lineage).toEqual({
targetId: currentTarget.targetId,
generation: currentTarget.generation,
});
});
test('target-resolution fallback remains incomparable with a bridge publication', async () => {
const source = sourceReturning(bridgeAcquisition());
const fallback = vi.fn(async () => runnerResult());
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source,
resolveTarget: vi.fn(async () => {
throw new Error('launch job unavailable');
}),
});
const result = await route.capture(ios, input, signal(), fallback);
expect(source.acquire).not.toHaveBeenCalled();
expect(result.comparisonIdentity).toMatchObject({
producer: 'apple-runner',
lineage: { targetId: target.targetId },
residue: [
{ kind: 'unknown-generation', captureId: expect.any(String) },
{ kind: 'fallback-source', producer: 'apple-runner' },
],
});
});
test('two target-resolution fallbacks cannot share comparison identity', async () => {
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source: sourceReturning(bridgeAcquisition()),
resolveTarget: vi.fn(async () => {
throw new Error('launch job unavailable');
}),
});
const first = await route.capture(ios, input, signal(), async () => runnerResult());
const second = await route.capture(ios, input, signal(), async () => runnerResult());
expect(first.comparisonIdentity).toBeDefined();
expect(second.comparisonIdentity).toBeDefined();
expect(
areIosSnapshotComparisonIdentitiesEqual(first.comparisonIdentity!, second.comparisonIdentity!),
).toBe(false);
});
test('runtime shutdown closes the process-owned bridge source', async () => {
const source = sourceReturning(bridgeAcquisition());
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), { source });
await route.shutdown();
expect(source.close).toHaveBeenCalledOnce();
});
test('cancelled acquisition does not start a fallback after the request aborts', async () => {
const controller = new AbortController();
const source = sourceReturning({
stage: 'failed',
failure: { kind: 'cancelled', code: 'abort-signal' },
});
vi.mocked(source.acquire).mockImplementation(async () => {
controller.abort(new DOMException('request ended', 'AbortError'));
return { stage: 'failed', failure: { kind: 'cancelled', code: 'abort-signal' } };
});
const fallback = vi.fn(async () => runnerResult());
const route = createAppleSnapshotRoute(platformRuntimeHostFixture(), {
source,
resolveTarget: vi.fn(async () => target),
});
await expect(route.capture(ios, input, controller.signal, fallback)).rejects.toThrow(
'request ended',
);
expect(fallback).not.toHaveBeenCalled();
});
function bridgeAcquisition(): Extract<SnapshotSourceOutcome, { stage: 'acquired' }> {
return {
stage: 'acquired',
acquisition: {
producer: 'simulator-ax-bridge',
intent: 'full',
hint: {
projection: 'regular',
rawTraversalDepth: null,
regularPresentedDepth: null,
interactiveOnly: false,
customActions: false,
acquisitionIntent: 'full',
},
nodes: [{ index: 0, type: 'Application' }],
truncated: false,
viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 200 } },
lineage: { targetId: target.targetId, generation: target.generation },
residue: [],
},
};
}
function sourceReturning(
outcome: Awaited<ReturnType<SimulatorSnapshotSource['acquire']>>,
): SimulatorSnapshotSource {
return { acquire: vi.fn(async () => outcome), close: vi.fn(async () => {}) };
}
function runnerResult() {
return { backend: 'xctest' as const, producer: 'apple-runner' as const, nodes: [] };
}
function signal(): AbortSignal {
return new AbortController().signal;
}
@@ -0,0 +1,287 @@
import { randomUUID } from 'node:crypto';
import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations';
import type {
CaptureSnapshotInput,
SnapshotResult,
SnapshotRuntimeAcquiredResult,
} from '@agent-device/contracts/snapshot-runtime';
import type {
IosAcquisitionResidue,
IosSnapshotComparisonIdentity,
IosSnapshotLineage,
} from '@agent-device/contracts/ios-snapshot';
import {
buildIosSnapshotPresentationKey,
createIosSnapshotRequest,
deriveIosCaptureHint,
} from '@agent-device/capture-kit/ios-snapshot-planning';
import { emitDiagnostic, withDiagnosticTimer } from '@agent-device/host-kit/diagnostics';
import { AppError } from '@agent-device/kernel/errors';
import type { DeviceInfo } from '@agent-device/kernel/device';
import {
createSimulatorSnapshotSource,
type SimulatorSnapshotSource,
type SnapshotSourceFailure,
} from './snapshot-source-facade.ts';
import {
createSimulatorSnapshotTargetResolver,
type SimulatorSnapshotTarget,
type SimulatorSnapshotTargetResolver,
} from './snapshot-target.ts';
type SnapshotFallback = (input: CaptureSnapshotInput) => Promise<SnapshotResult>;
export type AppleSnapshotRoute = Readonly<{
capture(
device: DeviceInfo,
input: CaptureSnapshotInput,
signal: AbortSignal,
fallback: SnapshotFallback,
): Promise<SnapshotResult>;
shutdown(): Promise<void>;
}>;
export function createAppleSnapshotRoute(
host: PlatformRuntimeHost,
options: Readonly<{
source?: SimulatorSnapshotSource;
resolveTarget?: SimulatorSnapshotTargetResolver;
}> = {},
): AppleSnapshotRoute {
const source = options.source ?? createSimulatorSnapshotSource();
const resolveTarget = options.resolveTarget ?? createSimulatorSnapshotTargetResolver();
const disabledGenerations = new Set<string>();
const latestGeneration = new Map<string, string>();
return Object.freeze({
shutdown: async () => await source.close(),
capture: async (device, input, signal, fallback) => {
if (!isEligible(device, input)) return await fallback(input);
let target: SimulatorSnapshotTarget;
try {
target = await resolveTarget(device, input.options!.appBundleId!, signal);
} catch (error) {
signal.throwIfAborted();
emitRouteDiagnostic('target-resolution-failed', device, undefined, error);
return await runFallback(
input,
fallback,
{ targetId: `${device.id}:${input.options!.appBundleId!}` },
requestFor(input),
'target-resolution-failed',
[unknownGenerationResidue()],
);
}
rebaselineGeneration(target, latestGeneration, disabledGenerations);
const circuitKey = generationKey(target);
if (disabledGenerations.has(circuitKey)) {
return await runFallback(input, fallback, target, requestFor(input), 'circuit-disabled');
}
const request = requestFor(input);
const outcome = await source.acquire({
target,
hint: deriveIosCaptureHint(request),
signal,
});
if (outcome.stage === 'failed') {
if (outcome.failure.kind === 'cancelled') {
signal.throwIfAborted();
throw new AppError('COMMAND_FAILED', 'Simulator AX snapshot acquisition was cancelled.', {
reason: outcome.failure.code,
...outcome.failure.details,
});
}
const fallbackIdentity = await resolveFailureFallbackIdentity(
outcome.failure,
target,
device,
input.options!.appBundleId!,
signal,
resolveTarget,
);
return await fallbackAfterFailure(
input,
fallback,
target,
fallbackIdentity,
request,
outcome.failure,
disabledGenerations,
);
}
try {
return await withDiagnosticTimer(
'ios.snapshot-source.present',
async () =>
await host.snapshot.presentIosAcquisition(
outcome as SnapshotRuntimeAcquiredResult,
input.options,
),
{ producer: 'simulator-ax-bridge' },
);
} catch (error) {
return await fallbackAfterFailure(
input,
fallback,
target,
{ lineage: target, residue: [] },
request,
{ kind: 'malformed-tree', code: 'presentation-invariant' },
disabledGenerations,
error,
);
}
},
});
}
function isEligible(device: DeviceInfo, input: CaptureSnapshotInput): boolean {
return (
device.platform === 'apple' &&
device.appleOs === 'ios' &&
device.kind === 'simulator' &&
Boolean(input.options?.appBundleId) &&
input.options?.customActions !== true &&
input.options?.preferredBackend === undefined
);
}
async function fallbackAfterFailure(
input: CaptureSnapshotInput,
fallback: SnapshotFallback,
failedTarget: SimulatorSnapshotTarget,
identity: FallbackIdentity,
request: ReturnType<typeof createIosSnapshotRequest>,
failure: SnapshotSourceFailure,
disabledGenerations: Set<string>,
cause?: unknown,
): Promise<SnapshotResult> {
disabledGenerations.add(generationKey(failedTarget));
emitRouteDiagnostic(
failure.code,
{ id: failedTarget.udid },
failedTarget.generation,
cause,
failure.details,
);
return await runFallback(
input,
fallback,
identity.lineage,
request,
failure.code,
identity.residue,
);
}
async function runFallback(
input: CaptureSnapshotInput,
fallback: SnapshotFallback,
lineage: IosSnapshotLineage,
request: ReturnType<typeof createIosSnapshotRequest>,
reason: string,
residue: readonly IosAcquisitionResidue[] = [],
): Promise<SnapshotResult> {
const result = await fallback(input);
const comparisonIdentity: IosSnapshotComparisonIdentity = Object.freeze({
producer: 'apple-runner',
intent: request.acquisitionIntent,
lineage: Object.freeze({
...(lineage.targetId ? { targetId: lineage.targetId } : {}),
...(lineage.generation ? { generation: lineage.generation } : {}),
}),
presentationKey: buildIosSnapshotPresentationKey(request),
residue: Object.freeze([
...residue,
{ kind: 'fallback-source', producer: 'apple-runner' } as const,
]),
});
const generation = lineage.generation ? 'this app generation' : 'an unverified app generation';
const warning = `Simulator AX snapshot unavailable (${reason}); used XCTest for ${generation}.`;
return {
...result,
comparisonIdentity,
warnings: [...(result.warnings ?? []), warning],
};
}
type FallbackIdentity = Readonly<{
lineage: IosSnapshotLineage;
residue: readonly IosAcquisitionResidue[];
}>;
async function resolveFailureFallbackIdentity(
failure: SnapshotSourceFailure,
target: SimulatorSnapshotTarget,
device: DeviceInfo,
appBundleId: string,
signal: AbortSignal,
resolveTarget: SimulatorSnapshotTargetResolver,
): Promise<FallbackIdentity> {
if (failure.kind !== 'stale-target') return { lineage: target, residue: [] };
try {
return {
lineage: await resolveTarget(device, appBundleId, signal, 'refresh'),
residue: [],
};
} catch (error) {
signal.throwIfAborted();
emitRouteDiagnostic('fallback-target-resolution-failed', device, undefined, error);
return {
lineage: { targetId: target.targetId },
residue: [unknownGenerationResidue()],
};
}
}
function unknownGenerationResidue(): IosAcquisitionResidue {
return { kind: 'unknown-generation', captureId: randomUUID() };
}
function requestFor(input: CaptureSnapshotInput) {
return createIosSnapshotRequest({
raw: input.options?.raw,
interactiveOnly: input.options?.interactiveOnly,
depth: input.options?.depth,
scope: input.options?.scope,
customActions: input.options?.customActions,
acquisitionIntent: input.options?.acquisitionIntent,
});
}
function rebaselineGeneration(
target: SimulatorSnapshotTarget,
latestGeneration: Map<string, string>,
disabledGenerations: Set<string>,
): void {
const previous = latestGeneration.get(target.targetId);
if (previous && previous !== target.generation) {
disabledGenerations.delete(`${target.targetId}:${previous}`);
}
latestGeneration.set(target.targetId, target.generation);
}
function generationKey(target: SimulatorSnapshotTarget): string {
return `${target.targetId}:${target.generation}`;
}
function emitRouteDiagnostic(
reason: string,
device: Pick<DeviceInfo, 'id'>,
generation?: string,
error?: unknown,
details?: Readonly<Record<string, unknown>>,
): void {
emitDiagnostic({
level: 'debug',
phase: 'ios_snapshot_route_fallback',
data: {
reason,
deviceId: device.id,
...(generation ? { generation } : {}),
...(error ? { error: error instanceof Error ? error.message : String(error) } : {}),
...(details ? { details } : {}),
},
});
}
@@ -1,6 +1,6 @@
/**
* Dormant Simulator AX acquisition. The implementation is loaded only when a caller explicitly
* creates the source; importing this facet keeps the platform package's startup surface inert.
* Lazy Simulator AX acquisition. The implementation is loaded on the first eligible local iOS
* Simulator snapshot; importing this facet keeps the platform package's startup surface inert.
*/
export type {
SnapshotSourceFailure,
@@ -34,16 +34,11 @@ test('the Simulator AX source returns raw acquisition facts and discloses unsupp
});
const request = createIosSnapshotRequest({ interactiveOnly: true });
const hint = deriveIosCaptureHint(request);
const sourceTarget = { ...targetForTest(), generation: 'generation-1' };
try {
const result = await source.acquire({
target: {
udid: 'simulator-1',
runtime: 'iOS 26.2',
pid: 321,
generation: 'generation-1',
targetId: 'target-1',
},
target: { ...sourceTarget, targetId: 'target-1' },
hint,
});
assert.equal(fixture.builds, 1);
@@ -66,14 +61,23 @@ test('the Simulator AX source returns raw acquisition facts and discloses unsupp
{ kind: 'unavailable-fact', fact: 'interactive-query' },
]);
const regularDepthOne = await source.acquire({
target: sourceTarget,
hint: deriveIosCaptureHint(createIosSnapshotRequest({ depth: 1 })),
});
assert.equal(regularDepthOne.stage, 'acquired');
assert.equal(fixture.requestedDepths.at(-1), 10);
const rawDepthOne = await source.acquire({
target: sourceTarget,
hint: deriveIosCaptureHint(createIosSnapshotRequest({ raw: true, depth: 1 })),
});
assert.equal(rawDepthOne.stage, 'acquired');
assert.equal(fixture.requestedDepths.at(-1), 1);
fixture.responsePid = 999;
const outcome = await source.acquire({
target: {
udid: 'simulator-1',
runtime: 'iOS 26.2',
pid: 321,
generation: 'generation-1',
},
target: sourceTarget,
hint,
});
assert.equal(outcome.stage, 'failed');
@@ -118,6 +122,7 @@ type AdapterFixture = {
builds: number;
runs: number;
responsePid: number;
requestedDepths: number[];
};
function targetForTest() {
@@ -135,6 +140,7 @@ function createAdapterHost(buildDelayMs = 0): AdapterFixture {
builds: 0,
runs: 0,
responsePid: 321,
requestedDepths: [],
};
const host: SnapshotSourceHost = {
...realHost,
@@ -160,7 +166,11 @@ function createAdapterHost(buildDelayMs = 0): AdapterFixture {
};
},
start: () => new AdapterProcess(),
connect: async () => new AdapterSocket(() => fixture.responsePid),
connect: async () =>
new AdapterSocket(
() => fixture.responsePid,
(depth) => fixture.requestedDepths.push(depth),
),
readTargetProcessStartTime: async () => 'target-start',
};
fixture.host = host;
@@ -196,10 +206,12 @@ class AdapterProcess implements SnapshotSourceProcess {
class AdapterSocket extends EventEmitter implements SnapshotSourceSocket {
destroyed = false;
private readonly readResponsePid: () => number;
private readonly recordMaxDepth: (depth: number) => void;
constructor(responsePid: () => number) {
constructor(responsePid: () => number, recordMaxDepth: (depth: number) => void) {
super();
this.readResponsePid = responsePid;
this.recordMaxDepth = recordMaxDepth;
}
write(frame: Buffer): boolean {
@@ -208,7 +220,9 @@ class AdapterSocket extends EventEmitter implements SnapshotSourceSocket {
requestId: string;
pid: number;
generation: string;
maxDepth: number;
};
this.recordMaxDepth(request.maxDepth);
queueMicrotask(() => {
if (this.destroyed) return;
this.emit(
@@ -149,7 +149,7 @@ function validateRequest(request: SnapshotSourceRequest): void {
}
function resolveRequestedDepth(hint: CaptureHint, maximum: number): number {
const requested = hint.rawTraversalDepth ?? hint.regularPresentedDepth ?? maximum;
const requested = hint.rawTraversalDepth ?? maximum;
if (requested > maximum) {
throw new AppError('INVALID_ARGS', 'Simulator snapshot source depth exceeds its bound', {
requested,
@@ -0,0 +1,127 @@
#import "SnapshotBridgeRuntime.h"
#import <CoreGraphics/CoreGraphics.h>
#import <dlfcn.h>
static id primaryApplication;
static id replacementApplication;
static NSUInteger captureCount;
@interface AXElement : NSObject
@property(nonatomic) pid_t pid;
+ (id)primaryApp;
@end
@implementation AXElement
+ (id)primaryApp { return primaryApplication; }
@end
@interface XCAccessibilityElement : NSObject
+ (instancetype)elementWithProcessIdentifier:(pid_t)pid;
- (void *)AXUIElement;
@end
@implementation XCAccessibilityElement
+ (instancetype)elementWithProcessIdentifier:(pid_t)pid { return [self new]; }
- (void *)AXUIElement { return (__bridge void *)self; }
@end
@interface XCTAccessibilityFramework : NSObject
- (instancetype)initForRemoteAccess;
- (id)userTestingSnapshotForElement:(id)element options:(NSDictionary *)options error:(NSError **)error;
@end
@implementation XCTAccessibilityFramework
- (instancetype)initForRemoteAccess { return [super init]; }
- (id)userTestingSnapshotForElement:(id)element options:(NSDictionary *)options error:(NSError **)error
{
captureCount++;
if (replacementApplication) primaryApplication = replacementApplication;
return @{ @"UIAccessibilitySnapshotKeyAttributes": @{ @2: @"fixture app" },
@"UIAccessibilitySnapshotKeyChildren": @[] };
}
@end
@interface FixtureRuntime : BridgeRuntime
@end
@implementation FixtureRuntime
- (BOOL)assertAutomationMode:(BOOL)wanted { return YES; }
@end
static NSDictionary *defaultParameters(void) { return @{}; }
static NSArray *attributeNumbers(NSArray *names)
{
NSMutableArray *numbers = [NSMutableArray array];
for (NSUInteger index = 0; index < names.count; index++) [numbers addObject:@(index)];
return numbers;
}
static uint32_t valueType(const void *value) { return 0; }
static Boolean valueGet(const void *value, uint32_t type, void *out) { return false; }
void *fixtureDlopen(const char *path, int mode) { return NULL; }
void *fixtureDlsym(void *handle, const char *symbol)
{
if (!strcmp(symbol, "XCTDefaultSnapshotParameters")) return (void *)defaultParameters;
if (!strcmp(symbol, "XCAXAccessibilityAttributesForStringAttributes")) return (void *)attributeNumbers;
if (!strcmp(symbol, "AXValueGetType")) return (void *)valueType;
if (!strcmp(symbol, "AXValueGetValue")) return (void *)valueGet;
return NULL;
}
NSDictionary *failureResponse(NSString *requestId, NSString *kind, NSString *code, NSString *message)
{
return @{ @"requestId": requestId, @"error_kind": kind, @"error_code": code, @"error": message };
}
static void require(BOOL condition, NSString *message)
{
if (condition) return;
fprintf(stderr, "%s\n", message.UTF8String);
exit(1);
}
int main(int argc, const char *argv[])
{
@autoreleasepool {
require(argc == 2, @"one capture scenario is required");
NSString *scenario = @(argv[1]);
AXElement *target = [AXElement new];
target.pid = 42;
AXElement *system = [AXElement new];
system.pid = 7;
primaryApplication = target;
NSString *expectedCode = nil;
NSUInteger expectedCaptures = 0;
if ([scenario isEqualToString:@"stable"]) {
expectedCaptures = 1;
} else if ([scenario isEqualToString:@"changed"]) {
replacementApplication = system;
expectedCode = @"foreground-owner-changed";
expectedCaptures = 1;
} else {
require([@[@"covered", @"missing", @"malformed"] containsObject:scenario], @"unknown scenario");
primaryApplication = [scenario isEqualToString:@"covered"] ? system :
[scenario isEqualToString:@"missing"] ? nil : @"invalid";
expectedCode = @"foreground-owner-unverified";
}
NSString *setupError = nil;
BridgeRuntime *runtime = [[FixtureRuntime alloc] initWithError:&setupError];
require(runtime != nil, setupError ?: @"fixture initialization failed");
NSDictionary *error = nil;
NSDictionary *result = [runtime snapshotForProcess:42 maxDepth:8 maxNodes:10
requestId:@"capture-1" generation:@"generation-1" maxDurationMs:4000 error:&error];
if (expectedCode) {
require(result == nil, @"refused capture must not publish the app tree");
require([error[@"error_kind"] isEqual:@"unsupported"], @"refusal must preserve the typed failure kind");
require([error[@"error_code"] isEqual:expectedCode], @"refusal must name the ownership phase");
require([error[@"requestId"] isEqual:@"capture-1"], @"refusal must preserve request identity");
} else {
require(error == nil && [result[@"ok"] boolValue], @"stable foreground must publish successfully");
require([result[@"tree"][@"XC_kAXXCAttributeLabel"] isEqual:@"fixture app"], @"stable capture must publish the materialized app tree");
}
require(captureCount == expectedCaptures, @"covered apps must be refused before native acquisition");
}
return 0;
}
@@ -24,6 +24,7 @@ import { findProjectRoot } from '@agent-device/host-kit/version';
import { SnapshotSourceError, snapshotSourceError } from './errors.ts';
import { remainingSnapshotSourceMs } from './deadline.ts';
import type { SnapshotSourceHost, SnapshotSourceProcess, SnapshotSourceSocket } from './types.ts';
import { readSnapshotTargetProcessStartTime } from '../snapshot-process.ts';
const BRIDGE_IDLE_TIMEOUT_SECONDS = 60;
const MAX_PROCESS_LOG_BYTES = 64 * 1024;
@@ -48,23 +49,10 @@ export function createSnapshotSourceHost(): SnapshotSourceHost {
emitDiagnostic,
withDiagnosticTimer,
processId: hostProcessId,
readTargetProcessStartTime,
readTargetProcessStartTime: readSnapshotTargetProcessStartTime,
};
}
async function readTargetProcessStartTime(
pid: number,
options: { signal?: AbortSignal; timeoutMs: number },
): Promise<string | null> {
const result = await runCmd('ps', ['-p', String(pid), '-o', 'lstart='], {
allowFailure: true,
signal: options.signal,
timeoutMs: options.timeoutMs,
});
if (result.exitCode !== 0) return null;
return result.stdout.trim() || null;
}
function startSnapshotBridge(
udid: string,
bridgePath: string,
@@ -251,7 +251,8 @@ test('a crashed helper is removed and the next request starts a fresh helper', a
const fixture = createLifecycleFixture({ responseDelayMs: 80 });
const manager = new SnapshotBridgeManager(fixture.host);
const request = manager.request({ target, bridge, limits, maxDepth: 10, deadline: deadline() });
setTimeout(() => fixture.processes[0]?.crash(), 10);
await waitForDispatch(fixture);
fixture.processes[0]!.crash();
await assert.rejects(
request,
@@ -269,7 +270,8 @@ test('a crashed helper emits its bounded log once and keeps exit facts typed', a
const fixture = createLifecycleFixture({ responseDelayMs: 80 });
const manager = new SnapshotBridgeManager(fixture.host);
const request = manager.request({ target, bridge, limits, maxDepth: 10, deadline: deadline() });
setTimeout(() => fixture.processes[0]?.crash(), 10);
await waitForDispatch(fixture);
fixture.processes[0]!.crash();
let failure: SnapshotSourceError | undefined;
await assert.rejects(request, (error: unknown) => {
@@ -314,6 +316,27 @@ test('the manager rejects a response carrying a previous target generation as st
await manager.close();
});
test('the manager rejects a cached target replaced before acquisition begins', async () => {
const fixture = createLifecycleFixture({
targetStartTimes: ['replacement-start', 'replacement-start'],
});
const manager = new SnapshotBridgeManager(fixture.host);
await assert.rejects(
manager.request({
target: { ...target, processStartTime: 'original-start' },
bridge,
limits,
maxDepth: 10,
deadline: deadline(),
}),
(error: unknown) =>
error instanceof SnapshotSourceError &&
error.failureKind === 'stale-target' &&
error.failureCode === 'target-process-changed',
);
await manager.close();
});
test('the manager rejects a tree when the target process changes during acquisition', async () => {
const fixture = createLifecycleFixture({ targetStartTimes: ['start-1', 'start-2'] });
const manager = new SnapshotBridgeManager(fixture.host);
@@ -240,6 +240,14 @@ export class SnapshotBridgeManager {
generation: target.generation,
});
}
if (target.processStartTime !== undefined && target.processStartTime !== startTime) {
throw snapshotSourceError('stale-target', 'target-process-changed', {
pid: target.pid,
generation: target.generation,
expectedStartTime: target.processStartTime,
observedStartTime: startTime,
});
}
return startTime;
}
@@ -0,0 +1,44 @@
import assert from 'node:assert/strict';
import path from 'node:path';
import { beforeAll, describe, test } from 'vitest';
import { runCmd } from '@agent-device/host-kit/command';
import { mkdtempForTest } from '../__tests__/tmp-dir.ts';
describe.skipIf(process.platform !== 'darwin')('native snapshot foreground ownership', () => {
let binary: string;
beforeAll(async () => {
binary = path.join(await mkdtempForTest('snapshot-foreground-'), 'foreground-owner');
const nativeRoot = path.resolve(import.meta.dirname, '../../../../apple/snapshot-bridge');
const compiled = await runCmd(
'xcrun',
[
'--sdk',
'macosx',
'clang',
'-fobjc-arc',
'-Ddlopen=fixtureDlopen',
'-Ddlsym=fixtureDlsym',
'-framework',
'Foundation',
'-framework',
'CoreGraphics',
'-I',
nativeRoot,
path.join(nativeRoot, 'SnapshotBridgeRuntime.m'),
path.join(import.meta.dirname, 'fixtures/foreground-owner.m'),
'-o',
binary,
],
{ allowFailure: true, timeoutMs: 45_000 },
);
assert.equal(compiled.exitCode, 0, compiled.stderr);
}, 60_000);
test.each(['stable', 'covered', 'changed', 'missing', 'malformed'])(
'snapshot capture enforces %s foreground ownership',
async (scenario) => {
const result = await runCmd(binary, [scenario], { allowFailure: true, timeoutMs: 5_000 });
assert.equal(result.exitCode, 0, result.stderr);
},
);
});
@@ -21,6 +21,7 @@ export type SnapshotSourceTarget = Readonly<{
pid: number;
generation: string;
targetId?: string;
processStartTime?: string;
}>;
export type SnapshotSourceRequest = Readonly<{
@@ -0,0 +1,139 @@
import { expect, test, vi } from 'vitest';
import type { DeviceInfo } from '@agent-device/kernel/device';
import { createLocalAppleToolProvider, withAppleToolProvider } from './core/tool-provider.ts';
import { createSimulatorSnapshotTargetResolver } from './snapshot-target.ts';
const ios = {
platform: 'apple',
appleOs: 'ios',
id: 'ios-1',
name: 'iPhone',
kind: 'simulator',
target: 'mobile',
booted: true,
} as const satisfies DeviceInfo;
const app = 'com.example.app';
const signal = () => new AbortController().signal;
function targetFixture() {
const state = { pid: 42, launch: 'launch-a', start: 'start-a' as string | null };
const run = vi.fn(async (args: string[]) => ({
stdout:
args[0] === 'spawn'
? `90\t0\tUIKitApplication:com.example.app.beta[wrong][rb-legacy]\n${state.pid}\t0\tUIKitApplication:${app}[${state.launch}][rb-legacy]`
: JSON.stringify({
devices: { 'com.apple.CoreSimulator.SimRuntime.iOS-26-0': [{ udid: ios.id }] },
}),
stderr: '',
exitCode: 0,
}));
const runCommand = vi.fn(async () => ({
stdout: state.start ?? '',
stderr: '',
exitCode: state.start ? 0 : 1,
}));
const provider = createLocalAppleToolProvider({ simctl: { run }, runCommand });
const resolve = createSimulatorSnapshotTargetResolver();
return {
state,
run,
runCommand,
provider,
resolve,
discoveryCount: () => run.mock.calls.filter(([args]) => args[0] === 'spawn').length,
};
}
test('an unchanged OS process reuses its exact app target without another simctl spawn', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
const first = await fixture.resolve(ios, app, signal());
const second = await fixture.resolve(ios, app, signal());
expect(second).toBe(first);
expect(first).toEqual({
udid: ios.id,
runtime: 'com.apple.CoreSimulator.SimRuntime.iOS-26-0',
pid: 42,
generation: `42:UIKitApplication:${app}[launch-a][rb-legacy]:start-a`,
targetId: `${ios.id}:${app}`,
processStartTime: 'start-a',
});
expect(fixture.discoveryCount()).toBe(1);
expect(fixture.runCommand).toHaveBeenCalledTimes(2);
});
});
test('PID reuse cannot reuse a target from a different OS process start', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
const first = await fixture.resolve(ios, app, signal());
fixture.state.start = 'start-b';
const second = await fixture.resolve(ios, app, signal());
expect(second.pid).toBe(first.pid);
expect(second.generation).not.toBe(first.generation);
expect(second.processStartTime).toBe('start-b');
expect(fixture.discoveryCount()).toBe(2);
});
});
test('a relaunched app resolves the replacement PID after the prior process disappears', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
await fixture.resolve(ios, app, signal());
fixture.runCommand.mockResolvedValueOnce({ stdout: '', stderr: '', exitCode: 1 });
fixture.state.pid = 84;
fixture.state.launch = 'launch-b';
fixture.state.start = 'start-b';
expect(await fixture.resolve(ios, app, signal())).toMatchObject({
pid: 84,
processStartTime: 'start-b',
});
expect(fixture.discoveryCount()).toBe(2);
});
});
test('unavailable OS identity never publishes or retains an unverified target', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
await fixture.resolve(ios, app, signal());
fixture.state.start = null;
await expect(fixture.resolve(ios, app, signal())).rejects.toMatchObject({
details: { reason: 'simulator-target-identity-unavailable' },
});
fixture.state.start = 'start-a';
await fixture.resolve(ios, app, signal());
expect(fixture.discoveryCount()).toBe(3);
});
});
test('stale acquisition refresh explicitly bypasses an otherwise live cached target', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
await fixture.resolve(ios, app, signal());
fixture.state.pid = 84;
fixture.state.launch = 'launch-b';
const next = await fixture.resolve(ios, app, signal(), 'refresh');
expect(next.pid).toBe(84);
expect(fixture.discoveryCount()).toBe(2);
});
});
test('target facts stay within their runtime owner', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
await fixture.resolve(ios, app, signal());
await createSimulatorSnapshotTargetResolver()(ios, app, signal());
expect(fixture.discoveryCount()).toBe(2);
});
});
test('an aborted request cannot reuse a cached target', async () => {
const fixture = targetFixture();
await withAppleToolProvider(fixture.provider, async () => {
await fixture.resolve(ios, app, signal());
const controller = new AbortController();
controller.abort(new Error('request-ended'));
await expect(fixture.resolve(ios, app, controller.signal)).rejects.toThrow('request-ended');
expect(fixture.runCommand).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,138 @@
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import { runSimctl } from './core/apps-simctl.ts';
import { readSnapshotTargetProcessStartTime } from './snapshot-process.ts';
const TARGET_PROBE_TIMEOUT_MS = 3_000;
export type SimulatorSnapshotTarget = Readonly<{
udid: string;
runtime: string;
pid: number;
generation: string;
targetId: string;
processStartTime: string;
}>;
export type SimulatorSnapshotTargetResolver = (
device: DeviceInfo,
appBundleId: string,
signal: AbortSignal,
refresh?: 'refresh',
) => Promise<SimulatorSnapshotTarget>;
export function createSimulatorSnapshotTargetResolver(): SimulatorSnapshotTargetResolver {
const targets = new Map<string, SimulatorSnapshotTarget>();
const runtimeByDevice = new Map<string, Promise<string>>();
return async (device, appBundleId, signal, refresh) => {
signal.throwIfAborted();
const key = `${device.id}:${appBundleId}`;
const cached = targets.get(key);
if (cached && refresh !== 'refresh') {
const observed = await readSnapshotTargetProcessStartTime(cached.pid, {
signal,
timeoutMs: TARGET_PROBE_TIMEOUT_MS,
});
if (observed === cached.processStartTime) return cached;
}
targets.delete(key);
const target = await resolveSimulatorSnapshotTarget(
device,
appBundleId,
signal,
runtimeByDevice,
);
targets.set(key, target);
return target;
};
}
async function resolveSimulatorSnapshotTarget(
device: DeviceInfo,
appBundleId: string,
signal: AbortSignal,
runtimeByDevice: Map<string, Promise<string>>,
): Promise<SimulatorSnapshotTarget> {
const [jobs, runtime] = await Promise.all([
runSimctl(device, ['spawn', device.id, 'launchctl', 'list'], {
allowFailure: true,
signal,
timeoutMs: TARGET_PROBE_TIMEOUT_MS,
}),
readSimulatorRuntime(device, signal, runtimeByDevice),
]);
if (jobs.exitCode !== 0) {
throw targetError('simulator-target-probe-failed', device, appBundleId);
}
const job = readApplicationJob(jobs.stdout, appBundleId);
if (!job) {
throw targetError('simulator-target-unavailable', device, appBundleId);
}
const processStartTime = await readSnapshotTargetProcessStartTime(job.pid, {
signal,
timeoutMs: TARGET_PROBE_TIMEOUT_MS,
});
if (!processStartTime) {
throw targetError('simulator-target-identity-unavailable', device, appBundleId);
}
return Object.freeze({
udid: device.id,
runtime,
pid: job.pid,
generation: `${job.pid}:${job.label}:${processStartTime}`,
targetId: `${device.id}:${appBundleId}`,
processStartTime,
});
}
async function readSimulatorRuntime(
device: DeviceInfo,
signal: AbortSignal,
runtimeByDevice: Map<string, Promise<string>>,
): Promise<string> {
const existing = runtimeByDevice.get(device.id);
if (existing) return await existing;
const pending = runSimctl(device, ['list', 'devices', '-j'], {
allowFailure: true,
signal,
timeoutMs: TARGET_PROBE_TIMEOUT_MS,
}).then((result) => {
if (result.exitCode !== 0) throw targetError('simulator-runtime-probe-failed', device, '');
const payload = JSON.parse(result.stdout) as {
devices?: Record<string, Array<{ udid?: string }>>;
};
const runtime = Object.entries(payload.devices ?? {}).find(([, devices]) =>
devices.some((candidate) => candidate.udid === device.id),
)?.[0];
if (!runtime) throw targetError('simulator-runtime-unavailable', device, '');
return runtime;
});
runtimeByDevice.set(device.id, pending);
try {
return await pending;
} catch (error) {
runtimeByDevice.delete(device.id);
throw error;
}
}
function readApplicationJob(
output: string,
appBundleId: string,
): { pid: number; label: string } | undefined {
for (const line of output.split('\n')) {
const [pidText, , label] = line.trim().split(/\s+/);
if (!pidText || !label || !label.startsWith(`UIKitApplication:${appBundleId}[`)) continue;
const pid = Number(pidText);
if (Number.isSafeInteger(pid) && pid > 0) return { pid, label };
}
return undefined;
}
function targetError(reason: string, device: DeviceInfo, appBundleId: string): AppError {
return new AppError('COMMAND_FAILED', 'Unable to resolve the running iOS Simulator app.', {
reason,
deviceId: device.id,
appBundleId,
});
}
+28
View File
@@ -60,6 +60,34 @@ test('buildSnapshotState carries the acquisition producer beside the channel', (
expect(state.producer).toBe('appium-source');
});
test('buildSnapshotState preserves the full iOS comparison identity as one opaque key', () => {
const comparisonIdentity = {
producer: 'simulator-ax-bridge' as const,
intent: 'full' as const,
lineage: { targetId: 'ios-1:com.example.app', generation: 'launch-a' },
presentationKey: {
projection: 'regular' as const,
interactiveOnly: false,
depth: null,
scope: null,
customActions: false,
},
residue: [],
};
const state = buildSnapshotState(
{
nodes: [{ index: 0, type: 'Application' }],
backend: 'xctest',
producer: 'simulator-ax-bridge',
comparisonIdentity,
},
undefined,
);
expect(state.comparisonKey).toContain('simulator-ax-bridge');
expect(state.comparisonKey).toContain('launch-a');
});
test('buildSnapshotState preserves Android effective geometry for post-wire consumers', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][400,800]" visible-to-user="true">
+11 -1
View File
@@ -23,6 +23,8 @@ import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-p
import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts';
import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine';
import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-acquisition';
import { iosSnapshotComparisonIdentityKey } from '@agent-device/capture-kit/ios-snapshot-planning';
import type { IosSnapshotComparisonIdentity } from '@agent-device/contracts/ios-snapshot';
/**
* The ONE daemon assembly of a captured tree (ADR 0004 / #1797): normalize, group prune,
@@ -38,6 +40,7 @@ export function buildSnapshotState(
nodes?: RawSnapshotNode[];
truncated?: boolean;
quality?: unknown;
comparisonIdentity?: IosSnapshotComparisonIdentity;
} & SnapshotStateProvenance,
flags:
| (Pick<CommandFlags, 'snapshotDepth' | 'snapshotInteractiveOnly' | 'snapshotRaw'> &
@@ -73,6 +76,9 @@ export function buildSnapshotState(
createdAt: Date.now(),
...snapshotStateProvenance(data),
...(snapshotQuality ? { snapshotQuality } : {}),
...(data.comparisonIdentity
? { comparisonKey: iosSnapshotComparisonIdentityKey(data.comparisonIdentity) }
: {}),
presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)),
// Only broad Android snapshots become freshness baselines. If the user asked for a scoped
// or filtered view, preserve that output contract but avoid pretending it is safe for
@@ -153,7 +159,11 @@ function iosSnapshotPresentationOwner(
function iosSnapshotCapabilities(provenance: SnapshotStateProvenance) {
if (provenance.backend !== 'xctest' || provenance.producer === undefined) return undefined;
return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[
provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree'
provenance.producer as
| 'apple-runner'
| 'simulator-ax-bridge'
| 'appium-source'
| 'limrun-ios-tree'
];
}
+6 -2
View File
@@ -124,7 +124,7 @@ function markPostGestureStabilization(
baselineSignature,
// Recorded so the loop can tell a comparable quiet capture from one
// served by a different backend, which is not comparable at all.
baselineBackend: session.snapshot?.snapshotQuality?.backend,
baselineBackend: snapshotComparisonKey(session.snapshot),
}
: {}),
};
@@ -350,7 +350,7 @@ export async function capturePostGestureStabilizedResult<T>(params: {
const snapshot = readSnapshot(value);
return {
signature: buildInteractionSurfaceSignature(snapshot.nodes),
backend: snapshot.snapshotQuality?.backend,
backend: snapshotComparisonKey(snapshot),
};
},
signaturesStable: areInteractionSurfaceSignaturesStable,
@@ -363,6 +363,10 @@ export async function capturePostGestureStabilizedResult<T>(params: {
return outcome;
}
function snapshotComparisonKey(snapshot: SnapshotState | undefined): string | undefined {
return snapshot?.comparisonKey ?? snapshot?.snapshotQuality?.backend;
}
function isPostGestureStabilizingAction(
action: string,
positionals: string[],
@@ -30,6 +30,7 @@ import {
resetGetRuntimeFixture,
} from '../../../__tests__/interaction-get-runtime-fixture.ts';
import { captureSnapshotWithInteractor } from '../../../snapshot-interactor-capture.ts';
import { corroborateIosTapFailure } from '../interaction-ios-tap-outcome.ts';
vi.mock('../../../snapshot-interactor-capture.ts', async () => {
const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts');
@@ -288,6 +289,33 @@ test('a changed capture from a different iOS backend keeps the tap failure', asy
expect(sessionStore.get(sessionName)?.actions).toHaveLength(0);
});
test('a producer or generation switch cannot corroborate a failed tap', async () => {
const sessionName = 'ios-comparison-identity-mismatch';
const sessionStore = makeSessionStore();
const baseline = snapshot(profileNodes);
baseline.comparisonKey = 'simulator-ax-bridge:launch-a';
const session = makeIosSession(sessionName, {
appBundleId: 'com.example.app',
snapshot: baseline,
});
sessionStore.set(sessionName, session);
const after = snapshot(imageViewerNodes);
after.comparisonKey = 'apple-runner:launch-a';
await expect(
corroborateIosTapFailure({
error: new AppError('XCTEST_RECORDED_FAILURE', 'tap failed'),
command: 'click',
requestId: undefined,
flags: {},
session,
sessionStore,
contextFromFlags,
captureSnapshotForSession: async () => after,
}),
).resolves.toBeUndefined();
});
test('a sparse changed capture keeps the tap failure', async () => {
const sessionName = 'ios-sparse-tap-corroboration';
const sessionStore = makeSessionStore();
@@ -152,6 +152,32 @@ function hasMatchingPresentation(
baseline: SnapshotState,
after: SnapshotState,
command: string,
): boolean {
const identityMatch = compareSnapshotIdentity(baseline, after);
if (identityMatch !== undefined) {
if (identityMatch) return true;
emitDiagnostic({
level: 'debug',
phase: 'ios_tap_failure_corroboration_identity_mismatch',
data: { command },
});
return false;
}
return hasMatchingLegacyPresentation(baseline, after, command);
}
function compareSnapshotIdentity(
baseline: SnapshotState,
after: SnapshotState,
): boolean | undefined {
if (baseline.comparisonKey === undefined && after.comparisonKey === undefined) return undefined;
return baseline.comparisonKey !== undefined && baseline.comparisonKey === after.comparisonKey;
}
function hasMatchingLegacyPresentation(
baseline: SnapshotState,
after: SnapshotState,
command: string,
): boolean {
const baselineBackend = baseline.snapshotQuality?.backend;
const afterBackend = after.snapshotQuality?.backend;
@@ -30,6 +30,7 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal)
return {
...actual,
prewarmIosRunnerSession: vi.fn(),
notifyIosRunnerAppRelaunched: vi.fn(async () => {}),
stopIosRunnerSession: vi.fn(async () => {}),
};
});
+2 -2
View File
@@ -13,7 +13,7 @@ import type {
DaemonRequest as WireRequest,
} from '@agent-device/kernel/contracts';
import type { DeviceInfo, PlatformSelector } from '@agent-device/kernel/device';
import type { Rect, SnapshotState, SnapshotCaptureBackend } from '@agent-device/kernel/snapshot';
import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot';
import type { SnapshotFreshnessWindow } from '../snapshot/snapshot-freshness/index.ts';
// Type-only import; erased at runtime. ref-frame.ts imports SessionState from
// here, so this back-edge must stay type-only to avoid a runtime cycle.
@@ -213,7 +213,7 @@ export type PostGestureStabilization = {
* a different backend can only be re-baselined against, never concluded from
* (#1569).
*/
baselineBackend?: SnapshotCaptureBackend;
baselineBackend?: string;
};
export type PendingInteractionOutcome = {
+4 -3
View File
@@ -1,6 +1,6 @@
import {
IosSnapshotEngineError,
presentIosSnapshot,
publishIosSnapshot,
toIosSnapshotEngineErrorDetails,
} from '@agent-device/capture-kit/ios-snapshot-engine';
import {
@@ -43,11 +43,12 @@ export function presentIosSnapshotAcquisition(
const input = iosSnapshotInput(acquired, request);
try {
const presentation = presentIosSnapshot(input, request);
const presentation = publishIosSnapshot(input, request);
return {
backend: 'xctest',
producer: acquired.acquisition.producer,
nodes: presentation.nodes,
nodes: [...presentation.payload.nodes],
comparisonIdentity: presentation.comparisonIdentity,
...(acquired.acquisition.truncated === undefined
? {}
: { truncated: acquired.acquisition.truncated }),
@@ -6,6 +6,8 @@ import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts';
const VISIBLE_DEPTH_DEEP_LINK = 'agent-device-test-app:///snapshot-depth';
const CHILD_ID = 'visible-depth-projected-child';
const MISSING_HITTABILITY_WARNING =
'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.';
type SnapshotNode = {
depth?: unknown;
@@ -32,7 +34,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
'--depth',
'1',
]);
assertSnapshotBackend(regular, 'regular depth-1 snapshot');
assertSimulatorBridgeSnapshot(regular, 'regular depth-1 snapshot');
const regularNodes = snapshotNodes(regular);
const regularRoot = requireRoot(regularNodes, 'regular depth-1 snapshot');
const projectedChild = requireIdentifier(regularNodes, CHILD_ID, 'regular depth-1 snapshot');
@@ -55,7 +57,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
'snapshot',
'--raw',
]);
assertSnapshotBackend(rawFull, 'full raw visible-depth snapshot');
assertSimulatorBridgeSnapshot(rawFull, 'full raw visible-depth snapshot');
const rawFullNodes = snapshotNodes(rawFull);
const rawChild = requireIdentifier(rawFullNodes, CHILD_ID, 'full raw visible-depth snapshot');
assert.ok(
@@ -69,7 +71,7 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P
'--depth',
'1',
]);
assertSnapshotBackend(rawDepthOne, 'raw depth-1 visible-depth snapshot');
assertSimulatorBridgeSnapshot(rawDepthOne, 'raw depth-1 visible-depth snapshot');
const rawDepthOneNodes = snapshotNodes(rawDepthOne);
assert.equal(
rawDepthOneNodes.some((node) => node.identifier === CHILD_ID),
@@ -127,10 +129,14 @@ function numericDepth(node: SnapshotNode): number {
return node.depth as number;
}
function assertSnapshotBackend(result: { json?: any }, description: string): void {
function assertSimulatorBridgeSnapshot(result: { json?: any }, description: string): void {
assert.equal(
result.json?.data?.snapshotQuality?.backend,
'tree',
`${description} must exercise the recursive tree backend: ${JSON.stringify(result)}`,
undefined,
`${description} must not carry XCTest tree quality metadata: ${JSON.stringify(result)}`,
);
assert.ok(
result.json?.data?.warnings?.includes(MISSING_HITTABILITY_WARNING),
`${description} must disclose the Simulator AX bridge evidence gap: ${JSON.stringify(result)}`,
);
}