fix(ios): pin tap-outcome corroboration probes to the baseline's backend (#1634)

* fix(ios): pin tap-outcome corroboration probes to the baseline's backend

The recorded-failure screens are exactly where the capture plan flips
between XCTest and private-AX (the penalty boundary), so #1605's
same-backend requirement failed closed right where XCTest tap false
negatives actually happen: the baseline was captured via private-AX
under penalty, the probe came back via tree, and a landed tap surfaced
as XCTEST_RECORDED_FAILURE. In the AppControlBench bsky-16 run this
fired four times, each sending the model into a re-observe/retry spiral.

The comparison stays same-backend by design (backends are not comparable
views of a screen); instead the probe is now CAPTURED the way its
baseline was: a new internal preferredBackend option (never CLI-exposed)
threads daemon -> runner, and a private-AX-preferred capture takes the
exact penalized route — privateAX-first plan, 'deferred' verdict, no
degradation warning, no settle budget reset.

Live-verified on the deterministic repro (Bluesky drawer-menu press
under penalty, seeded bench feed): errored with the backend-mismatch
diagnostic before, corroborates as landed after, with no mismatch phase
in the request diagnostics. Daemon tests cover pinned and unpinned
baselines end to end through the dispatch context; the Swift plan gate
is a pure function with an executed in-bundle test (added to the ios.yml
regression list).

* style: oxfmt

* fix: exclude raw baselines from corroboration and prove the pin end to end (review)

Raw baselines could not be pinned: the raw diagnostic plan keeps
tree-first error propagation by contract and is never rerouted by the
penalty or the preferred backend, so preserving 'raw: true' on the probe
recreated exactly the backend-mismatch false failure this PR removes.
Corroboration now declines raw baselines up front (they are diagnostics,
not evidence baselines) with a regression pinning that no probe capture
is dispatched at all.

The wire is now regression-proven at every hop: a dispatch-level test
drives dispatchCommand with the context flag and asserts the emitted
RunnerCommand carries preferredBackend (red if handleSnapshotCommand or
the interactor stops forwarding); the injected-transport test asserts
the interactor's snapshot payload both ways; and a runner unit test
decodes the wire JSON, projects it through the extracted
snapshotOptions(from:), and composes it with the plan rule — pinned
regular plan defers to privateAX-first, RAW plan stays untouched.
Executed on-simulator; added to the ios.yml regression list.
This commit is contained in:
Michał Pierzchała
2026-08-06 13:27:30 +02:00
committed by GitHub
parent d5f99bab1c
commit a67c72c211
17 changed files with 310 additions and 8 deletions
+2
View File
@@ -108,6 +108,8 @@ jobs:
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \
@@ -1358,13 +1358,20 @@ extension RunnerTests {
}
}
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
let options = SnapshotOptions(
/// Pure command→options projection, extracted so the runner unit bundle can
/// prove the decoded wire field actually reaches capture options (#1634 P2).
static func snapshotOptions(from command: Command) -> SnapshotOptions {
SnapshotOptions(
interactiveOnly: command.interactiveOnly ?? false,
depth: command.depth,
scope: command.scope,
raw: command.raw ?? false
raw: command.raw ?? false,
preferredBackend: command.preferredBackend
)
}
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
let options = Self.snapshotOptions(from: command)
do {
let payload: DataPayload
if options.raw {
@@ -138,6 +138,7 @@ struct Command: Codable {
let fps: Int?
let maxSize: Int?
let interactiveOnly: Bool?
let preferredBackend: String?
let depth: Int?
let scope: String?
let raw: Bool?
@@ -396,4 +397,8 @@ struct SnapshotOptions {
let depth: Int?
let scope: String?
let raw: Bool
/// Internal daemon ask: capture with this backend first regardless of channel
/// health ("private-ax"). Same-backend evidence probes (tap-outcome
/// corroboration) must be captured the way their baseline was.
var preferredBackend: String? = nil
}
@@ -168,6 +168,17 @@ extension RunnerTests {
}
}
/// Pure gate: a capture is planned as penalized when the channel penalty is
/// active OR the daemon pinned the private-AX backend (same-backend evidence
/// probe) — both mean "do not enter XCTest tree work first, and stamp the
/// pre-selection as 'deferred' rather than a degradation".
static func snapshotXCTestChannelTreatedAsPenalized(
penalized: Bool,
preferredBackend: String?
) -> Bool {
penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue
}
/// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends
/// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw
/// diagnostic plan keeps tree-first errors, and unknown plans are left untouched.
@@ -222,9 +233,15 @@ extension RunnerTests {
// Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while
// the app remains visually responsive. Simulators can avoid that channel through private AX;
// physical devices have no independent semantic backend yet, so they use a bounded probe.
// A daemon-preferred private-AX capture (same-backend evidence probe) takes the exact
// penalized route: privateAX-first plan, 'deferred' verdict — the backend was pre-selected
// deliberately, so no degradation warning should render for it.
var xCTestChannelPenalized = false
#if os(iOS)
xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized(
penalized: isSnapshotXCTestChannelPenalized(bundleId: currentBundleId),
preferredBackend: options.preferredBackend
)
#endif
let effective = Self.effectiveSnapshotCapturePlan(
plan,
@@ -671,6 +688,59 @@ extension RunnerTests {
XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget")
}
/// #1634 P2: the decoded wire field must reach capture options and its
/// applicable plan. A pinned REGULAR capture defers to privateAX-first; the
/// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first
/// error propagation, which is exactly why raw baselines are excluded from
/// corroboration daemon-side.
func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws {
let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"#
let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8))
let options = Self.snapshotOptions(from: command)
XCTAssertEqual(options.preferredBackend, "private-ax")
XCTAssertFalse(options.raw)
let treated = Self.snapshotXCTestChannelTreatedAsPenalized(
penalized: false, preferredBackend: options.preferredBackend)
let pinned = Self.effectiveSnapshotCapturePlan(
Self.regularVisiblePlan, xCTestChannelPenalized: treated)
XCTAssertEqual(pinned.plan, [.privateAX])
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
let raw = Self.effectiveSnapshotCapturePlan(
Self.rawDiagnosticPlan, xCTestChannelPenalized: treated)
XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan)
// A command without the field decodes to no pin and a normal plan.
let bare = try JSONDecoder().decode(
Command.self, from: Data(#"{"command":"snapshot"}"#.utf8))
XCTAssertNil(Self.snapshotOptions(from: bare).preferredBackend)
}
/// Same-backend evidence probes: a daemon-pinned private-AX capture takes the
/// penalized route even with a healthy channel, so tap-outcome corroboration
/// baselines and probes are always captured by the same backend (backends are
/// never comparable views of a screen). Composed with the plan rule, the pin
/// yields the privateAX-first deferred plan.
func testPreferredPrivateAXBackendPlansAsPenalized() {
XCTAssertTrue(
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax"))
XCTAssertTrue(
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil))
XCTAssertFalse(
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil))
XCTAssertFalse(
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree"))
let pinned = Self.effectiveSnapshotCapturePlan(
Self.regularVisiblePlan,
xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized(
penalized: false, preferredBackend: "private-ax")
)
XCTAssertEqual(pinned.plan, [.privateAX])
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
}
func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() {
let regular = Self.effectiveSnapshotCapturePlan(
Self.regularVisiblePlan,
+2
View File
@@ -71,6 +71,8 @@ export type CliFlags = CloudProviderProfileFields &
cost?: boolean;
responseLevel?: ResponseLevel;
snapshotInteractiveOnly?: boolean;
/** Internal (no CLI flag): pin the capture backend for same-backend evidence probes. */
snapshotPreferredBackend?: 'private-ax';
snapshotDiff?: boolean;
snapshotDepth?: number;
snapshotScope?: string;
+7
View File
@@ -40,6 +40,13 @@ export type SnapshotOptions = {
depth?: number;
scope?: string;
raw?: boolean;
/**
* Internal (never CLI-exposed): capture with this backend first regardless of
* channel health. Evidence comparisons are only valid same-backend (backends
* are not comparable views of a screen), so a corroboration probe must be
* captured the way its baseline was.
*/
preferredBackend?: 'private-ax';
};
export type SnapshotPresentationFlagInput = {
@@ -0,0 +1,48 @@
import { beforeEach, test, vi } from 'vitest';
import assert from 'node:assert/strict';
vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => {
const actual =
await importOriginal<typeof import('../../platforms/apple/core/runner/runner-client.ts')>();
return { ...actual, runAppleRunnerCommand: vi.fn() };
});
import { dispatchCommand } from '../dispatch.ts';
import { runAppleRunnerCommand } from '../../platforms/apple/core/runner/runner-client.ts';
import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
const mockRunAppleRunnerCommand = vi.mocked(runAppleRunnerCommand);
beforeEach(() => {
vi.resetAllMocks();
mockRunAppleRunnerCommand.mockResolvedValue({
nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }],
});
});
// #1634 P2: proves the whole daemon-side wire — dispatch context ->
// handleSnapshotCommand -> interactor -> emitted RunnerCommand. Removing the
// forwarding at any of those hops turns this red; the daemon corroboration
// test (mocked dispatch) and the Swift plan-gate test cover the layers on
// either side of it.
test('dispatch snapshot forwards snapshotPreferredBackend to the runner command', async () => {
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {
snapshotPreferredBackend: 'private-ax',
});
const call = mockRunAppleRunnerCommand.mock.calls.find(
([, command]) => command.command === 'snapshot',
);
assert.ok(call, 'expected a snapshot runner command');
assert.equal(call[1].preferredBackend, 'private-ax');
});
test('dispatch snapshot without a pin emits no preferredBackend', async () => {
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {});
const call = mockRunAppleRunnerCommand.mock.calls.find(
([, command]) => command.command === 'snapshot',
);
assert.ok(call, 'expected a snapshot runner command');
assert.equal(call[1].preferredBackend, undefined);
});
+1
View File
@@ -33,6 +33,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
runnerLeaseContext?: RunnerLogicalLeaseContext;
screenshotCaptureBackend?: 'runner';
snapshotInteractiveOnly?: boolean;
snapshotPreferredBackend?: 'private-ax';
snapshotDepth?: number;
snapshotScope?: string;
snapshotRaw?: boolean;
+1
View File
@@ -653,6 +653,7 @@ async function handleSnapshotCommand(
appBundleId: snapshotContext.appBundleId,
signal: snapshotContext.signal,
interactiveOnly: snapshotContext.snapshotInteractiveOnly,
preferredBackend: snapshotContext.snapshotPreferredBackend,
depth: snapshotContext.snapshotDepth,
scope: snapshotContext.snapshotScope,
raw: snapshotContext.snapshotRaw,
+1
View File
@@ -38,6 +38,7 @@ export function contextFromFlags(
iosXctestEnvDir: flags?.iosXctestEnvDir,
screenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend,
snapshotInteractiveOnly: flags?.snapshotInteractiveOnly,
snapshotPreferredBackend: flags?.snapshotPreferredBackend,
snapshotDepth: flags?.snapshotDepth,
snapshotScope: flags?.snapshotScope,
snapshotRaw: flags?.snapshotRaw,
@@ -37,14 +37,18 @@ export const imageViewerNodes: RawSnapshotNode[] = [
},
];
export function snapshot(nodes: RawSnapshotNode[]) {
export function snapshot(
nodes: RawSnapshotNode[],
backend: 'tree' | 'queries' | 'private-ax' = 'tree',
options: { raw?: boolean } = {},
) {
return buildSnapshotState(
{
nodes,
backend: 'xctest',
quality: { state: 'healthy', backend: 'tree' },
quality: { state: 'healthy', backend },
},
{ snapshotInteractiveOnly: false },
{ snapshotInteractiveOnly: false, ...(options.raw ? { snapshotRaw: true } : {}) },
);
}
@@ -36,6 +36,9 @@ const contextFromFlags = (flags: CommandFlags | undefined) => ({
jitterPx: flags?.jitterPx,
doubleTap: flags?.doubleTap,
clickButton: flags?.clickButton,
// Mirrors the production context builder (daemon/context.ts) for the fields
// the corroboration capture path depends on.
snapshotPreferredBackend: flags?.snapshotPreferredBackend,
});
async function runClick(
@@ -115,6 +118,110 @@ test('an unchanged post-action capture keeps a failed iOS tap failed', async ()
expect(sessionStore.get(sessionName)?.actions).toHaveLength(0);
});
test('a private-ax baseline pins the corroboration probe to private-ax', async () => {
// The recorded-failure screens are exactly where the capture plan flips
// between XCTest and private-AX (the penalty boundary). Without the pin, the
// probe comes back on a different backend, the same-backend requirement
// correctly refuses to compare, and a landed tap surfaces as a failure.
const sessionName = 'ios-private-ax-pinned-corroboration';
const sessionStore = makeSessionStore();
sessionStore.set(
sessionName,
makeIosSession(sessionName, {
appBundleId: 'com.example.app',
snapshot: snapshot(profileNodes, 'private-ax'),
}),
);
const snapshotContexts: Array<Record<string, unknown> | undefined> = [];
mockDispatch.mockImplementation(async (_device, command, _positionals, _outPath, context) => {
if (command === 'press') {
throw new AppError(
'XCTEST_RECORDED_FAILURE',
'XCTest recorded a failure while executing tap; the action may not have been performed.',
);
}
if (command === 'snapshot') {
snapshotContexts.push(context as Record<string, unknown> | undefined);
return snapshotPayload(imageViewerNodes, 'private-ax');
}
return {};
});
const response = await runClick(sessionStore, sessionName);
expect(response?.ok).toBe(true);
if (response?.ok) {
expect(response.data?.warning).toMatch(/post-action accessibility capture changed/);
}
expect(snapshotContexts).toHaveLength(1);
expect(snapshotContexts[0]?.snapshotPreferredBackend).toBe('private-ax');
});
test('a raw baseline is excluded from corroboration entirely (#1634 P1)', async () => {
// The raw diagnostic plan keeps tree-first error propagation by contract and
// is never rerouted by the pin, so a raw private-AX baseline could not be
// matched same-backend — corroboration must decline up front (no probe
// capture at all) and the recorded failure surfaces unchanged.
const sessionName = 'ios-raw-baseline-no-corroboration';
const sessionStore = makeSessionStore();
sessionStore.set(
sessionName,
makeIosSession(sessionName, {
appBundleId: 'com.example.app',
snapshot: snapshot(profileNodes, 'private-ax', { raw: true }),
}),
);
mockDispatch.mockImplementation(async (_device, command) => {
if (command === 'press') {
throw new AppError(
'XCTEST_RECORDED_FAILURE',
'XCTest recorded a failure while executing tap; the action may not have been performed.',
);
}
if (command === 'snapshot') return snapshotPayload(imageViewerNodes, 'private-ax');
return {};
});
const response = await runClick(sessionStore, sessionName);
expect(response?.ok).toBe(false);
if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE');
// Declined before any probe: no corroboration snapshot was dispatched.
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
});
test('a tree baseline does not pin the corroboration probe backend', async () => {
const sessionName = 'ios-tree-baseline-unpinned-corroboration';
const sessionStore = makeSessionStore();
sessionStore.set(
sessionName,
makeIosSession(sessionName, {
appBundleId: 'com.example.app',
snapshot: snapshot(profileNodes),
}),
);
const snapshotContexts: Array<Record<string, unknown> | undefined> = [];
mockDispatch.mockImplementation(async (_device, command, _positionals, _outPath, context) => {
if (command === 'press') {
throw new AppError(
'XCTEST_RECORDED_FAILURE',
'XCTest recorded a failure while executing tap; the action may not have been performed.',
);
}
if (command === 'snapshot') {
snapshotContexts.push(context as Record<string, unknown> | undefined);
return snapshotPayload(imageViewerNodes);
}
return {};
});
const response = await runClick(sessionStore, sessionName);
expect(response?.ok).toBe(true);
expect(snapshotContexts).toHaveLength(1);
expect(snapshotContexts[0]?.snapshotPreferredBackend).toBeUndefined();
});
test('a changed capture from a different iOS backend keeps the tap failure', async () => {
const sessionName = 'ios-cross-backend-tap-corroboration';
const sessionStore = makeSessionStore();
@@ -53,7 +53,11 @@ export async function corroborateIosTapFailure(
const baseline = readCorroborationBaseline(params.session.snapshot);
if (!baseline) return undefined;
const after = await captureCorroborationSnapshot(params, baseline.presentation);
const after = await captureCorroborationSnapshot(
params,
baseline.snapshot.snapshotQuality?.backend,
baseline.presentation,
);
if (!after || !hasMatchingPresentation(baseline.snapshot, after, params.command)) {
return undefined;
}
@@ -87,11 +91,26 @@ function readCorroborationBaseline(
}
const presentation = readSnapshotPresentation(snapshot.presentationKey);
if (!presentation) return undefined;
// Raw baselines are excluded from corroboration entirely: the probe would
// replay `raw: true`, and the raw diagnostic plan keeps tree-first error
// propagation by contract — it is never rerouted by the penalty or by a
// preferred backend, so a raw private-AX baseline could not be matched
// same-backend and would recreate the mismatch false failure. Raw captures
// are diagnostics, not evidence baselines.
if (presentation.raw) {
emitDiagnostic({
level: 'debug',
phase: 'ios_tap_failure_corroboration_raw_baseline',
data: { presentationKey: snapshot.presentationKey },
});
return undefined;
}
return { snapshot, presentation };
}
async function captureCorroborationSnapshot(
params: IosTapCorroborationParams,
baselineBackend: string | undefined,
presentation: SnapshotPresentation | undefined,
): Promise<SnapshotState | undefined> {
try {
@@ -102,6 +121,11 @@ async function captureCorroborationSnapshot(
params.contextFromFlags,
{
interactiveOnly: presentation?.interactiveOnly ?? true,
// Evidence comparison is only valid same-backend, and the recorded-failure
// screens are exactly where the capture plan flips between XCTest and
// private-AX (the penalty boundary) — pin the probe to the baseline's
// backend instead of failing closed on the mismatch.
...(baselineBackend === 'private-ax' ? { preferredBackend: 'private-ax' as const } : {}),
signal: getRequestSignal(params.requestId),
},
);
@@ -14,6 +14,7 @@ export type CaptureSnapshotForSession = (
contextFromFlags: ContextFromFlags,
options: {
interactiveOnly: boolean;
preferredBackend?: 'private-ax';
androidFreshnessMode?: 'ref-refresh';
includeRects?: boolean;
signal?: AbortSignal;
@@ -27,6 +28,7 @@ export async function captureSnapshotForSession(
contextFromFlags: ContextFromFlags,
options: {
interactiveOnly: boolean;
preferredBackend?: 'private-ax';
androidFreshnessMode?: 'ref-refresh';
includeRects?: boolean;
signal?: AbortSignal;
@@ -35,6 +37,7 @@ export async function captureSnapshotForSession(
const effectiveFlags = {
...(flags ?? {}),
snapshotInteractiveOnly: options.interactiveOnly,
...(options.preferredBackend ? { snapshotPreferredBackend: options.preferredBackend } : {}),
};
const dispatchContext = contextFromFlags(
effectiveFlags,
@@ -148,6 +148,23 @@ test('snapshot over the injected transport keeps the shared xctest result shape'
assert.equal(result.nodes?.length, 2);
});
// #1634 P2: the backend pin must actually reach the wire — the daemon test
// stops at the dispatch context and the Swift test starts at the parsed
// command, so this is the assertion that fails if the interactor stops
// forwarding preferredBackend into the emitted RunnerCommand.
test('snapshot forwards preferredBackend into the emitted runner command', async () => {
const calls: RecordedRunnerCall[] = [];
const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider(calls));
await interactor.snapshot({ preferredBackend: 'private-ax' });
await interactor.snapshot();
const snapshots = calls.filter((call) => call.command.command === 'snapshot');
assert.equal(snapshots.length, 2);
assert.equal(snapshots[0]?.command.preferredBackend, 'private-ax');
assert.equal(snapshots[1]?.command.preferredBackend, undefined);
});
function recordingRunnerProvider(calls: RecordedRunnerCall[]): AppleRunnerProvider {
return {
runCommand: async (_device, command, options) => {
@@ -92,6 +92,8 @@ export type RunnerCommand = {
fps?: number;
maxSize?: number;
interactiveOnly?: boolean;
/** Pin the snapshot capture backend (same-backend evidence probes). */
preferredBackend?: 'private-ax';
depth?: number;
scope?: string;
raw?: boolean;
+1
View File
@@ -75,6 +75,7 @@ export function createAppleInteractor(
command: 'snapshot',
appBundleId: options?.appBundleId,
interactiveOnly: options?.interactiveOnly,
preferredBackend: options?.preferredBackend,
depth: options?.depth,
scope: options?.scope,
raw: options?.raw,