mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
fix(ios): pin tap-outcome corroboration probes to the baseline's backend (#1634)
* fix(ios): pin tap-outcome corroboration probes to the baseline's backend The recorded-failure screens are exactly where the capture plan flips between XCTest and private-AX (the penalty boundary), so #1605's same-backend requirement failed closed right where XCTest tap false negatives actually happen: the baseline was captured via private-AX under penalty, the probe came back via tree, and a landed tap surfaced as XCTEST_RECORDED_FAILURE. In the AppControlBench bsky-16 run this fired four times, each sending the model into a re-observe/retry spiral. The comparison stays same-backend by design (backends are not comparable views of a screen); instead the probe is now CAPTURED the way its baseline was: a new internal preferredBackend option (never CLI-exposed) threads daemon -> runner, and a private-AX-preferred capture takes the exact penalized route — privateAX-first plan, 'deferred' verdict, no degradation warning, no settle budget reset. Live-verified on the deterministic repro (Bluesky drawer-menu press under penalty, seeded bench feed): errored with the backend-mismatch diagnostic before, corroborates as landed after, with no mismatch phase in the request diagnostics. Daemon tests cover pinned and unpinned baselines end to end through the dispatch context; the Swift plan gate is a pure function with an executed in-bundle test (added to the ios.yml regression list). * style: oxfmt * fix: exclude raw baselines from corroboration and prove the pin end to end (review) Raw baselines could not be pinned: the raw diagnostic plan keeps tree-first error propagation by contract and is never rerouted by the penalty or the preferred backend, so preserving 'raw: true' on the probe recreated exactly the backend-mismatch false failure this PR removes. Corroboration now declines raw baselines up front (they are diagnostics, not evidence baselines) with a regression pinning that no probe capture is dispatched at all. The wire is now regression-proven at every hop: a dispatch-level test drives dispatchCommand with the context flag and asserts the emitted RunnerCommand carries preferredBackend (red if handleSnapshotCommand or the interactor stops forwarding); the injected-transport test asserts the interactor's snapshot payload both ways; and a runner unit test decodes the wire JSON, projects it through the extracted snapshotOptions(from:), and composes it with the plan rule — pinned regular plan defers to privateAX-first, RAW plan stays untouched. Executed on-simulator; added to the ios.yml regression list.
This commit is contained in:
committed by
GitHub
parent
d5f99bab1c
commit
a67c72c211
@@ -108,6 +108,8 @@ jobs:
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \
|
||||
|
||||
+10
-3
@@ -1358,13 +1358,20 @@ extension RunnerTests {
|
||||
}
|
||||
}
|
||||
|
||||
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
|
||||
let options = SnapshotOptions(
|
||||
/// Pure command→options projection, extracted so the runner unit bundle can
|
||||
/// prove the decoded wire field actually reaches capture options (#1634 P2).
|
||||
static func snapshotOptions(from command: Command) -> SnapshotOptions {
|
||||
SnapshotOptions(
|
||||
interactiveOnly: command.interactiveOnly ?? false,
|
||||
depth: command.depth,
|
||||
scope: command.scope,
|
||||
raw: command.raw ?? false
|
||||
raw: command.raw ?? false,
|
||||
preferredBackend: command.preferredBackend
|
||||
)
|
||||
}
|
||||
|
||||
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
|
||||
let options = Self.snapshotOptions(from: command)
|
||||
do {
|
||||
let payload: DataPayload
|
||||
if options.raw {
|
||||
|
||||
@@ -138,6 +138,7 @@ struct Command: Codable {
|
||||
let fps: Int?
|
||||
let maxSize: Int?
|
||||
let interactiveOnly: Bool?
|
||||
let preferredBackend: String?
|
||||
let depth: Int?
|
||||
let scope: String?
|
||||
let raw: Bool?
|
||||
@@ -396,4 +397,8 @@ struct SnapshotOptions {
|
||||
let depth: Int?
|
||||
let scope: String?
|
||||
let raw: Bool
|
||||
/// Internal daemon ask: capture with this backend first regardless of channel
|
||||
/// health ("private-ax"). Same-backend evidence probes (tap-outcome
|
||||
/// corroboration) must be captured the way their baseline was.
|
||||
var preferredBackend: String? = nil
|
||||
}
|
||||
|
||||
+71
-1
@@ -168,6 +168,17 @@ extension RunnerTests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure gate: a capture is planned as penalized when the channel penalty is
|
||||
/// active OR the daemon pinned the private-AX backend (same-backend evidence
|
||||
/// probe) — both mean "do not enter XCTest tree work first, and stamp the
|
||||
/// pre-selection as 'deferred' rather than a degradation".
|
||||
static func snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: Bool,
|
||||
preferredBackend: String?
|
||||
) -> Bool {
|
||||
penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue
|
||||
}
|
||||
|
||||
/// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends
|
||||
/// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw
|
||||
/// diagnostic plan keeps tree-first errors, and unknown plans are left untouched.
|
||||
@@ -222,9 +233,15 @@ extension RunnerTests {
|
||||
// Reorder is iOS-only because hostile screens can make XCTest tree/query work grind while
|
||||
// the app remains visually responsive. Simulators can avoid that channel through private AX;
|
||||
// physical devices have no independent semantic backend yet, so they use a bounded probe.
|
||||
// A daemon-preferred private-AX capture (same-backend evidence probe) takes the exact
|
||||
// penalized route: privateAX-first plan, 'deferred' verdict — the backend was pre-selected
|
||||
// deliberately, so no degradation warning should render for it.
|
||||
var xCTestChannelPenalized = false
|
||||
#if os(iOS)
|
||||
xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
|
||||
xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: isSnapshotXCTestChannelPenalized(bundleId: currentBundleId),
|
||||
preferredBackend: options.preferredBackend
|
||||
)
|
||||
#endif
|
||||
let effective = Self.effectiveSnapshotCapturePlan(
|
||||
plan,
|
||||
@@ -671,6 +688,59 @@ extension RunnerTests {
|
||||
XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget")
|
||||
}
|
||||
|
||||
/// #1634 P2: the decoded wire field must reach capture options and its
|
||||
/// applicable plan. A pinned REGULAR capture defers to privateAX-first; the
|
||||
/// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first
|
||||
/// error propagation, which is exactly why raw baselines are excluded from
|
||||
/// corroboration daemon-side.
|
||||
func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws {
|
||||
let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"#
|
||||
let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8))
|
||||
let options = Self.snapshotOptions(from: command)
|
||||
XCTAssertEqual(options.preferredBackend, "private-ax")
|
||||
XCTAssertFalse(options.raw)
|
||||
|
||||
let treated = Self.snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: false, preferredBackend: options.preferredBackend)
|
||||
let pinned = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan, xCTestChannelPenalized: treated)
|
||||
XCTAssertEqual(pinned.plan, [.privateAX])
|
||||
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
|
||||
|
||||
let raw = Self.effectiveSnapshotCapturePlan(
|
||||
Self.rawDiagnosticPlan, xCTestChannelPenalized: treated)
|
||||
XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan)
|
||||
|
||||
// A command without the field decodes to no pin and a normal plan.
|
||||
let bare = try JSONDecoder().decode(
|
||||
Command.self, from: Data(#"{"command":"snapshot"}"#.utf8))
|
||||
XCTAssertNil(Self.snapshotOptions(from: bare).preferredBackend)
|
||||
}
|
||||
|
||||
/// Same-backend evidence probes: a daemon-pinned private-AX capture takes the
|
||||
/// penalized route even with a healthy channel, so tap-outcome corroboration
|
||||
/// baselines and probes are always captured by the same backend (backends are
|
||||
/// never comparable views of a screen). Composed with the plan rule, the pin
|
||||
/// yields the privateAX-first deferred plan.
|
||||
func testPreferredPrivateAXBackendPlansAsPenalized() {
|
||||
XCTAssertTrue(
|
||||
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "private-ax"))
|
||||
XCTAssertTrue(
|
||||
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: true, preferredBackend: nil))
|
||||
XCTAssertFalse(
|
||||
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: nil))
|
||||
XCTAssertFalse(
|
||||
Self.snapshotXCTestChannelTreatedAsPenalized(penalized: false, preferredBackend: "tree"))
|
||||
|
||||
let pinned = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan,
|
||||
xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: false, preferredBackend: "private-ax")
|
||||
)
|
||||
XCTAssertEqual(pinned.plan, [.privateAX])
|
||||
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
|
||||
}
|
||||
|
||||
func testEffectiveSnapshotCapturePlanDefersXCTestBackedTiersOnlyWhenPenalizedRegularPlan() {
|
||||
let regular = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan,
|
||||
|
||||
@@ -71,6 +71,8 @@ export type CliFlags = CloudProviderProfileFields &
|
||||
cost?: boolean;
|
||||
responseLevel?: ResponseLevel;
|
||||
snapshotInteractiveOnly?: boolean;
|
||||
/** Internal (no CLI flag): pin the capture backend for same-backend evidence probes. */
|
||||
snapshotPreferredBackend?: 'private-ax';
|
||||
snapshotDiff?: boolean;
|
||||
snapshotDepth?: number;
|
||||
snapshotScope?: string;
|
||||
|
||||
@@ -40,6 +40,13 @@ export type SnapshotOptions = {
|
||||
depth?: number;
|
||||
scope?: string;
|
||||
raw?: boolean;
|
||||
/**
|
||||
* Internal (never CLI-exposed): capture with this backend first regardless of
|
||||
* channel health. Evidence comparisons are only valid same-backend (backends
|
||||
* are not comparable views of a screen), so a corroboration probe must be
|
||||
* captured the way its baseline was.
|
||||
*/
|
||||
preferredBackend?: 'private-ax';
|
||||
};
|
||||
|
||||
export type SnapshotPresentationFlagInput = {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { beforeEach, test, vi } from 'vitest';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import('../../platforms/apple/core/runner/runner-client.ts')>();
|
||||
return { ...actual, runAppleRunnerCommand: vi.fn() };
|
||||
});
|
||||
|
||||
import { dispatchCommand } from '../dispatch.ts';
|
||||
import { runAppleRunnerCommand } from '../../platforms/apple/core/runner/runner-client.ts';
|
||||
import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
|
||||
|
||||
const mockRunAppleRunnerCommand = vi.mocked(runAppleRunnerCommand);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mockRunAppleRunnerCommand.mockResolvedValue({
|
||||
nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }],
|
||||
});
|
||||
});
|
||||
|
||||
// #1634 P2: proves the whole daemon-side wire — dispatch context ->
|
||||
// handleSnapshotCommand -> interactor -> emitted RunnerCommand. Removing the
|
||||
// forwarding at any of those hops turns this red; the daemon corroboration
|
||||
// test (mocked dispatch) and the Swift plan-gate test cover the layers on
|
||||
// either side of it.
|
||||
test('dispatch snapshot forwards snapshotPreferredBackend to the runner command', async () => {
|
||||
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {
|
||||
snapshotPreferredBackend: 'private-ax',
|
||||
});
|
||||
|
||||
const call = mockRunAppleRunnerCommand.mock.calls.find(
|
||||
([, command]) => command.command === 'snapshot',
|
||||
);
|
||||
assert.ok(call, 'expected a snapshot runner command');
|
||||
assert.equal(call[1].preferredBackend, 'private-ax');
|
||||
});
|
||||
|
||||
test('dispatch snapshot without a pin emits no preferredBackend', async () => {
|
||||
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {});
|
||||
|
||||
const call = mockRunAppleRunnerCommand.mock.calls.find(
|
||||
([, command]) => command.command === 'snapshot',
|
||||
);
|
||||
assert.ok(call, 'expected a snapshot runner command');
|
||||
assert.equal(call[1].preferredBackend, undefined);
|
||||
});
|
||||
@@ -33,6 +33,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
|
||||
runnerLeaseContext?: RunnerLogicalLeaseContext;
|
||||
screenshotCaptureBackend?: 'runner';
|
||||
snapshotInteractiveOnly?: boolean;
|
||||
snapshotPreferredBackend?: 'private-ax';
|
||||
snapshotDepth?: number;
|
||||
snapshotScope?: string;
|
||||
snapshotRaw?: boolean;
|
||||
|
||||
@@ -653,6 +653,7 @@ async function handleSnapshotCommand(
|
||||
appBundleId: snapshotContext.appBundleId,
|
||||
signal: snapshotContext.signal,
|
||||
interactiveOnly: snapshotContext.snapshotInteractiveOnly,
|
||||
preferredBackend: snapshotContext.snapshotPreferredBackend,
|
||||
depth: snapshotContext.snapshotDepth,
|
||||
scope: snapshotContext.snapshotScope,
|
||||
raw: snapshotContext.snapshotRaw,
|
||||
|
||||
@@ -38,6 +38,7 @@ export function contextFromFlags(
|
||||
iosXctestEnvDir: flags?.iosXctestEnvDir,
|
||||
screenshotCaptureBackend: flags?.maestro?.screenshotCaptureBackend,
|
||||
snapshotInteractiveOnly: flags?.snapshotInteractiveOnly,
|
||||
snapshotPreferredBackend: flags?.snapshotPreferredBackend,
|
||||
snapshotDepth: flags?.snapshotDepth,
|
||||
snapshotScope: flags?.snapshotScope,
|
||||
snapshotRaw: flags?.snapshotRaw,
|
||||
|
||||
@@ -37,14 +37,18 @@ export const imageViewerNodes: RawSnapshotNode[] = [
|
||||
},
|
||||
];
|
||||
|
||||
export function snapshot(nodes: RawSnapshotNode[]) {
|
||||
export function snapshot(
|
||||
nodes: RawSnapshotNode[],
|
||||
backend: 'tree' | 'queries' | 'private-ax' = 'tree',
|
||||
options: { raw?: boolean } = {},
|
||||
) {
|
||||
return buildSnapshotState(
|
||||
{
|
||||
nodes,
|
||||
backend: 'xctest',
|
||||
quality: { state: 'healthy', backend: 'tree' },
|
||||
quality: { state: 'healthy', backend },
|
||||
},
|
||||
{ snapshotInteractiveOnly: false },
|
||||
{ snapshotInteractiveOnly: false, ...(options.raw ? { snapshotRaw: true } : {}) },
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ const contextFromFlags = (flags: CommandFlags | undefined) => ({
|
||||
jitterPx: flags?.jitterPx,
|
||||
doubleTap: flags?.doubleTap,
|
||||
clickButton: flags?.clickButton,
|
||||
// Mirrors the production context builder (daemon/context.ts) for the fields
|
||||
// the corroboration capture path depends on.
|
||||
snapshotPreferredBackend: flags?.snapshotPreferredBackend,
|
||||
});
|
||||
|
||||
async function runClick(
|
||||
@@ -115,6 +118,110 @@ test('an unchanged post-action capture keeps a failed iOS tap failed', async ()
|
||||
expect(sessionStore.get(sessionName)?.actions).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('a private-ax baseline pins the corroboration probe to private-ax', async () => {
|
||||
// The recorded-failure screens are exactly where the capture plan flips
|
||||
// between XCTest and private-AX (the penalty boundary). Without the pin, the
|
||||
// probe comes back on a different backend, the same-backend requirement
|
||||
// correctly refuses to compare, and a landed tap surfaces as a failure.
|
||||
const sessionName = 'ios-private-ax-pinned-corroboration';
|
||||
const sessionStore = makeSessionStore();
|
||||
sessionStore.set(
|
||||
sessionName,
|
||||
makeIosSession(sessionName, {
|
||||
appBundleId: 'com.example.app',
|
||||
snapshot: snapshot(profileNodes, 'private-ax'),
|
||||
}),
|
||||
);
|
||||
const snapshotContexts: Array<Record<string, unknown> | undefined> = [];
|
||||
mockDispatch.mockImplementation(async (_device, command, _positionals, _outPath, context) => {
|
||||
if (command === 'press') {
|
||||
throw new AppError(
|
||||
'XCTEST_RECORDED_FAILURE',
|
||||
'XCTest recorded a failure while executing tap; the action may not have been performed.',
|
||||
);
|
||||
}
|
||||
if (command === 'snapshot') {
|
||||
snapshotContexts.push(context as Record<string, unknown> | undefined);
|
||||
return snapshotPayload(imageViewerNodes, 'private-ax');
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const response = await runClick(sessionStore, sessionName);
|
||||
|
||||
expect(response?.ok).toBe(true);
|
||||
if (response?.ok) {
|
||||
expect(response.data?.warning).toMatch(/post-action accessibility capture changed/);
|
||||
}
|
||||
expect(snapshotContexts).toHaveLength(1);
|
||||
expect(snapshotContexts[0]?.snapshotPreferredBackend).toBe('private-ax');
|
||||
});
|
||||
|
||||
test('a raw baseline is excluded from corroboration entirely (#1634 P1)', async () => {
|
||||
// The raw diagnostic plan keeps tree-first error propagation by contract and
|
||||
// is never rerouted by the pin, so a raw private-AX baseline could not be
|
||||
// matched same-backend — corroboration must decline up front (no probe
|
||||
// capture at all) and the recorded failure surfaces unchanged.
|
||||
const sessionName = 'ios-raw-baseline-no-corroboration';
|
||||
const sessionStore = makeSessionStore();
|
||||
sessionStore.set(
|
||||
sessionName,
|
||||
makeIosSession(sessionName, {
|
||||
appBundleId: 'com.example.app',
|
||||
snapshot: snapshot(profileNodes, 'private-ax', { raw: true }),
|
||||
}),
|
||||
);
|
||||
mockDispatch.mockImplementation(async (_device, command) => {
|
||||
if (command === 'press') {
|
||||
throw new AppError(
|
||||
'XCTEST_RECORDED_FAILURE',
|
||||
'XCTest recorded a failure while executing tap; the action may not have been performed.',
|
||||
);
|
||||
}
|
||||
if (command === 'snapshot') return snapshotPayload(imageViewerNodes, 'private-ax');
|
||||
return {};
|
||||
});
|
||||
|
||||
const response = await runClick(sessionStore, sessionName);
|
||||
|
||||
expect(response?.ok).toBe(false);
|
||||
if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE');
|
||||
// Declined before any probe: no corroboration snapshot was dispatched.
|
||||
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('a tree baseline does not pin the corroboration probe backend', async () => {
|
||||
const sessionName = 'ios-tree-baseline-unpinned-corroboration';
|
||||
const sessionStore = makeSessionStore();
|
||||
sessionStore.set(
|
||||
sessionName,
|
||||
makeIosSession(sessionName, {
|
||||
appBundleId: 'com.example.app',
|
||||
snapshot: snapshot(profileNodes),
|
||||
}),
|
||||
);
|
||||
const snapshotContexts: Array<Record<string, unknown> | undefined> = [];
|
||||
mockDispatch.mockImplementation(async (_device, command, _positionals, _outPath, context) => {
|
||||
if (command === 'press') {
|
||||
throw new AppError(
|
||||
'XCTEST_RECORDED_FAILURE',
|
||||
'XCTest recorded a failure while executing tap; the action may not have been performed.',
|
||||
);
|
||||
}
|
||||
if (command === 'snapshot') {
|
||||
snapshotContexts.push(context as Record<string, unknown> | undefined);
|
||||
return snapshotPayload(imageViewerNodes);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
|
||||
const response = await runClick(sessionStore, sessionName);
|
||||
|
||||
expect(response?.ok).toBe(true);
|
||||
expect(snapshotContexts).toHaveLength(1);
|
||||
expect(snapshotContexts[0]?.snapshotPreferredBackend).toBeUndefined();
|
||||
});
|
||||
|
||||
test('a changed capture from a different iOS backend keeps the tap failure', async () => {
|
||||
const sessionName = 'ios-cross-backend-tap-corroboration';
|
||||
const sessionStore = makeSessionStore();
|
||||
|
||||
@@ -53,7 +53,11 @@ export async function corroborateIosTapFailure(
|
||||
const baseline = readCorroborationBaseline(params.session.snapshot);
|
||||
if (!baseline) return undefined;
|
||||
|
||||
const after = await captureCorroborationSnapshot(params, baseline.presentation);
|
||||
const after = await captureCorroborationSnapshot(
|
||||
params,
|
||||
baseline.snapshot.snapshotQuality?.backend,
|
||||
baseline.presentation,
|
||||
);
|
||||
if (!after || !hasMatchingPresentation(baseline.snapshot, after, params.command)) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -87,11 +91,26 @@ function readCorroborationBaseline(
|
||||
}
|
||||
const presentation = readSnapshotPresentation(snapshot.presentationKey);
|
||||
if (!presentation) return undefined;
|
||||
// Raw baselines are excluded from corroboration entirely: the probe would
|
||||
// replay `raw: true`, and the raw diagnostic plan keeps tree-first error
|
||||
// propagation by contract — it is never rerouted by the penalty or by a
|
||||
// preferred backend, so a raw private-AX baseline could not be matched
|
||||
// same-backend and would recreate the mismatch false failure. Raw captures
|
||||
// are diagnostics, not evidence baselines.
|
||||
if (presentation.raw) {
|
||||
emitDiagnostic({
|
||||
level: 'debug',
|
||||
phase: 'ios_tap_failure_corroboration_raw_baseline',
|
||||
data: { presentationKey: snapshot.presentationKey },
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
return { snapshot, presentation };
|
||||
}
|
||||
|
||||
async function captureCorroborationSnapshot(
|
||||
params: IosTapCorroborationParams,
|
||||
baselineBackend: string | undefined,
|
||||
presentation: SnapshotPresentation | undefined,
|
||||
): Promise<SnapshotState | undefined> {
|
||||
try {
|
||||
@@ -102,6 +121,11 @@ async function captureCorroborationSnapshot(
|
||||
params.contextFromFlags,
|
||||
{
|
||||
interactiveOnly: presentation?.interactiveOnly ?? true,
|
||||
// Evidence comparison is only valid same-backend, and the recorded-failure
|
||||
// screens are exactly where the capture plan flips between XCTest and
|
||||
// private-AX (the penalty boundary) — pin the probe to the baseline's
|
||||
// backend instead of failing closed on the mismatch.
|
||||
...(baselineBackend === 'private-ax' ? { preferredBackend: 'private-ax' as const } : {}),
|
||||
signal: getRequestSignal(params.requestId),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ export type CaptureSnapshotForSession = (
|
||||
contextFromFlags: ContextFromFlags,
|
||||
options: {
|
||||
interactiveOnly: boolean;
|
||||
preferredBackend?: 'private-ax';
|
||||
androidFreshnessMode?: 'ref-refresh';
|
||||
includeRects?: boolean;
|
||||
signal?: AbortSignal;
|
||||
@@ -27,6 +28,7 @@ export async function captureSnapshotForSession(
|
||||
contextFromFlags: ContextFromFlags,
|
||||
options: {
|
||||
interactiveOnly: boolean;
|
||||
preferredBackend?: 'private-ax';
|
||||
androidFreshnessMode?: 'ref-refresh';
|
||||
includeRects?: boolean;
|
||||
signal?: AbortSignal;
|
||||
@@ -35,6 +37,7 @@ export async function captureSnapshotForSession(
|
||||
const effectiveFlags = {
|
||||
...(flags ?? {}),
|
||||
snapshotInteractiveOnly: options.interactiveOnly,
|
||||
...(options.preferredBackend ? { snapshotPreferredBackend: options.preferredBackend } : {}),
|
||||
};
|
||||
const dispatchContext = contextFromFlags(
|
||||
effectiveFlags,
|
||||
|
||||
@@ -148,6 +148,23 @@ test('snapshot over the injected transport keeps the shared xctest result shape'
|
||||
assert.equal(result.nodes?.length, 2);
|
||||
});
|
||||
|
||||
// #1634 P2: the backend pin must actually reach the wire — the daemon test
|
||||
// stops at the dispatch context and the Swift test starts at the parsed
|
||||
// command, so this is the assertion that fails if the interactor stops
|
||||
// forwarding preferredBackend into the emitted RunnerCommand.
|
||||
test('snapshot forwards preferredBackend into the emitted runner command', async () => {
|
||||
const calls: RecordedRunnerCall[] = [];
|
||||
const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider(calls));
|
||||
|
||||
await interactor.snapshot({ preferredBackend: 'private-ax' });
|
||||
await interactor.snapshot();
|
||||
|
||||
const snapshots = calls.filter((call) => call.command.command === 'snapshot');
|
||||
assert.equal(snapshots.length, 2);
|
||||
assert.equal(snapshots[0]?.command.preferredBackend, 'private-ax');
|
||||
assert.equal(snapshots[1]?.command.preferredBackend, undefined);
|
||||
});
|
||||
|
||||
function recordingRunnerProvider(calls: RecordedRunnerCall[]): AppleRunnerProvider {
|
||||
return {
|
||||
runCommand: async (_device, command, options) => {
|
||||
|
||||
@@ -92,6 +92,8 @@ export type RunnerCommand = {
|
||||
fps?: number;
|
||||
maxSize?: number;
|
||||
interactiveOnly?: boolean;
|
||||
/** Pin the snapshot capture backend (same-backend evidence probes). */
|
||||
preferredBackend?: 'private-ax';
|
||||
depth?: number;
|
||||
scope?: string;
|
||||
raw?: boolean;
|
||||
|
||||
@@ -75,6 +75,7 @@ export function createAppleInteractor(
|
||||
command: 'snapshot',
|
||||
appBundleId: options?.appBundleId,
|
||||
interactiveOnly: options?.interactiveOnly,
|
||||
preferredBackend: options?.preferredBackend,
|
||||
depth: options?.depth,
|
||||
scope: options?.scope,
|
||||
raw: options?.raw,
|
||||
|
||||
Reference in New Issue
Block a user