mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
feat(ios): add snapshot backend conformance (#1930)
* feat(ios): add snapshot backend conformance * fix(ios): load built SDK at live runtime * test(client): isolate snapshot forwarding regression * refactor(snapshot): keep backend capability metadata internal * fix(test): merge backend conformance imports * fix(snapshot): keep backend forcing internal * refactor(snapshot): isolate backend capability fixtures * refactor(snapshot): keep capability governance internal * fix(ios): align snapshot actionability contract
This commit is contained in:
committed by
GitHub
parent
30de1597d3
commit
17da776350
@@ -173,7 +173,9 @@ jobs:
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXDepthLimitedRequiresEveryFrontierResolved \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDeepExtensionCountsMissedFrontiers \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRawDiagnosticPlanCarriesOnlyBackendsThatCanServeRaw \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotBackendDeclarationsMatchCapabilityFixture \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testProjectionMismatchFailureIsStructuredAndNotAnAxFailure \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRegularPresentationProjectsToViewportAndKeepsScrollHint \
|
||||
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXRawProjectionKeepsEveryAcquiredNode \
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
import XCTest
|
||||
|
||||
enum SnapshotBackendEnvironment {
|
||||
case simulator
|
||||
case physicalDevice
|
||||
}
|
||||
|
||||
enum SnapshotBackendKind: String, CaseIterable {
|
||||
case recursiveTree = "tree"
|
||||
case querySweep = "queries"
|
||||
case privateAX = "private-ax"
|
||||
|
||||
var isForceable: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .privateAX:
|
||||
return true
|
||||
case .querySweep:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
var hittableSemantics: String {
|
||||
"geometric-actionability"
|
||||
}
|
||||
|
||||
var usesXCTestAccessibilityChannel: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .querySweep:
|
||||
return true
|
||||
case .privateAX:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// The raw projection is the acquired tree, so only a backend that enumerates a hierarchy can
|
||||
/// serve it. The query sweep answers an interactive element query: it has no hierarchy to
|
||||
/// return, and planning it for `--raw` is exactly how a raw request gets answered with regular
|
||||
/// membership.
|
||||
var supportsRawProjection: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .privateAX:
|
||||
return true
|
||||
case .querySweep:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
var isAvailableOnCurrentPlatform: Bool {
|
||||
#if os(iOS) && targetEnvironment(simulator)
|
||||
return isAvailable(on: .simulator)
|
||||
#else
|
||||
return isAvailable(on: .physicalDevice)
|
||||
#endif
|
||||
}
|
||||
|
||||
func isAvailable(on environment: SnapshotBackendEnvironment) -> Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .querySweep:
|
||||
return true
|
||||
case .privateAX:
|
||||
return environment == .simulator
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
|
||||
private struct SnapshotBackendParityFixture: Decodable {
|
||||
struct Availability: Decodable {
|
||||
let simulator: Bool
|
||||
let physicalDevice: Bool
|
||||
}
|
||||
|
||||
struct Backend: Decodable {
|
||||
let name: String
|
||||
let forceable: Bool
|
||||
let supportsRawProjection: Bool
|
||||
let hittable: String
|
||||
let availability: Availability
|
||||
}
|
||||
|
||||
let backends: [Backend]
|
||||
}
|
||||
|
||||
extension RunnerTests {
|
||||
private func loadSnapshotBackendParityFixture() throws -> SnapshotBackendParityFixture {
|
||||
let fixtureURL = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent() // AgentDeviceRunnerUITests
|
||||
.deletingLastPathComponent() // AgentDeviceRunner
|
||||
.deletingLastPathComponent() // runner
|
||||
.deletingLastPathComponent() // apple
|
||||
.deletingLastPathComponent() // repo root
|
||||
.appendingPathComponent("contracts")
|
||||
.appendingPathComponent("fixtures")
|
||||
.appendingPathComponent("ios-snapshot-backends.json")
|
||||
return try JSONDecoder().decode(
|
||||
SnapshotBackendParityFixture.self,
|
||||
from: Data(contentsOf: fixtureURL)
|
||||
)
|
||||
}
|
||||
|
||||
/// The JSON table is the cross-runtime declaration used by the TypeScript capability registry
|
||||
/// and this runner. A backend case, forceability branch, raw projection claim, or availability
|
||||
/// change that is not classified in both implementations fails before an iOS smoke can drift.
|
||||
func testSnapshotBackendDeclarationsMatchCapabilityFixture() throws {
|
||||
let fixture = try loadSnapshotBackendParityFixture()
|
||||
XCTAssertEqual(
|
||||
fixture.backends.map(\.name),
|
||||
SnapshotBackendKind.allCases.map(\.rawValue)
|
||||
)
|
||||
|
||||
for expected in fixture.backends {
|
||||
guard let backend = SnapshotBackendKind(rawValue: expected.name) else {
|
||||
XCTFail("fixture contains an unknown snapshot backend: \(expected.name)")
|
||||
continue
|
||||
}
|
||||
XCTAssertEqual(backend.isForceable, expected.forceable, expected.name)
|
||||
XCTAssertEqual(backend.supportsRawProjection, expected.supportsRawProjection, expected.name)
|
||||
XCTAssertEqual(
|
||||
backend.hittableSemantics,
|
||||
expected.hittable,
|
||||
"hittable semantics: \(expected.name)"
|
||||
)
|
||||
XCTAssertEqual(
|
||||
backend.isAvailable(on: .simulator),
|
||||
expected.availability.simulator,
|
||||
"simulator availability: \(expected.name)"
|
||||
)
|
||||
XCTAssertEqual(
|
||||
backend.isAvailable(on: .physicalDevice),
|
||||
expected.availability.physicalDevice,
|
||||
"physical-device availability: \(expected.name)"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
+76
-57
@@ -14,9 +14,10 @@ struct SnapshotQuality: Codable {
|
||||
let state: String
|
||||
/// Backend that produced the returned payload: tree | queries | private-ax.
|
||||
let backend: String
|
||||
/// Why recovery ran (first failure) or why the payload is degraded.
|
||||
/// Why recovery ran (first failure), why the payload is degraded, or why an internal backend
|
||||
/// selection was honored.
|
||||
let reason: String?
|
||||
/// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes.
|
||||
/// Machine-readable reason: ax-rejected | sparse-tree | budget | no-nodes | requested-backend.
|
||||
let reasonCode: String?
|
||||
/// Private AX ladder cap when the accepted tree is shallower than requested.
|
||||
let effectiveDepth: Int?
|
||||
@@ -41,47 +42,6 @@ struct SnapshotCustomActionCoverage: Codable {
|
||||
let blocked: Bool
|
||||
}
|
||||
|
||||
enum SnapshotBackendKind: String, CaseIterable {
|
||||
case recursiveTree = "tree"
|
||||
case querySweep = "queries"
|
||||
case privateAX = "private-ax"
|
||||
|
||||
var usesXCTestAccessibilityChannel: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .querySweep:
|
||||
return true
|
||||
case .privateAX:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/// The raw projection is the acquired tree, so only a backend that enumerates a hierarchy can
|
||||
/// serve it. The query sweep answers an interactive element query: it has no hierarchy to
|
||||
/// return, and planning it for `--raw` is exactly how a raw request gets answered with regular
|
||||
/// membership (#1797 D4).
|
||||
var supportsRawProjection: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .privateAX:
|
||||
return true
|
||||
case .querySweep:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
var isAvailableOnCurrentPlatform: Bool {
|
||||
switch self {
|
||||
case .recursiveTree, .querySweep:
|
||||
return true
|
||||
case .privateAX:
|
||||
#if os(iOS) && targetEnvironment(simulator)
|
||||
return true
|
||||
#else
|
||||
return false
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum SnapshotXCTestChannelPlanState: Equatable {
|
||||
case normal
|
||||
case deferredToIndependentBackend
|
||||
@@ -92,6 +52,9 @@ struct EffectiveSnapshotCapturePlan {
|
||||
let plan: [SnapshotBackendKind]
|
||||
let xCTestChannelState: SnapshotXCTestChannelPlanState
|
||||
let treeCaptureSliceBudgetOverride: TimeInterval?
|
||||
/// Non-nil only when the plan was narrowed by an explicit internal backend preference. This
|
||||
/// keeps the quality marker tied to the plan decision rather than to an untrusted request field.
|
||||
let preferredBackend: SnapshotBackendKind?
|
||||
}
|
||||
|
||||
/// What the plan runner does when every backend failed or stayed sparse.
|
||||
@@ -194,8 +157,15 @@ extension RunnerTests {
|
||||
/// cause that does not exist.
|
||||
static func xcTestChannelStateFirstFailure(
|
||||
_ state: SnapshotXCTestChannelPlanState,
|
||||
requestPinnedBackend: Bool = false
|
||||
requestPinnedBackend: Bool = false,
|
||||
preferredBackend: String? = nil
|
||||
) -> (reason: String, code: String)? {
|
||||
if state == .normal && preferredBackend == SnapshotBackendKind.recursiveTree.rawValue {
|
||||
return (
|
||||
"the recursive XCTest tree backend was explicitly selected for this capture",
|
||||
"requested-backend"
|
||||
)
|
||||
}
|
||||
switch state {
|
||||
case .normal:
|
||||
return nil
|
||||
@@ -229,19 +199,36 @@ extension RunnerTests {
|
||||
penalized || preferredBackend == SnapshotBackendKind.privateAX.rawValue
|
||||
}
|
||||
|
||||
/// Pure plan-reorder rule: a penalized XCTest accessibility channel uses independent backends
|
||||
/// when the platform has one, otherwise it keeps XCTest work on a short probe. The raw
|
||||
/// diagnostic plan keeps tree-first errors, and unknown plans are left untouched.
|
||||
/// Pure plan-reorder rule: an internal preferred backend pins the regular plan to that backend;
|
||||
/// this is the only force seam used by same-backend evidence and conformance captures. A
|
||||
/// penalized XCTest accessibility channel uses independent backends when the platform has one,
|
||||
/// otherwise it keeps XCTest work on a short probe. The raw diagnostic plan keeps tree-first
|
||||
/// errors, and unknown plans are left untouched.
|
||||
static func effectiveSnapshotCapturePlan(
|
||||
_ plan: [SnapshotBackendKind],
|
||||
xCTestChannelPenalized: Bool,
|
||||
availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases)
|
||||
availableBackends: Set<SnapshotBackendKind> = Set(SnapshotBackendKind.allCases),
|
||||
preferredBackend: String? = nil
|
||||
) -> EffectiveSnapshotCapturePlan {
|
||||
if
|
||||
plan == Self.regularVisiblePlan,
|
||||
let preferred = preferredBackend.flatMap(SnapshotBackendKind.init(rawValue:)),
|
||||
preferred.isForceable,
|
||||
availableBackends.contains(preferred)
|
||||
{
|
||||
return EffectiveSnapshotCapturePlan(
|
||||
plan: [preferred],
|
||||
xCTestChannelState: preferred == .privateAX ? .deferredToIndependentBackend : .normal,
|
||||
treeCaptureSliceBudgetOverride: nil,
|
||||
preferredBackend: preferred
|
||||
)
|
||||
}
|
||||
guard xCTestChannelPenalized, plan == Self.regularVisiblePlan else {
|
||||
return EffectiveSnapshotCapturePlan(
|
||||
plan: plan,
|
||||
xCTestChannelState: .normal,
|
||||
treeCaptureSliceBudgetOverride: nil
|
||||
treeCaptureSliceBudgetOverride: nil,
|
||||
preferredBackend: nil
|
||||
)
|
||||
}
|
||||
let availablePlan = plan.filter { availableBackends.contains($0) }
|
||||
@@ -250,13 +237,15 @@ extension RunnerTests {
|
||||
return EffectiveSnapshotCapturePlan(
|
||||
plan: recoveryPlan,
|
||||
xCTestChannelState: .deferredToIndependentBackend,
|
||||
treeCaptureSliceBudgetOverride: nil
|
||||
treeCaptureSliceBudgetOverride: nil,
|
||||
preferredBackend: nil
|
||||
)
|
||||
}
|
||||
return EffectiveSnapshotCapturePlan(
|
||||
plan: availablePlan.filter(\.usesXCTestAccessibilityChannel),
|
||||
xCTestChannelState: .boundedXCTestProbe,
|
||||
treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget
|
||||
treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget,
|
||||
preferredBackend: nil
|
||||
)
|
||||
}
|
||||
|
||||
@@ -298,14 +287,16 @@ extension RunnerTests {
|
||||
let effective = Self.effectiveSnapshotCapturePlan(
|
||||
plan,
|
||||
xCTestChannelPenalized: xCTestChannelPenalized,
|
||||
availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform))
|
||||
availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)),
|
||||
preferredBackend: options.preferredBackend
|
||||
)
|
||||
let effectivePlan = effective.plan
|
||||
// Only a customActions-implied pin is request-pinned; the daemon's
|
||||
// same-backend evidence probe pins for its own reasons and keeps 'deferred'.
|
||||
firstFailure = Self.xcTestChannelStateFirstFailure(
|
||||
effective.xCTestChannelState,
|
||||
requestPinnedBackend: options.customActions && !xCTestChannelPenalizedByBreaker
|
||||
requestPinnedBackend: options.customActions && !xCTestChannelPenalizedByBreaker,
|
||||
preferredBackend: effective.preferredBackend?.rawValue
|
||||
)
|
||||
switch effective.xCTestChannelState {
|
||||
case .normal:
|
||||
@@ -404,7 +395,7 @@ extension RunnerTests {
|
||||
capture,
|
||||
backend: kind,
|
||||
state: recovered ? "recovered" : "healthy",
|
||||
reason: recovered ? firstFailure : nil
|
||||
reason: recovered || firstFailure?.code == "requested-backend" ? firstFailure : nil
|
||||
)
|
||||
}
|
||||
|
||||
@@ -895,12 +886,18 @@ extension RunnerTests {
|
||||
let treated = Self.snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: false, preferredBackend: options.preferredBackend)
|
||||
let pinned = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan, xCTestChannelPenalized: treated)
|
||||
Self.regularVisiblePlan,
|
||||
xCTestChannelPenalized: treated,
|
||||
preferredBackend: options.preferredBackend
|
||||
)
|
||||
XCTAssertEqual(pinned.plan, [.privateAX])
|
||||
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
|
||||
|
||||
let raw = Self.effectiveSnapshotCapturePlan(
|
||||
Self.rawDiagnosticPlan, xCTestChannelPenalized: treated)
|
||||
Self.rawDiagnosticPlan,
|
||||
xCTestChannelPenalized: treated,
|
||||
preferredBackend: options.preferredBackend
|
||||
)
|
||||
XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan)
|
||||
|
||||
// A command without the field decodes to no pin and a normal plan.
|
||||
@@ -909,6 +906,26 @@ extension RunnerTests {
|
||||
XCTAssertNil(Self.presentationOptions(from: bare).preferredBackend)
|
||||
}
|
||||
|
||||
/// #1635: the force seam must select the recursive tree even when the XCTest
|
||||
/// channel is currently penalized. Without the preferred-backend argument,
|
||||
/// this call returns the independent private-AX recovery plan instead.
|
||||
func testPreferredTreeBackendPinsRegularPlanAndLeavesStructuredEvidence() {
|
||||
let forced = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan,
|
||||
xCTestChannelPenalized: true,
|
||||
preferredBackend: SnapshotBackendKind.recursiveTree.rawValue
|
||||
)
|
||||
XCTAssertEqual(forced.plan, [.recursiveTree])
|
||||
XCTAssertEqual(forced.xCTestChannelState, .normal)
|
||||
XCTAssertEqual(
|
||||
Self.xcTestChannelStateFirstFailure(
|
||||
forced.xCTestChannelState,
|
||||
preferredBackend: forced.preferredBackend?.rawValue
|
||||
)?.code,
|
||||
"requested-backend"
|
||||
)
|
||||
}
|
||||
|
||||
/// Same-backend evidence probes: a daemon-pinned private-AX capture takes the
|
||||
/// penalized route even with a healthy channel, so tap-outcome corroboration
|
||||
/// baselines and probes are always captured by the same backend (backends are
|
||||
@@ -927,7 +944,9 @@ extension RunnerTests {
|
||||
let pinned = Self.effectiveSnapshotCapturePlan(
|
||||
Self.regularVisiblePlan,
|
||||
xCTestChannelPenalized: Self.snapshotXCTestChannelTreatedAsPenalized(
|
||||
penalized: false, preferredBackend: "private-ax")
|
||||
penalized: false, preferredBackend: "private-ax"
|
||||
),
|
||||
preferredBackend: "private-ax"
|
||||
)
|
||||
XCTAssertEqual(pinned.plan, [.privateAX])
|
||||
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"screen": "checkout-form-after-text-entry",
|
||||
"minimumNodeCount": 3,
|
||||
"requiredControls": [
|
||||
{
|
||||
"identifier": "field-name",
|
||||
"label": "Full name",
|
||||
"role": "text-field",
|
||||
"value": "Ada Lovelace",
|
||||
"interactive": true
|
||||
},
|
||||
{
|
||||
"identifier": "field-email",
|
||||
"label": "Email",
|
||||
"role": "text-field",
|
||||
"value": "ada@example.test",
|
||||
"interactive": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"backends": [
|
||||
{
|
||||
"name": "tree",
|
||||
"forceable": true,
|
||||
"supportsRawProjection": true,
|
||||
"hittable": "geometric-actionability",
|
||||
"deepExtension": "no",
|
||||
"depthLadder": "n/a",
|
||||
"availability": {
|
||||
"simulator": true,
|
||||
"physicalDevice": true
|
||||
},
|
||||
"knownGaps": [
|
||||
{
|
||||
"id": "deep-extension",
|
||||
"owner": "iOS snapshot maintainers",
|
||||
"trackingIssue": 1635,
|
||||
"expiresOn": "2027-03-31"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "queries",
|
||||
"forceable": false,
|
||||
"supportsRawProjection": false,
|
||||
"hittable": "geometric-actionability",
|
||||
"deepExtension": "n/a",
|
||||
"depthLadder": "n/a",
|
||||
"availability": {
|
||||
"simulator": true,
|
||||
"physicalDevice": true
|
||||
},
|
||||
"knownGaps": []
|
||||
},
|
||||
{
|
||||
"name": "private-ax",
|
||||
"forceable": true,
|
||||
"supportsRawProjection": true,
|
||||
"hittable": "geometric-actionability",
|
||||
"deepExtension": "yes",
|
||||
"depthLadder": "yes",
|
||||
"availability": {
|
||||
"simulator": true,
|
||||
"physicalDevice": false
|
||||
},
|
||||
"knownGaps": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import type { BackMode } from './back-mode.ts';
|
||||
import type { ClickButton } from './click-button.ts';
|
||||
import type { SwipePattern } from './scroll-gesture.ts';
|
||||
import type { DeviceTarget, PlatformSelector } from '@agent-device/kernel/device';
|
||||
import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot';
|
||||
import type {
|
||||
DaemonInstallSource,
|
||||
DaemonServerMode,
|
||||
@@ -74,7 +75,7 @@ export type CliFlags = CloudProviderProfileFields &
|
||||
responseLevel?: ResponseLevel;
|
||||
snapshotInteractiveOnly?: boolean;
|
||||
/** Internal (no CLI flag): pin the capture backend for same-backend evidence probes. */
|
||||
snapshotPreferredBackend?: 'private-ax';
|
||||
snapshotPreferredBackend?: SnapshotPreferredBackend;
|
||||
snapshotDiff?: boolean;
|
||||
snapshotDepth?: number;
|
||||
snapshotScope?: string;
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
*/
|
||||
export type SnapshotCaptureBackend = 'tree' | 'queries' | 'private-ax';
|
||||
|
||||
/** Internal backends that evidence probes may select explicitly. */
|
||||
export type SnapshotPreferredBackend = 'tree' | 'private-ax';
|
||||
|
||||
export type SnapshotQualityVerdict = {
|
||||
state: 'healthy' | 'recovered' | 'sparse';
|
||||
backend: SnapshotCaptureBackend;
|
||||
@@ -67,7 +70,7 @@ export type SnapshotOptions = {
|
||||
* are not comparable views of a screen), so a corroboration probe must be
|
||||
* captured the way its baseline was.
|
||||
*/
|
||||
preferredBackend?: 'private-ax';
|
||||
preferredBackend?: SnapshotPreferredBackend;
|
||||
/**
|
||||
* Read accessibility custom actions for elements that merge their children
|
||||
* away. Opt-in because each such element costs its own accessibility round
|
||||
|
||||
@@ -11,7 +11,7 @@ test('snapshot options map every backend preference into command flags', () => {
|
||||
raw: true,
|
||||
customActions: true,
|
||||
includeHiddenContentHints: true,
|
||||
preferredBackend: 'private-ax',
|
||||
preferredBackend: 'tree',
|
||||
}),
|
||||
{
|
||||
snapshotInteractiveOnly: true,
|
||||
@@ -20,7 +20,7 @@ test('snapshot options map every backend preference into command flags', () => {
|
||||
snapshotRaw: true,
|
||||
snapshotCustomActions: true,
|
||||
snapshotIncludeHiddenContentHints: true,
|
||||
snapshotPreferredBackend: 'private-ax',
|
||||
snapshotPreferredBackend: 'tree',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { snapshotCommandFacet } from './snapshot.ts';
|
||||
|
||||
test('snapshot help classifies iOS hittable as shared geometric actionability', () => {
|
||||
const detail = snapshotCommandFacet.text.cliDetail ?? '';
|
||||
|
||||
for (const backend of ['tree', 'queries', 'private-ax']) {
|
||||
expect(detail).toContain(`${backend}: hittable=geometric-actionability`);
|
||||
}
|
||||
expect(detail).not.toMatch(/hittable=(?:hit-tested|approximated)/);
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { PUBLIC_COMMANDS } from '../../command-catalog.ts';
|
||||
import { SNAPSHOT_BACKEND_CAPABILITIES } from '../../snapshot-quality/backend-capabilities.ts';
|
||||
import { SNAPSHOT_FLAGS } from '../cli-grammar/flag-groups.ts';
|
||||
import { booleanField, integerField, stringField } from '../command-input.ts';
|
||||
import { defineExecutableCommand } from '../command-contract.ts';
|
||||
@@ -17,6 +18,13 @@ const SNAPSHOT_COMMAND_NAME = 'snapshot';
|
||||
const snapshotCommandDescription =
|
||||
'Capture the accessibility tree or compare it with the previous session baseline. Use the returned refs for subsequent semantic interactions and the diff option to verify UI changes.';
|
||||
|
||||
const snapshotBackendCapabilityHelp = Object.entries(SNAPSHOT_BACKEND_CAPABILITIES)
|
||||
.map(([backend, capability]) => {
|
||||
const gaps = capability.knownGaps.map((gap) => `known gap ${gap}`);
|
||||
return `${backend}: hittable=${capability.hittable}, deep-extension=${capability.deepExtension}, depth-ladder=${capability.depthLadder}${gaps.length > 0 ? `, ${gaps.join(', ')}` : ''}`;
|
||||
})
|
||||
.join('; ');
|
||||
|
||||
const snapshotCommandMetadata = defineFieldCommandMetadata(
|
||||
SNAPSHOT_COMMAND_NAME,
|
||||
snapshotCommandDescription,
|
||||
@@ -77,8 +85,7 @@ export const snapshotCommandFacet = defineCommandFacet({
|
||||
name: SNAPSHOT_COMMAND_NAME,
|
||||
text: {
|
||||
summary: 'Capture or diff the accessibility tree',
|
||||
cliDetail:
|
||||
'For iOS raw-coordinate fallback after a no-op ref press, inspect rects with snapshot -i --json, press the rect center, then verify with diff snapshot -i or snapshot --diff.',
|
||||
cliDetail: `For iOS raw-coordinate fallback after a no-op ref press, inspect rects with snapshot -i --json, press the rect center, then verify with diff snapshot -i or snapshot --diff. iOS backend capability contract: ${snapshotBackendCapabilityHelp}.`,
|
||||
},
|
||||
metadata: snapshotCommandMetadata,
|
||||
definition: snapshotCommandDefinition,
|
||||
|
||||
@@ -4,7 +4,11 @@ import type {
|
||||
CommandSessionRecord,
|
||||
} from '../../../runtime-contract.ts';
|
||||
import { AppError } from '@agent-device/kernel/errors';
|
||||
import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
|
||||
import type {
|
||||
SnapshotNode,
|
||||
SnapshotPreferredBackend,
|
||||
SnapshotState,
|
||||
} from '@agent-device/kernel/snapshot';
|
||||
import { findNodeByRef, normalizeRef } from '@agent-device/kernel/snapshot';
|
||||
import { isSparseSnapshotQualityVerdict } from '../../../snapshot-quality/verdict.ts';
|
||||
import { extractReadableText } from '../../../utils/text-surface.ts';
|
||||
@@ -52,7 +56,7 @@ export async function captureSelectorSnapshot(
|
||||
includeRects?: boolean;
|
||||
interactiveOnly?: boolean;
|
||||
includeHiddenContentHints?: boolean;
|
||||
preferredBackend?: 'private-ax';
|
||||
preferredBackend?: SnapshotPreferredBackend;
|
||||
} = {
|
||||
updateSession: true,
|
||||
},
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import type { SnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot';
|
||||
import type {
|
||||
SnapshotNode,
|
||||
SnapshotPreferredBackend,
|
||||
SnapshotQualityVerdict,
|
||||
} from '@agent-device/kernel/snapshot';
|
||||
import { isViewportRootNode } from '@agent-device/contracts/snapshot';
|
||||
import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts';
|
||||
import { now, sleep } from '../../runtime-common.ts';
|
||||
@@ -308,7 +312,7 @@ async function captureStableSignalWithinDeadline(
|
||||
runtime: AgentDeviceRuntime,
|
||||
options: CommandContext & SelectorSnapshotOptions,
|
||||
remainingMs: number,
|
||||
preferredBackend?: 'private-ax',
|
||||
preferredBackend?: SnapshotPreferredBackend,
|
||||
): Promise<CapturedSnapshot | undefined> {
|
||||
const result = await runWithinWaitDeadline(runtime, options, remainingMs, async (signal) => {
|
||||
return await captureSelectorSnapshot(
|
||||
|
||||
@@ -10,7 +10,7 @@ import type {
|
||||
} from '@agent-device/contracts/interaction';
|
||||
import type { RunnerLogicalLeaseContext } from '@agent-device/contracts/platform';
|
||||
import type { SessionSurface } from '@agent-device/contracts/session';
|
||||
import type { Point } from '@agent-device/kernel/snapshot';
|
||||
import type { Point, SnapshotPreferredBackend } from '@agent-device/kernel/snapshot';
|
||||
|
||||
export type DispatchContext = ScreenshotDispatchFlags & {
|
||||
requestId?: string;
|
||||
@@ -32,7 +32,7 @@ export type DispatchContext = ScreenshotDispatchFlags & {
|
||||
runnerLeaseContext?: RunnerLogicalLeaseContext;
|
||||
screenshotCaptureBackend?: 'runner';
|
||||
snapshotInteractiveOnly?: boolean;
|
||||
snapshotPreferredBackend?: 'private-ax';
|
||||
snapshotPreferredBackend?: SnapshotPreferredBackend;
|
||||
snapshotDepth?: number;
|
||||
snapshotScope?: string;
|
||||
snapshotRaw?: boolean;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { CommandFlags } from '@agent-device/contracts/command';
|
||||
import type { SessionStore } from '../session-store.ts';
|
||||
import type { SessionState } from '../types.ts';
|
||||
import type { SnapshotState } from '@agent-device/kernel/snapshot';
|
||||
import type { SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot';
|
||||
import type { ContextFromFlags } from './interaction-common.ts';
|
||||
import { captureSnapshot } from './snapshot-capture.ts';
|
||||
import { setSessionSnapshot } from '../session-snapshot.ts';
|
||||
@@ -15,7 +15,7 @@ export type CaptureSnapshotForSession = (
|
||||
contextFromFlags: ContextFromFlags,
|
||||
options: {
|
||||
interactiveOnly: boolean;
|
||||
preferredBackend?: 'private-ax';
|
||||
preferredBackend?: SnapshotPreferredBackend;
|
||||
androidFreshnessMode?: 'ref-refresh';
|
||||
includeRects?: boolean;
|
||||
signal?: AbortSignal;
|
||||
@@ -29,7 +29,7 @@ export async function captureSnapshotForSession(
|
||||
contextFromFlags: ContextFromFlags,
|
||||
options: {
|
||||
interactiveOnly: boolean;
|
||||
preferredBackend?: 'private-ax';
|
||||
preferredBackend?: SnapshotPreferredBackend;
|
||||
androidFreshnessMode?: 'ref-refresh';
|
||||
includeRects?: boolean;
|
||||
signal?: AbortSignal;
|
||||
|
||||
@@ -196,16 +196,16 @@ test('snapshot accepts only structured healthy empty scope results', async () =>
|
||||
// stops at the dispatch context and the Swift test starts at the parsed
|
||||
// command, so this is the assertion that fails if the interactor stops
|
||||
// forwarding preferredBackend into the emitted RunnerCommand.
|
||||
test('snapshot forwards preferredBackend into the emitted runner command', async () => {
|
||||
test('snapshot forwards either forceable preferredBackend into the emitted runner command', async () => {
|
||||
const calls: RecordedRunnerCall[] = [];
|
||||
const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider(calls));
|
||||
|
||||
await interactor.snapshot({ preferredBackend: 'private-ax' });
|
||||
await interactor.snapshot({ preferredBackend: 'tree' });
|
||||
await interactor.snapshot();
|
||||
|
||||
const snapshots = calls.filter((call) => call.command.command === 'snapshot');
|
||||
assert.equal(snapshots.length, 2);
|
||||
assert.equal(snapshots[0]?.command.preferredBackend, 'private-ax');
|
||||
assert.equal(snapshots[0]?.command.preferredBackend, 'tree');
|
||||
assert.equal(snapshots[1]?.command.preferredBackend, undefined);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors';
|
||||
import crypto from 'node:crypto';
|
||||
import type { DeviceRotation } from '@agent-device/contracts/device';
|
||||
import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot';
|
||||
import type {
|
||||
ClickButton,
|
||||
ElementSelectorKey,
|
||||
@@ -90,7 +91,7 @@ export type RunnerCommand = {
|
||||
fps?: number;
|
||||
interactiveOnly?: boolean;
|
||||
/** Pin the snapshot capture backend (same-backend evidence probes). */
|
||||
preferredBackend?: 'private-ax';
|
||||
preferredBackend?: SnapshotPreferredBackend;
|
||||
/**
|
||||
* Read accessibility custom actions for merged leaves. Opt-in: each element
|
||||
* costs its own AX round trip, and the runner pins the private-AX backend
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { expect, test } from 'vitest';
|
||||
|
||||
import { SNAPSHOT_BACKEND_CAPABILITIES } from './backend-capabilities.ts';
|
||||
|
||||
type SnapshotBackendParityFixture = {
|
||||
backends: Array<{
|
||||
name: string;
|
||||
forceable: boolean;
|
||||
supportsRawProjection: boolean;
|
||||
hittable: string;
|
||||
deepExtension: string;
|
||||
depthLadder: string;
|
||||
availability: { simulator: boolean; physicalDevice: boolean };
|
||||
knownGaps: SnapshotBackendGap[];
|
||||
}>;
|
||||
};
|
||||
|
||||
type SnapshotBackendGap = {
|
||||
id: string;
|
||||
owner: string;
|
||||
trackingIssue: number;
|
||||
expiresOn: string;
|
||||
};
|
||||
|
||||
const SNAPSHOT_BACKEND_PARITY_FIXTURE_PATH = path.resolve(
|
||||
import.meta.dirname,
|
||||
'..',
|
||||
'..',
|
||||
'contracts',
|
||||
'fixtures',
|
||||
'ios-snapshot-backends.json',
|
||||
);
|
||||
|
||||
function readSnapshotBackendParityFixture(): SnapshotBackendParityFixture {
|
||||
return JSON.parse(
|
||||
fs.readFileSync(SNAPSHOT_BACKEND_PARITY_FIXTURE_PATH, 'utf8'),
|
||||
) as SnapshotBackendParityFixture;
|
||||
}
|
||||
|
||||
function validateSnapshotBackendGaps(
|
||||
backends: readonly { name: string; knownGaps: readonly SnapshotBackendGap[] }[],
|
||||
asOf = new Date().toISOString().slice(0, 10),
|
||||
): string[] {
|
||||
const gapIds = new Set<string>();
|
||||
return backends.flatMap((backend) =>
|
||||
backend.knownGaps.flatMap((gap) => [
|
||||
...validateGapId(backend.name, gap.id, gapIds),
|
||||
...validateGapMetadata(backend.name, gap, asOf),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function validateGapId(backend: string, id: string, gapIds: Set<string>): string[] {
|
||||
if (id.trim().length === 0) return [`${backend} gap must have an id`];
|
||||
if (gapIds.has(id)) return [`duplicate snapshot backend gap id ${id}`];
|
||||
gapIds.add(id);
|
||||
return [];
|
||||
}
|
||||
|
||||
function validateGapMetadata(backend: string, gap: SnapshotBackendGap, asOf: string): string[] {
|
||||
return [
|
||||
gap.owner.trim().length === 0 ? `${backend} gap ${gap.id} must name an owner` : undefined,
|
||||
!Number.isInteger(gap.trackingIssue) || gap.trackingIssue <= 0
|
||||
? `${backend} gap ${gap.id} must name a positive tracking issue`
|
||||
: undefined,
|
||||
!/^\d{4}-\d{2}-\d{2}$/.test(gap.expiresOn) || gap.expiresOn <= asOf
|
||||
? `${backend} gap ${gap.id} expiresOn must be after ${asOf}`
|
||||
: undefined,
|
||||
].filter((error): error is string => error !== undefined);
|
||||
}
|
||||
|
||||
test('iOS snapshot registry classifies every backend and conformance target', () => {
|
||||
expect(Object.keys(SNAPSHOT_BACKEND_CAPABILITIES).sort()).toEqual([
|
||||
'private-ax',
|
||||
'queries',
|
||||
'tree',
|
||||
]);
|
||||
expect(
|
||||
Object.entries(SNAPSHOT_BACKEND_CAPABILITIES)
|
||||
.filter(([, capability]) => capability.forceable)
|
||||
.map(([backend]) => backend),
|
||||
).toEqual(['tree', 'private-ax']);
|
||||
|
||||
expect(SNAPSHOT_BACKEND_CAPABILITIES.tree).toMatchObject({
|
||||
forceable: true,
|
||||
supportsRawProjection: true,
|
||||
hittable: 'geometric-actionability',
|
||||
deepExtension: 'no',
|
||||
depthLadder: 'n/a',
|
||||
});
|
||||
expect(SNAPSHOT_BACKEND_CAPABILITIES['private-ax']).toMatchObject({
|
||||
forceable: true,
|
||||
supportsRawProjection: true,
|
||||
hittable: 'geometric-actionability',
|
||||
deepExtension: 'yes',
|
||||
depthLadder: 'yes',
|
||||
});
|
||||
expect(SNAPSHOT_BACKEND_CAPABILITIES.queries).toMatchObject({
|
||||
forceable: false,
|
||||
supportsRawProjection: false,
|
||||
hittable: 'geometric-actionability',
|
||||
});
|
||||
});
|
||||
|
||||
test('Swift and TypeScript snapshot backend declarations match the parity table', () => {
|
||||
const fixture = readSnapshotBackendParityFixture();
|
||||
expect(fixture.backends.map((backend) => backend.name)).toEqual(
|
||||
Object.keys(SNAPSHOT_BACKEND_CAPABILITIES),
|
||||
);
|
||||
for (const backend of fixture.backends) {
|
||||
const capability =
|
||||
SNAPSHOT_BACKEND_CAPABILITIES[backend.name as keyof typeof SNAPSHOT_BACKEND_CAPABILITIES];
|
||||
expect(capability).toMatchObject({
|
||||
forceable: backend.forceable,
|
||||
supportsRawProjection: backend.supportsRawProjection,
|
||||
hittable: backend.hittable,
|
||||
deepExtension: backend.deepExtension,
|
||||
depthLadder: backend.depthLadder,
|
||||
});
|
||||
expect(capability.knownGaps).toEqual(backend.knownGaps.map((gap) => gap.id));
|
||||
}
|
||||
});
|
||||
|
||||
test('snapshot backend gaps require unique, owned, tracked, future-dated metadata', () => {
|
||||
const fixture = readSnapshotBackendParityFixture();
|
||||
expect(validateSnapshotBackendGaps(fixture.backends, '2026-08-21')).toEqual([]);
|
||||
expect(validateSnapshotBackendGaps(fixture.backends)).toEqual([]);
|
||||
|
||||
const tree = fixture.backends.find((backend) => backend.name === 'tree');
|
||||
const deepExtensionGap = tree?.knownGaps.find((gap) => gap.id === 'deep-extension');
|
||||
if (!tree || !deepExtensionGap) throw new Error('tree deep-extension fixture gap is missing');
|
||||
const invalidBackends = fixture.backends.map((backend) =>
|
||||
backend === tree
|
||||
? {
|
||||
...backend,
|
||||
knownGaps: [
|
||||
{
|
||||
...deepExtensionGap,
|
||||
owner: '',
|
||||
trackingIssue: 0,
|
||||
expiresOn: '2026-08-20',
|
||||
},
|
||||
],
|
||||
}
|
||||
: backend,
|
||||
);
|
||||
|
||||
expect(validateSnapshotBackendGaps(invalidBackends, '2026-08-21')).toEqual([
|
||||
'tree gap deep-extension must name an owner',
|
||||
'tree gap deep-extension must name a positive tracking issue',
|
||||
'tree gap deep-extension expiresOn must be after 2026-08-21',
|
||||
]);
|
||||
});
|
||||
|
||||
test('snapshot backend gaps expire on their boundary date', () => {
|
||||
const fixture = readSnapshotBackendParityFixture();
|
||||
const expiringBackends = fixture.backends.map((backend) =>
|
||||
backend.name === 'tree'
|
||||
? {
|
||||
...backend,
|
||||
knownGaps: backend.knownGaps.map((gap) => ({
|
||||
...gap,
|
||||
expiresOn: '2026-08-21',
|
||||
})),
|
||||
}
|
||||
: backend,
|
||||
);
|
||||
|
||||
expect(validateSnapshotBackendGaps(expiringBackends, '2026-08-20')).toEqual([]);
|
||||
expect(validateSnapshotBackendGaps(expiringBackends, '2026-08-21')).toEqual([
|
||||
'tree gap deep-extension expiresOn must be after 2026-08-21',
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import type {
|
||||
SnapshotCaptureBackend,
|
||||
SnapshotPreferredBackend,
|
||||
} from '@agent-device/kernel/snapshot';
|
||||
|
||||
/** #1933: every iOS backend publishes this shared predicate in the wire `hittable` field. */
|
||||
type SnapshotBackendHittable = 'geometric-actionability';
|
||||
type SnapshotBackendSupport = 'yes' | 'no' | 'n/a';
|
||||
|
||||
type SnapshotBackendCapability = {
|
||||
supportsRawProjection: boolean;
|
||||
hittable: SnapshotBackendHittable;
|
||||
deepExtension: SnapshotBackendSupport;
|
||||
depthLadder: SnapshotBackendSupport;
|
||||
knownGaps: readonly string[];
|
||||
};
|
||||
|
||||
type SnapshotBackendCapabilityRegistry = {
|
||||
[Backend in SnapshotCaptureBackend]: SnapshotBackendCapability & {
|
||||
forceable: Backend extends SnapshotPreferredBackend ? true : false;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Internal contract for the iOS snapshot strategies. The mapped registry keeps every backend
|
||||
* classified and makes forceability agree with the request wire type at compile time.
|
||||
*/
|
||||
export const SNAPSHOT_BACKEND_CAPABILITIES = {
|
||||
tree: {
|
||||
forceable: true,
|
||||
supportsRawProjection: true,
|
||||
hittable: 'geometric-actionability',
|
||||
deepExtension: 'no',
|
||||
depthLadder: 'n/a',
|
||||
knownGaps: ['deep-extension'],
|
||||
},
|
||||
queries: {
|
||||
forceable: false,
|
||||
supportsRawProjection: false,
|
||||
hittable: 'geometric-actionability',
|
||||
deepExtension: 'n/a',
|
||||
depthLadder: 'n/a',
|
||||
knownGaps: [],
|
||||
},
|
||||
'private-ax': {
|
||||
forceable: true,
|
||||
supportsRawProjection: true,
|
||||
hittable: 'geometric-actionability',
|
||||
deepExtension: 'yes',
|
||||
depthLadder: 'yes',
|
||||
knownGaps: [],
|
||||
},
|
||||
} as const satisfies SnapshotBackendCapabilityRegistry;
|
||||
@@ -1,15 +1,14 @@
|
||||
import type { SnapshotQualityVerdict } from '@agent-device/kernel/snapshot';
|
||||
import { SNAPSHOT_BACKEND_CAPABILITIES } from './backend-capabilities.ts';
|
||||
|
||||
const SNAPSHOT_QUALITY_STATES = new Set<SnapshotQualityVerdict['state']>([
|
||||
'healthy',
|
||||
'recovered',
|
||||
'sparse',
|
||||
]);
|
||||
const SNAPSHOT_QUALITY_BACKENDS = new Set<SnapshotQualityVerdict['backend']>([
|
||||
'tree',
|
||||
'queries',
|
||||
'private-ax',
|
||||
]);
|
||||
const SNAPSHOT_QUALITY_BACKENDS = new Set<SnapshotQualityVerdict['backend']>(
|
||||
Object.keys(SNAPSHOT_BACKEND_CAPABILITIES) as SnapshotQualityVerdict['backend'][],
|
||||
);
|
||||
const SNAPSHOT_QUALITY_REASON_CODES = new Set<NonNullable<SnapshotQualityVerdict['reasonCode']>>([
|
||||
'ax-rejected',
|
||||
'sparse-tree',
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client';
|
||||
import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts';
|
||||
import { sendToDaemon } from '../../../src/daemon/client/daemon-client.ts';
|
||||
import { assertPngFile } from '../provider-scenarios/assertions.ts';
|
||||
import {
|
||||
assertFilesDiffer,
|
||||
@@ -30,8 +34,35 @@ import {
|
||||
writeCoverageReport,
|
||||
} from './live-harness.ts';
|
||||
import { bindIosSimulatorScenarios } from './scenarios.ts';
|
||||
import {
|
||||
assertSnapshotBackendConformance,
|
||||
createSnapshotBackendConformanceTransport,
|
||||
SNAPSHOT_BACKEND_CONFORMANCE_TARGETS,
|
||||
loadSnapshotBackendConformanceFixture,
|
||||
snapshotBackendEvidence,
|
||||
} from './snapshot-backend-conformance.ts';
|
||||
|
||||
const C = PUBLIC_COMMANDS;
|
||||
|
||||
type AgentDeviceSdk = typeof import('../../../src/sdk/index.ts');
|
||||
|
||||
const sendToDaemonTransport: AgentDeviceDaemonTransport = async (request, context) => {
|
||||
if (request.session === undefined) {
|
||||
throw new Error('Snapshot conformance transport requires an explicit session.');
|
||||
}
|
||||
return await sendToDaemon({ ...request, session: request.session }, context);
|
||||
};
|
||||
|
||||
async function loadBuiltAgentDeviceClient() {
|
||||
// The live harness drives the built CLI, so use the built SDK entry as well. Importing the
|
||||
// source SDK here would intentionally take over the daemon on code-signature mismatch and make
|
||||
// the forced-backend evidence come from a different runtime than the rest of the scenario.
|
||||
const builtSdk = (await import(
|
||||
pathToFileURL(path.resolve('dist/src/index.js')).href
|
||||
)) as AgentDeviceSdk;
|
||||
return builtSdk.createAgentDeviceClient;
|
||||
}
|
||||
|
||||
const LIVE_SCENARIOS = bindIosSimulatorScenarios<LiveContext>({
|
||||
automationInput: assertAutomationInput,
|
||||
captureClose: async (context) => {
|
||||
@@ -225,6 +256,48 @@ async function assertFormInput(context: LiveContext): Promise<void> {
|
||||
C.type,
|
||||
'AX-independent first-responder typing appends a suffix to the coordinate-focused field',
|
||||
);
|
||||
|
||||
await assertSnapshotBackendConformanceLive(context);
|
||||
}
|
||||
|
||||
async function assertSnapshotBackendConformanceLive(context: LiveContext): Promise<void> {
|
||||
await runStep(context, 'dismiss keyboard before backend conformance capture', [
|
||||
'keyboard',
|
||||
'dismiss',
|
||||
]);
|
||||
const fixture = loadSnapshotBackendConformanceFixture();
|
||||
const createAgentDeviceClient = await loadBuiltAgentDeviceClient();
|
||||
const evidence = [];
|
||||
|
||||
for (const backend of SNAPSHOT_BACKEND_CONFORMANCE_TARGETS) {
|
||||
const client = createAgentDeviceClient(
|
||||
{
|
||||
session: context.session,
|
||||
stateDir: context.stateDir,
|
||||
},
|
||||
{ transport: createSnapshotBackendConformanceTransport(backend, sendToDaemonTransport) },
|
||||
);
|
||||
const snapshot = await client.capture.snapshot({
|
||||
interactiveOnly: true,
|
||||
platform: 'ios',
|
||||
udid: context.udid,
|
||||
});
|
||||
assertSnapshotBackendConformance(snapshot, backend, fixture);
|
||||
if (backend === 'tree') {
|
||||
assert.equal(
|
||||
snapshot.snapshotQuality?.reasonCode,
|
||||
'requested-backend',
|
||||
'tree conformance capture must disclose that the force seam was honored',
|
||||
);
|
||||
}
|
||||
evidence.push(snapshotBackendEvidence(snapshot, backend));
|
||||
}
|
||||
|
||||
const evidencePath = path.join(context.artifactDir, 'snapshot-backend-conformance.json');
|
||||
fs.writeFileSync(
|
||||
evidencePath,
|
||||
JSON.stringify({ fixture: fixture.screen, captures: evidence }, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
async function assertCapture(context: LiveContext): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { CaptureSnapshotResult } from '@agent-device/contracts/client';
|
||||
|
||||
import type { SnapshotQualityVerdict } from '@agent-device/kernel/snapshot';
|
||||
|
||||
export type SnapshotBackendConformanceInput = Pick<
|
||||
CaptureSnapshotResult,
|
||||
'nodes' | 'snapshotQuality' | 'truncated'
|
||||
>;
|
||||
|
||||
export function buildSnapshotBackendConformanceBase(): SnapshotBackendConformanceInput {
|
||||
return {
|
||||
truncated: false,
|
||||
snapshotQuality: { state: 'healthy', backend: 'tree' } satisfies SnapshotQualityVerdict,
|
||||
nodes: [
|
||||
{
|
||||
index: 0,
|
||||
ref: 'e1',
|
||||
identifier: 'field-name',
|
||||
label: 'Full name',
|
||||
type: 'TextField',
|
||||
value: 'Ada Lovelace',
|
||||
enabled: true,
|
||||
hittable: true,
|
||||
rect: { x: 0, y: 0, width: 100, height: 20 },
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
ref: 'e2',
|
||||
identifier: 'field-email',
|
||||
label: 'Email',
|
||||
type: 'TextField',
|
||||
value: 'ada@example.test',
|
||||
enabled: true,
|
||||
hittable: true,
|
||||
rect: { x: 0, y: 20, width: 100, height: 20 },
|
||||
},
|
||||
{ index: 2, ref: 'e3', type: 'ScrollView' },
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import type {
|
||||
AgentDeviceDaemonTransport,
|
||||
CaptureSnapshotResult,
|
||||
} from '@agent-device/contracts/client';
|
||||
import { normalizeType } from '@agent-device/contracts/snapshot';
|
||||
import {
|
||||
type SnapshotCaptureBackend,
|
||||
type SnapshotPreferredBackend,
|
||||
} from '@agent-device/kernel/snapshot';
|
||||
import { SNAPSHOT_BACKEND_CAPABILITIES } from '../../../src/snapshot-quality/backend-capabilities.ts';
|
||||
import { isSemanticTouchTarget } from '../../../src/core/interaction-targeting.ts';
|
||||
|
||||
export type SnapshotBackendConformanceFixture = {
|
||||
screen: string;
|
||||
minimumNodeCount: number;
|
||||
requiredControls: readonly SnapshotBackendControl[];
|
||||
};
|
||||
|
||||
type SnapshotBackendControl = {
|
||||
identifier: string;
|
||||
label: string;
|
||||
role: string;
|
||||
value?: string;
|
||||
interactive: boolean;
|
||||
};
|
||||
|
||||
export const SNAPSHOT_BACKEND_CONFORMANCE_TARGETS = Object.entries(SNAPSHOT_BACKEND_CAPABILITIES)
|
||||
.filter(([, capability]) => capability.forceable)
|
||||
.map(([backend]) => backend as SnapshotPreferredBackend);
|
||||
|
||||
/**
|
||||
* Test-owned transport seam for the backend conformance probe. The public SDK deliberately has
|
||||
* no backend-selection option; this wrapper adds the internal daemon flag after the public client
|
||||
* has projected its ordinary snapshot request. Keeping the force field here makes it impossible
|
||||
* for a published CaptureSnapshotOptions or generic CommandExecutionOptions value to leak this
|
||||
* evidence-only control.
|
||||
*/
|
||||
export function createSnapshotBackendConformanceTransport(
|
||||
backend: SnapshotPreferredBackend,
|
||||
transport: AgentDeviceDaemonTransport,
|
||||
): AgentDeviceDaemonTransport {
|
||||
return async (request, context) =>
|
||||
await transport(
|
||||
{
|
||||
...request,
|
||||
flags: {
|
||||
...(request.flags ?? {}),
|
||||
snapshotPreferredBackend: backend,
|
||||
},
|
||||
},
|
||||
context,
|
||||
);
|
||||
}
|
||||
|
||||
export function loadSnapshotBackendConformanceFixture(
|
||||
fixturePath = path.resolve('contracts/fixtures/ios-snapshot-backend-conformance.json'),
|
||||
): SnapshotBackendConformanceFixture {
|
||||
return JSON.parse(fs.readFileSync(fixturePath, 'utf8')) as SnapshotBackendConformanceFixture;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks one backend against the fixture contract. It deliberately never accepts a second
|
||||
* snapshot as a comparison oracle: a backend passes only by satisfying the seeded controls,
|
||||
* semantic identity, interactivity, values, and its own quality verdict.
|
||||
*/
|
||||
export function assertSnapshotBackendConformance(
|
||||
snapshot: Pick<CaptureSnapshotResult, 'nodes' | 'snapshotQuality' | 'truncated'>,
|
||||
backend: SnapshotPreferredBackend,
|
||||
fixture: SnapshotBackendConformanceFixture,
|
||||
): void {
|
||||
const quality = snapshot.snapshotQuality;
|
||||
assert.equal(
|
||||
quality?.backend,
|
||||
backend,
|
||||
`${backend} capture must prove its backend in snapshotQuality: ${JSON.stringify(quality)}`,
|
||||
);
|
||||
assert.ok(
|
||||
quality?.state === 'healthy' || quality?.state === 'recovered',
|
||||
`${backend} capture must have a non-sparse quality verdict: ${JSON.stringify(quality)}`,
|
||||
);
|
||||
// The existing wire contract marks any recovered capture as truncated, including a complete
|
||||
// private-AX payload selected after the XCTest channel was deferred. Assert that relationship
|
||||
// instead of conflating recovery provenance with missing fixture controls.
|
||||
assert.equal(
|
||||
snapshot.truncated,
|
||||
quality.state !== 'healthy',
|
||||
`${backend} quality/truncation flags disagree: ${JSON.stringify(quality)}`,
|
||||
);
|
||||
assert.ok(
|
||||
snapshot.nodes.length >= fixture.minimumNodeCount,
|
||||
`${backend} capture returned too few nodes (${snapshot.nodes.length} < ${fixture.minimumNodeCount})`,
|
||||
);
|
||||
|
||||
for (const expected of fixture.requiredControls) {
|
||||
const node = snapshot.nodes.find((candidate) => candidate.identifier === expected.identifier);
|
||||
assert.ok(
|
||||
node,
|
||||
`${backend} capture is missing ${expected.identifier}: ${JSON.stringify(snapshot.nodes)}`,
|
||||
);
|
||||
assert.equal(node.label, expected.label, `${backend} label drift for ${expected.identifier}`);
|
||||
assert.equal(
|
||||
canonicalRole(node.role ?? node.type ?? ''),
|
||||
expected.role,
|
||||
`${backend} role drift for ${expected.identifier}`,
|
||||
);
|
||||
assert.equal(node.enabled, true, `${backend} did not mark ${expected.identifier} enabled`);
|
||||
if (expected.interactive) {
|
||||
// #1933 makes iOS snapshot hittable a backend-independent geometric-actionability
|
||||
// predicate: enabled, non-empty geometry whose center lies inside the viewport. It is not
|
||||
// native hit-testing or occlusion evidence, but it is still a promised control invariant.
|
||||
assert.equal(
|
||||
isSemanticTouchTarget(node),
|
||||
true,
|
||||
`${backend} did not expose ${expected.identifier} as a semantic control`,
|
||||
);
|
||||
assert.ok(
|
||||
node.rect && node.rect.width > 0 && node.rect.height > 0,
|
||||
`${backend} did not expose positive interaction geometry for ${expected.identifier}`,
|
||||
);
|
||||
assert.equal(
|
||||
typeof node.hittable,
|
||||
'boolean',
|
||||
`${backend} omitted its structured hittable result for ${expected.identifier}`,
|
||||
);
|
||||
assert.equal(
|
||||
node.hittable,
|
||||
true,
|
||||
`${backend} did not expose ${expected.identifier} as geometrically actionable`,
|
||||
);
|
||||
}
|
||||
if (expected.value !== undefined) {
|
||||
assert.equal(node.value, expected.value, `${backend} value drift for ${expected.identifier}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalRole(value: string): string {
|
||||
const normalized = normalizeType(value);
|
||||
return (
|
||||
(
|
||||
{
|
||||
edittext: 'text-field',
|
||||
textfield: 'text-field',
|
||||
textarea: 'text-view',
|
||||
} as Record<string, string>
|
||||
)[normalized] ?? normalized
|
||||
);
|
||||
}
|
||||
|
||||
export function snapshotBackendEvidence(
|
||||
snapshot: Pick<CaptureSnapshotResult, 'nodes' | 'snapshotQuality' | 'truncated'>,
|
||||
backend: SnapshotCaptureBackend,
|
||||
) {
|
||||
return {
|
||||
backend,
|
||||
quality: snapshot.snapshotQuality,
|
||||
nodeCount: snapshot.nodes.length,
|
||||
truncated: snapshot.truncated,
|
||||
controls: snapshot.nodes
|
||||
.filter((node) => typeof node.identifier === 'string')
|
||||
.map((node) => ({
|
||||
identifier: node.identifier,
|
||||
label: node.label,
|
||||
role: canonicalRole(node.role ?? node.type ?? ''),
|
||||
value: node.value,
|
||||
enabled: node.enabled,
|
||||
hittable: node.hittable,
|
||||
})),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
assertSnapshotBackendConformance,
|
||||
createSnapshotBackendConformanceTransport,
|
||||
loadSnapshotBackendConformanceFixture,
|
||||
} from './ios-simulator-e2e/snapshot-backend-conformance.ts';
|
||||
import {
|
||||
buildSnapshotBackendConformanceBase,
|
||||
type SnapshotBackendConformanceInput,
|
||||
} from './ios-simulator-e2e/snapshot-backend-conformance-fixtures.ts';
|
||||
|
||||
const fixture = loadSnapshotBackendConformanceFixture();
|
||||
|
||||
test('snapshot backend conformance checks each backend contract independently', () => {
|
||||
const base = buildSnapshotBackendConformanceBase();
|
||||
|
||||
assert.doesNotThrow(() => assertSnapshotBackendConformance(base, 'tree', fixture));
|
||||
assert.throws(
|
||||
() =>
|
||||
assertSnapshotBackendConformance(
|
||||
{ ...base, snapshotQuality: { state: 'healthy', backend: 'tree' } },
|
||||
'private-ax',
|
||||
fixture,
|
||||
),
|
||||
/private-ax capture must prove its backend/,
|
||||
);
|
||||
});
|
||||
|
||||
test('snapshot backend conformance rejects every promised control invariant', () => {
|
||||
const base = buildSnapshotBackendConformanceBase();
|
||||
const updateControl = (identifier: string, update: Record<string, unknown>) =>
|
||||
base.nodes.map((node) => (node.identifier === identifier ? { ...node, ...update } : node));
|
||||
const expectFailure = (
|
||||
name: string,
|
||||
snapshot: SnapshotBackendConformanceInput,
|
||||
message: RegExp,
|
||||
) =>
|
||||
assert.throws(() => assertSnapshotBackendConformance(snapshot, 'tree', fixture), message, name);
|
||||
|
||||
expectFailure(
|
||||
'sparse quality',
|
||||
{ ...base, snapshotQuality: { state: 'sparse', backend: 'tree' } },
|
||||
/must have a non-sparse quality verdict/,
|
||||
);
|
||||
expectFailure(
|
||||
'recovered/truncated mismatch',
|
||||
{ ...base, truncated: true },
|
||||
/quality\/truncation/,
|
||||
);
|
||||
expectFailure('minimum node count', { ...base, nodes: base.nodes.slice(0, 2) }, /too few nodes/);
|
||||
expectFailure(
|
||||
'seeded control presence',
|
||||
{
|
||||
...base,
|
||||
nodes: [
|
||||
...base.nodes.filter((node) => node.identifier !== 'field-email'),
|
||||
{ index: 3, ref: 'e4', type: 'Other' },
|
||||
],
|
||||
},
|
||||
/missing field-email/,
|
||||
);
|
||||
expectFailure(
|
||||
'label identity',
|
||||
{ ...base, nodes: updateControl('field-name', { label: 'Name' }) },
|
||||
/label drift/,
|
||||
);
|
||||
expectFailure(
|
||||
'canonical role identity',
|
||||
{ ...base, nodes: updateControl('field-name', { type: 'StaticText' }) },
|
||||
/role drift/,
|
||||
);
|
||||
expectFailure(
|
||||
'enabled interactivity',
|
||||
{ ...base, nodes: updateControl('field-name', { enabled: false }) },
|
||||
/did not mark field-name enabled/,
|
||||
);
|
||||
expectFailure(
|
||||
'semantic interactivity',
|
||||
{ ...base, nodes: updateControl('field-name', { type: 'StaticText', role: 'text-field' }) },
|
||||
/did not expose field-name as a semantic control/,
|
||||
);
|
||||
expectFailure(
|
||||
'positive interaction geometry',
|
||||
{ ...base, nodes: updateControl('field-name', { rect: { x: 0, y: 0, width: 0, height: 20 } }) },
|
||||
/did not expose positive interaction geometry/,
|
||||
);
|
||||
expectFailure(
|
||||
'structured hittable evidence',
|
||||
{ ...base, nodes: updateControl('field-name', { hittable: undefined }) },
|
||||
/omitted its structured hittable result/,
|
||||
);
|
||||
expectFailure(
|
||||
'geometric actionability',
|
||||
{ ...base, nodes: updateControl('field-name', { hittable: false }) },
|
||||
/did not expose field-name as geometrically actionable/,
|
||||
);
|
||||
expectFailure(
|
||||
'seeded field value',
|
||||
{ ...base, nodes: updateControl('field-email', { value: 'wrong@example.test' }) },
|
||||
/value drift/,
|
||||
);
|
||||
});
|
||||
|
||||
test('backend forcing stays in the test-owned daemon transport seam', async () => {
|
||||
type Request = Parameters<import('@agent-device/contracts/client').AgentDeviceDaemonTransport>[0];
|
||||
let received: Request | undefined;
|
||||
const transport = createSnapshotBackendConformanceTransport('tree', async (request) => {
|
||||
received = request;
|
||||
return { ok: true, data: {} };
|
||||
});
|
||||
|
||||
await transport({ command: 'snapshot', positionals: [], session: 'default', flags: {} });
|
||||
|
||||
assert.equal(received?.flags?.snapshotPreferredBackend, 'tree');
|
||||
assert.equal(
|
||||
(received?.flags as Record<string, unknown> | undefined)?.preferredBackend,
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user