mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
ab0c7a4328
* fix(ios): clip a scroll's swipe above the keyboard, refuse when it cannot The runner owns the live keyboard frame, so it does the clip and reports what it left: a scroll answers with `keyboardAvoided` and `keyboardMinY` beside its plan, and refuses with `SCROLL_KEYBOARD_OCCLUDES_SURFACE` when the keys leave too little band to swipe in instead of flinging into them. It never dismisses the keyboard, which would drop focus and mutate state that session-action provenance does not record. Scroll's keyboard policy moves to `requiredWhenAvailable`. The probe costs a live AX fetch, but gating it on a healthy tree left the first scroll of a session swiping under the keys, which is the failure this is for. Every scroll logs its decision, including the two ways it avoids reading the keyboard at all. Scroll no longer shares `frameAvoidingKeyboard`, whose 25% fail-open was a tap-reference-frame rule; that path is unchanged for its remaining callers. * chore(gates): run the scroll viewport policy tests on the iOS lane The parity table only detects drift if both halves run in CI. Two of these three were reachable by no lane, so the Swift half of the table was a local assertion. * fix(ios): keep the keyboard clip out of the scroll's rotation basis `resolvedScrollViewport` handed the command one frame for both jobs, and the coordinate rotation reads a frame's HEIGHT to map a `landscapeRight` native x. Clipping an 834pt landscape viewport to 576pt therefore moved the dispatched gesture 258pt sideways off the lane the plan had just been built for: the clip fixed the keyboard and broke the gesture. The resolved viewport now names both frames, and the gesture comes from one dispatch decision, so the band the plan is planned inside and the frame its coordinates rotate against cannot be swapped. The landscape case asserts through that decision and fails on the swap. * fix(ios): report a scroll's clipped band in its response
1397 lines
48 KiB
Swift
1397 lines
48 KiB
Swift
import XCTest
|
|
import AgentDeviceSnapshotPresentation
|
|
|
|
#if os(macOS)
|
|
import CoreGraphics
|
|
#endif
|
|
|
|
private struct RunnerUnsupportedOperationError: LocalizedError {
|
|
let message: String
|
|
|
|
var errorDescription: String? { message }
|
|
}
|
|
|
|
enum RunnerInterfaceOrientation {
|
|
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
|
|
static let unknown = 0
|
|
#endif
|
|
static let portrait = 1
|
|
static let portraitUpsideDown = 2
|
|
static let landscapeRight = 3
|
|
static let landscapeLeft = 4
|
|
}
|
|
|
|
extension RunnerTests {
|
|
enum PlannedGestureExecution: Equatable {
|
|
case fastSwipe
|
|
case sampled
|
|
}
|
|
|
|
enum SynthesizedDragProfile: Equatable {
|
|
case continuous
|
|
case controlledScroll
|
|
case fastSwipe
|
|
}
|
|
|
|
func scrollDragProfile(
|
|
releaseBehavior: ScrollReleaseBehavior?
|
|
) -> SynthesizedDragProfile {
|
|
releaseBehavior == .inertial ? .fastSwipe : .controlledScroll
|
|
}
|
|
|
|
struct TouchVisualizationFrame {
|
|
let x: Double
|
|
let y: Double
|
|
let referenceWidth: Double
|
|
let referenceHeight: Double
|
|
}
|
|
|
|
struct DragVisualizationFrame {
|
|
let x: Double
|
|
let y: Double
|
|
let x2: Double
|
|
let y2: Double
|
|
let referenceWidth: Double
|
|
let referenceHeight: Double
|
|
}
|
|
|
|
struct DragPoints {
|
|
let x: Double
|
|
let y: Double
|
|
let x2: Double
|
|
let y2: Double
|
|
}
|
|
|
|
struct SynthesizedDragPlan {
|
|
let points: DragPoints
|
|
let context: SynthesizedCoordinateContext
|
|
|
|
var referenceFrame: CGRect {
|
|
context.referenceFrame
|
|
}
|
|
}
|
|
|
|
struct SelectorElementMatch {
|
|
let element: XCUIElement?
|
|
let isAmbiguous: Bool
|
|
let usedNonHittableFallback: Bool
|
|
}
|
|
|
|
func performBackGesture(app: XCUIApplication) {
|
|
if pressTvRemote(.menu) {
|
|
return
|
|
}
|
|
performCoordinateBackGesture(app: app)
|
|
}
|
|
|
|
private func performCoordinateBackGesture(app: XCUIApplication) {
|
|
#if !os(tvOS)
|
|
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
|
|
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
start.press(forDuration: 0.05, thenDragTo: end)
|
|
#endif
|
|
}
|
|
|
|
func performSystemBackAction(app: XCUIApplication) -> Bool {
|
|
#if os(macOS)
|
|
return false
|
|
#else
|
|
if pressTvRemote(.menu) {
|
|
return true
|
|
}
|
|
performBackGesture(app: app)
|
|
return true
|
|
#endif
|
|
}
|
|
|
|
func performAppSwitcherGesture(app: XCUIApplication) {
|
|
if pressTvRemote(.home) {
|
|
sleepFor(resolveTvRemoteDoublePressDelay())
|
|
_ = pressTvRemote(.home)
|
|
return
|
|
}
|
|
performCoordinateAppSwitcherGesture(app: app)
|
|
}
|
|
|
|
private func performCoordinateAppSwitcherGesture(app: XCUIApplication) {
|
|
#if !os(tvOS)
|
|
let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
|
|
let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
|
|
let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
|
|
start.press(forDuration: 0.6, thenDragTo: end)
|
|
#endif
|
|
}
|
|
|
|
func pressHomeButton() {
|
|
#if os(macOS)
|
|
return
|
|
#else
|
|
if pressTvRemote(.home) {
|
|
return
|
|
}
|
|
XCUIDevice.shared.press(.home)
|
|
#endif
|
|
}
|
|
|
|
func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
|
|
let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
|
|
let element = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
return element.exists ? element : nil
|
|
}
|
|
|
|
func findElement(
|
|
app: XCUIApplication,
|
|
selectorKey: String,
|
|
selectorValue: String,
|
|
allowNonHittableFallback: Bool = false,
|
|
expectedPoint: CGPoint? = nil,
|
|
rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches
|
|
) -> SelectorElementMatch {
|
|
let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
guard !value.isEmpty else {
|
|
return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
|
|
}
|
|
let predicate: NSPredicate
|
|
switch selectorKey {
|
|
case "id":
|
|
predicate = NSPredicate(format: "identifier ==[c] %@", value)
|
|
case "label":
|
|
predicate = NSPredicate(format: "label ==[c] %@", value)
|
|
case "value":
|
|
predicate = NSPredicate(format: "value ==[c] %@", value)
|
|
case "text":
|
|
predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value)
|
|
default:
|
|
return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
|
|
}
|
|
|
|
let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
|
|
.filter(\.exists)
|
|
let facts = matches.map { element in
|
|
SelectorCandidateFacts(
|
|
isHittable: element.isHittable,
|
|
hasTappableFrame: hasTappableFrame(app: app, element: element),
|
|
containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true
|
|
)
|
|
}
|
|
switch classifyDirectSelectorCandidates(
|
|
facts,
|
|
allowNonHittableFallback: allowNonHittableFallback,
|
|
filtersByExpectedPoint: expectedPoint != nil,
|
|
rawMatchPolicy: rawMatchPolicy
|
|
) {
|
|
case .noMatch:
|
|
return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
|
|
case .ambiguous:
|
|
return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
|
|
case let .selected(index, usedNonHittableFallback):
|
|
return SelectorElementMatch(
|
|
element: matches[index],
|
|
isAmbiguous: false,
|
|
usedNonHittableFallback: usedNonHittableFallback
|
|
)
|
|
}
|
|
}
|
|
|
|
// Maestro-compat gate for the non-hittable coordinate fallback: an element
|
|
// with no frame at all cannot be coordinate-tapped, otherwise the decision
|
|
// is the shared TapPointPolicy center-in-frame rule (golden parity table
|
|
// with the TS twin). app.frame is the frame source here — replay taps
|
|
// resolved bounds Maestro-style, so the union frame is intentional.
|
|
private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool {
|
|
let frame = element.frame
|
|
if frame.isEmpty {
|
|
return false
|
|
}
|
|
return TapPointPolicy.isAllowed(elementFrame: frame, windowFrame: app.frame)
|
|
}
|
|
|
|
// The tappable on-screen viewport. app.frame is unsuitable: it unions
|
|
// transformed subtrees, so a closed drawer at negative x inflates it and
|
|
// out-of-window coordinates still pass containment. Falls back to app.frame
|
|
// when no window frame is readable.
|
|
func onScreenWindowFrame(app: XCUIApplication) -> CGRect {
|
|
let window = app.windows.element(boundBy: 0)
|
|
if window.exists {
|
|
let frame = window.frame
|
|
if !frame.isEmpty {
|
|
return frame
|
|
}
|
|
}
|
|
return app.frame
|
|
}
|
|
|
|
func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
|
|
// querySelector is a read — it backs get/is/wait and the offscreen-refusal
|
|
// double-check, none of which mutate. The fail-closed raw-match rule exists
|
|
// to stop a mutation acting on an unseen duplicate; applying it here would
|
|
// instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH
|
|
// for readers that previously resolved the hittable element.
|
|
let match = findElement(
|
|
app: app,
|
|
selectorKey: selectorKey,
|
|
selectorValue: selectorValue,
|
|
rawMatchPolicy: .preferHittableMatch
|
|
)
|
|
if match.isAmbiguous {
|
|
return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
|
|
}
|
|
guard let element = match.element else {
|
|
return Response(ok: true, data: DataPayload(found: false, nodes: []))
|
|
}
|
|
|
|
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let valueText = String(describing: element.value ?? "")
|
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let node = SnapshotPresentation.singleElementRead(
|
|
RawAXNode(
|
|
index: 0,
|
|
type: elementTypeName(element.elementType),
|
|
label: label.isEmpty ? nil : label,
|
|
identifier: identifier.isEmpty ? nil : identifier,
|
|
value: valueText.isEmpty ? nil : valueText,
|
|
rect: snapshotRect(from: element.frame),
|
|
enabled: element.isEnabled,
|
|
focused: nil,
|
|
selected: element.isSelected ? true : nil,
|
|
hittable: element.isHittable,
|
|
depth: 0,
|
|
parentIndex: nil,
|
|
hiddenContentAbove: nil,
|
|
hiddenContentBelow: nil
|
|
)
|
|
)
|
|
return Response(
|
|
ok: true,
|
|
data: DataPayload(
|
|
text: readableText(for: element),
|
|
found: true,
|
|
nodes: [node]
|
|
)
|
|
)
|
|
}
|
|
|
|
/// Shared ordering for point-hit candidates: smallest area wins, then top-to-bottom,
|
|
/// left-to-right, then stable element-type order for ties.
|
|
func smallestElementFirst(_ left: XCUIElement, _ right: XCUIElement) -> Bool {
|
|
let leftArea = max(1, left.frame.width * left.frame.height)
|
|
let rightArea = max(1, right.frame.width * right.frame.height)
|
|
if leftArea != rightArea {
|
|
return leftArea < rightArea
|
|
}
|
|
if left.frame.minY != right.frame.minY {
|
|
return left.frame.minY < right.frame.minY
|
|
}
|
|
if left.frame.minX != right.frame.minX {
|
|
return left.frame.minX < right.frame.minX
|
|
}
|
|
return left.elementType.rawValue < right.elementType.rawValue
|
|
}
|
|
|
|
func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
|
|
let point = CGPoint(x: x, y: y)
|
|
let textInputCandidates = textInputCandidatesAt(app: app, point: point)
|
|
for element in textInputCandidates where prefersExpandedTextRead(element) {
|
|
if let text = readableText(for: element) {
|
|
return text
|
|
}
|
|
}
|
|
|
|
let candidates = app.descendants(matching: .any).allElementsBoundByIndex
|
|
.filter { element in
|
|
element.exists && !element.frame.isEmpty && element.frame.contains(point)
|
|
}
|
|
.sorted(by: smallestElementFirst)
|
|
|
|
for element in candidates where prefersExpandedTextRead(element) {
|
|
if let text = readableText(for: element) {
|
|
return text
|
|
}
|
|
}
|
|
for element in candidates {
|
|
if let text = readableText(for: element) {
|
|
return text
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private func readableText(for element: XCUIElement) -> String? {
|
|
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let valueText = String(describing: element.value ?? "")
|
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
switch element.elementType {
|
|
case .textField, .secureTextField, .searchField, .textView:
|
|
if !valueText.isEmpty { return valueText }
|
|
if !label.isEmpty { return label }
|
|
return identifier.isEmpty ? nil : identifier
|
|
default:
|
|
if !label.isEmpty { return label }
|
|
if !valueText.isEmpty { return valueText }
|
|
return identifier.isEmpty ? nil : identifier
|
|
}
|
|
}
|
|
|
|
private func prefersExpandedTextRead(_ element: XCUIElement) -> Bool {
|
|
switch element.elementType {
|
|
case .textField, .secureTextField, .searchField, .textView:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
|
|
if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") {
|
|
return outcome
|
|
}
|
|
return performCoordinateTap(app: app, x: x, y: y)
|
|
}
|
|
|
|
func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws {
|
|
#if os(macOS)
|
|
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
switch button {
|
|
case "primary":
|
|
coordinate.tap()
|
|
case "secondary":
|
|
coordinate.rightClick()
|
|
case "middle":
|
|
throw RunnerUnsupportedOperationError(message: "middle mouse button is not supported")
|
|
default:
|
|
throw RunnerUnsupportedOperationError(message: "unsupported mouse button: \(button)")
|
|
}
|
|
#elseif os(tvOS)
|
|
throw RunnerUnsupportedOperationError(message: "mouseClick is not supported on tvOS")
|
|
#else
|
|
throw RunnerUnsupportedOperationError(message: "mouseClick is only supported on macOS")
|
|
#endif
|
|
}
|
|
|
|
func desktopScrollAt(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
direction: RunnerScrollDirection,
|
|
pixels: Double,
|
|
durationMs: Double?
|
|
) throws {
|
|
#if os(macOS)
|
|
let events = desktopScrollWheelDeltaEvents(
|
|
direction: direction,
|
|
pixels: pixels,
|
|
durationMs: durationMs
|
|
)
|
|
|
|
let coordinate = interactionCoordinate(app: app, x: x, y: y)
|
|
let interval = desktopScrollEventIntervalSeconds(durationMs: durationMs, eventCount: events.count)
|
|
for (index, deltas) in events.enumerated() {
|
|
// Keep desktop scrolling on XCTest's coordinate API so macOS owns wheel synthesis, natural
|
|
// scrolling preference handling, and cursor placement instead of posting raw CGEvents.
|
|
coordinate.scroll(
|
|
byDeltaX: CGFloat(deltas.horizontal),
|
|
deltaY: CGFloat(deltas.vertical)
|
|
)
|
|
if interval > 0 && index < events.count - 1 {
|
|
Thread.sleep(forTimeInterval: interval)
|
|
}
|
|
}
|
|
#elseif os(tvOS)
|
|
throw RunnerUnsupportedOperationError(message: "desktopScroll is not supported on tvOS")
|
|
#else
|
|
throw RunnerUnsupportedOperationError(message: "desktopScroll is only supported on macOS")
|
|
#endif
|
|
}
|
|
|
|
func desktopScrollWheelDeltas(
|
|
direction: RunnerScrollDirection,
|
|
pixels: Double
|
|
) -> (vertical: Int32, horizontal: Int32) {
|
|
let magnitude = Int32(max(1, min(Double(Int32.max), pixels.rounded())))
|
|
switch direction {
|
|
case .up:
|
|
return (vertical: magnitude, horizontal: 0)
|
|
case .down:
|
|
return (vertical: -magnitude, horizontal: 0)
|
|
case .left:
|
|
return (vertical: 0, horizontal: magnitude)
|
|
case .right:
|
|
return (vertical: 0, horizontal: -magnitude)
|
|
}
|
|
}
|
|
|
|
func desktopScrollWheelDeltaEvents(
|
|
direction: RunnerScrollDirection,
|
|
pixels: Double,
|
|
durationMs: Double?
|
|
) -> [(vertical: Int32, horizontal: Int32)] {
|
|
let totalDeltas = desktopScrollWheelDeltas(direction: direction, pixels: pixels)
|
|
let magnitude = max(abs(Int(totalDeltas.vertical)), abs(Int(totalDeltas.horizontal)))
|
|
let duration = max(0, durationMs ?? 0)
|
|
let requestedEventCount = duration > 0 ? Int(ceil(duration / 16.0)) : 1
|
|
let eventCount = max(1, min(magnitude, requestedEventCount))
|
|
guard eventCount > 1 else {
|
|
return [totalDeltas]
|
|
}
|
|
|
|
if totalDeltas.vertical != 0 {
|
|
return distributeDesktopScrollDelta(totalDeltas.vertical, eventCount: eventCount)
|
|
.map { (vertical: $0, horizontal: 0) }
|
|
}
|
|
return distributeDesktopScrollDelta(totalDeltas.horizontal, eventCount: eventCount)
|
|
.map { (vertical: 0, horizontal: $0) }
|
|
}
|
|
|
|
func desktopScrollEventIntervalSeconds(durationMs: Double?, eventCount: Int) -> TimeInterval {
|
|
guard let durationMs, durationMs > 0, eventCount > 1 else { return 0 }
|
|
return (durationMs / 1000.0) / Double(eventCount - 1)
|
|
}
|
|
|
|
private func distributeDesktopScrollDelta(_ delta: Int32, eventCount: Int) -> [Int32] {
|
|
let sign: Int32 = delta < 0 ? -1 : 1
|
|
let magnitude = abs(Int(delta))
|
|
let base = magnitude / eventCount
|
|
let remainder = magnitude % eventCount
|
|
return (0..<eventCount).map { index in
|
|
sign * Int32(base + (index < remainder ? 1 : 0))
|
|
}
|
|
}
|
|
|
|
func doubleTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
|
|
if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "double tap") {
|
|
guard case .performed = outcome else { return outcome }
|
|
sleepFor(0.1)
|
|
_ = pressTvRemote(.select)
|
|
return .performed
|
|
}
|
|
return performCoordinateDoubleTap(app: app, x: x, y: y)
|
|
}
|
|
|
|
func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
|
|
if let outcome = longSelectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), duration: duration) {
|
|
return outcome
|
|
}
|
|
return performCoordinateLongPress(app: app, x: x, y: y, duration: duration)
|
|
}
|
|
|
|
func dragAt(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
holdDuration: TimeInterval
|
|
) -> RunnerInteractionOutcome {
|
|
// tvOS has no coordinate drag. Preserve the direction as a focus move.
|
|
let dx = x2 - x
|
|
let dy = y2 - y
|
|
let button: TvRemoteButton = abs(dx) > abs(dy)
|
|
? (dx > 0 ? .right : .left)
|
|
: (dy > 0 ? .down : .up)
|
|
if pressTvRemote(button) {
|
|
return .performed
|
|
}
|
|
return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
|
|
}
|
|
|
|
/// Rotates an interface-oriented point into the device-native (portrait) space the
|
|
/// synthesized event path consumes — synthesized events skip XCTest's orientation
|
|
/// handling, so without this a landscape tap lands in the wrong place.
|
|
func nativeSynthesizedPoint(
|
|
orientedX x: Double,
|
|
orientedY y: Double,
|
|
in frame: CGRect,
|
|
interfaceOrientation: Int
|
|
) -> CGPoint {
|
|
let localX = x - Double(frame.minX)
|
|
let localY = y - Double(frame.minY)
|
|
let width = Double(frame.width)
|
|
let height = Double(frame.height)
|
|
switch interfaceOrientation {
|
|
case RunnerInterfaceOrientation.landscapeRight:
|
|
return CGPoint(x: height - localY, y: localX)
|
|
case RunnerInterfaceOrientation.landscapeLeft:
|
|
return CGPoint(x: localY, y: width - localX)
|
|
case RunnerInterfaceOrientation.portraitUpsideDown:
|
|
return CGPoint(x: width - localX, y: height - localY)
|
|
default: // portrait or unknown
|
|
return CGPoint(x: localX, y: localY)
|
|
}
|
|
}
|
|
|
|
/// Rotates an interface-oriented translation vector into the same native
|
|
/// coordinate space as `nativeSynthesizedPoint`.
|
|
func nativeSynthesizedVector(
|
|
orientedDx dx: Double,
|
|
orientedDy dy: Double,
|
|
interfaceOrientation: Int
|
|
) -> CGVector {
|
|
switch interfaceOrientation {
|
|
case RunnerInterfaceOrientation.landscapeRight:
|
|
return CGVector(dx: -dy, dy: dx)
|
|
case RunnerInterfaceOrientation.landscapeLeft:
|
|
return CGVector(dx: dy, dy: -dx)
|
|
case RunnerInterfaceOrientation.portraitUpsideDown:
|
|
return CGVector(dx: -dx, dy: -dy)
|
|
default: // portrait or unknown
|
|
return CGVector(dx: dx, dy: dy)
|
|
}
|
|
}
|
|
|
|
func synthesizedDragAt(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
durationMs: Double,
|
|
profile: SynthesizedDragProfile = .continuous,
|
|
context: SynthesizedCoordinateContext? = nil
|
|
) -> RunnerInteractionOutcome {
|
|
#if os(iOS)
|
|
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else {
|
|
return .unsupported(
|
|
message: "synthesized coordinate drag requires finite coordinates",
|
|
hint: "Retry with finite x, y, x2, and y2 values."
|
|
)
|
|
}
|
|
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
guard let context = context ?? synthesizedCoordinateContext(
|
|
app: app,
|
|
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
) else {
|
|
return .unsupported(
|
|
message: "synthesized coordinate drag could not resolve a finite screen frame",
|
|
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
|
|
)
|
|
}
|
|
let frame = context.referenceFrame
|
|
let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
|
|
let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
|
|
let message = switch profile {
|
|
case .continuous:
|
|
RunnerSynthesizedGesture.synthesizeContinuousDrag(
|
|
withApplication: app,
|
|
x: Double(start.x),
|
|
y: Double(start.y),
|
|
x2: Double(end.x),
|
|
y2: Double(end.y),
|
|
durationMs: durationMs
|
|
)
|
|
case .controlledScroll:
|
|
RunnerSynthesizedGesture.synthesizeControlledScroll(
|
|
withApplication: app,
|
|
x: Double(start.x),
|
|
y: Double(start.y),
|
|
x2: Double(end.x),
|
|
y2: Double(end.y),
|
|
durationMs: durationMs
|
|
)
|
|
case .fastSwipe:
|
|
RunnerSynthesizedGesture.synthesizeSwipe(
|
|
withApplication: app,
|
|
x: Double(start.x),
|
|
y: Double(start.y),
|
|
x2: Double(end.x),
|
|
y2: Double(end.y),
|
|
durationMs: durationMs
|
|
)
|
|
}
|
|
if let message {
|
|
return .unsupported(
|
|
message: message,
|
|
hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists."
|
|
)
|
|
}
|
|
return .performed
|
|
#elseif os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate drag is not supported on tvOS",
|
|
hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
|
|
)
|
|
#else
|
|
return .unsupported(
|
|
message: "coordinate drag is not supported on macOS",
|
|
hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
|
|
)
|
|
#endif
|
|
}
|
|
|
|
func synthesizedTapAt(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
context: SynthesizedCoordinateContext? = nil
|
|
) -> RunnerInteractionOutcome {
|
|
#if os(iOS)
|
|
guard x.isFinite, y.isFinite else {
|
|
return .unsupported(
|
|
message: "synthesized coordinate tap requires finite coordinates",
|
|
hint: "Retry with finite x and y values."
|
|
)
|
|
}
|
|
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
guard let context = context ?? synthesizedCoordinateContext(
|
|
app: app,
|
|
policy: synthesizedGesturePolicy(.coordinateTap)
|
|
) else {
|
|
return .unsupported(
|
|
message: "synthesized coordinate tap could not resolve a finite screen frame",
|
|
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
|
|
)
|
|
}
|
|
let frame = context.referenceFrame
|
|
let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
|
|
if let message = RunnerSynthesizedGesture.synthesizeTap(
|
|
withApplication: app,
|
|
x: Double(point.x),
|
|
y: Double(point.y)
|
|
) {
|
|
return .unsupported(
|
|
message: message,
|
|
hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree."
|
|
)
|
|
}
|
|
return .performed
|
|
#elseif os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
|
|
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
|
|
)
|
|
#else
|
|
return .unsupported(
|
|
message: "synthesized coordinate tap is not supported on macOS",
|
|
hint: "macOS automation has no touchscreen; use mouse-driven interactions instead."
|
|
)
|
|
#endif
|
|
}
|
|
|
|
func keyboardAvoidingDragPoints(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double
|
|
) -> DragPoints {
|
|
let original = DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
#if os(iOS)
|
|
guard let keyboardFrame = visibleKeyboardFrame(app: app) else {
|
|
return original
|
|
}
|
|
let minX = min(x, x2)
|
|
let minY = min(y, y2)
|
|
let gestureBounds = CGRect(
|
|
x: CGFloat(minX),
|
|
y: CGFloat(minY),
|
|
width: CGFloat(max(abs(x2 - x), 1)),
|
|
height: CGFloat(max(abs(y2 - y), 1))
|
|
)
|
|
guard gestureBounds.intersects(keyboardFrame) else {
|
|
return original
|
|
}
|
|
|
|
let window = app.windows.firstMatch
|
|
let appFrame = window.exists && !window.frame.isEmpty ? window.frame : app.frame
|
|
guard !appFrame.isEmpty else {
|
|
return original
|
|
}
|
|
|
|
let padding: Double = 12
|
|
let targetMaxY = Double(keyboardFrame.minY) - padding
|
|
let currentMaxY = max(y, y2)
|
|
let shift = currentMaxY - targetMaxY
|
|
guard shift > 0 else {
|
|
return original
|
|
}
|
|
|
|
let adjustedY = y - shift
|
|
let adjustedY2 = y2 - shift
|
|
guard min(adjustedY, adjustedY2) >= Double(appFrame.minY) + padding else {
|
|
return original
|
|
}
|
|
|
|
NSLog(
|
|
"AGENT_DEVICE_RUNNER_KEYBOARD_AVOIDING_DRAG from=(%.1f,%.1f)->(%.1f,%.1f) adjusted=(%.1f,%.1f)->(%.1f,%.1f) keyboardMinY=%.1f",
|
|
x,
|
|
y,
|
|
x2,
|
|
y2,
|
|
x,
|
|
adjustedY,
|
|
x2,
|
|
adjustedY2,
|
|
Double(keyboardFrame.minY)
|
|
)
|
|
return DragPoints(x: x, y: adjustedY, x2: x2, y2: adjustedY2)
|
|
#else
|
|
return original
|
|
#endif
|
|
}
|
|
|
|
func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame {
|
|
let appFrame = app.frame
|
|
let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame)
|
|
let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX
|
|
let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY
|
|
return TouchVisualizationFrame(
|
|
x: originX + x,
|
|
y: originY + y,
|
|
referenceWidth: referenceFrame.width,
|
|
referenceHeight: referenceFrame.height
|
|
)
|
|
}
|
|
|
|
func resolvedDragVisualizationFrame(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double
|
|
) -> DragVisualizationFrame {
|
|
let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y)
|
|
let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2)
|
|
return DragVisualizationFrame(
|
|
x: start.x,
|
|
y: start.y,
|
|
x2: end.x,
|
|
y2: end.y,
|
|
referenceWidth: start.referenceWidth,
|
|
referenceHeight: start.referenceHeight
|
|
)
|
|
}
|
|
|
|
func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
|
|
let window = app.windows.firstMatch
|
|
if window.exists {
|
|
let windowFrame = window.frame
|
|
if !windowFrame.isEmpty {
|
|
return frameAvoidingKeyboard(app: app, frame: windowFrame)
|
|
}
|
|
}
|
|
if !appFrame.isEmpty {
|
|
return frameAvoidingKeyboard(app: app, frame: appFrame)
|
|
}
|
|
return CGRect(x: 0, y: 0, width: 0, height: 0)
|
|
}
|
|
|
|
private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect {
|
|
#if os(iOS)
|
|
guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else {
|
|
return frame
|
|
}
|
|
let intersection = frame.intersection(keyboardFrame)
|
|
guard !intersection.isNull && intersection.height > 0 else {
|
|
return frame
|
|
}
|
|
let keyboardCoverage = intersection.width / max(frame.width, 1)
|
|
guard keyboardCoverage >= 0.5 else {
|
|
return frame
|
|
}
|
|
let safeHeight = keyboardFrame.minY - frame.minY
|
|
guard safeHeight >= frame.height * 0.25 else {
|
|
return frame
|
|
}
|
|
return CGRect(x: frame.minX, y: frame.minY, width: frame.width, height: safeHeight)
|
|
#else
|
|
return frame
|
|
#endif
|
|
}
|
|
|
|
func axFreeSynthesizedDragPlan(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
context: SynthesizedCoordinateContext? = nil
|
|
) -> SynthesizedDragPlan? {
|
|
#if os(iOS)
|
|
let context = context ?? synthesizedCoordinateContext(
|
|
app: app,
|
|
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
)
|
|
guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite,
|
|
let context
|
|
else {
|
|
return nil
|
|
}
|
|
let points = keyboardAvoidingSynthesizedDragPoints(
|
|
app: app,
|
|
x: x,
|
|
y: y,
|
|
x2: x2,
|
|
y2: y2,
|
|
context: context
|
|
)
|
|
return SynthesizedDragPlan(
|
|
points: points,
|
|
context: context
|
|
)
|
|
#else
|
|
return nil
|
|
#endif
|
|
}
|
|
|
|
func axFreeDragVisualizationFrame(
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
referenceFrame: CGRect
|
|
) -> DragVisualizationFrame {
|
|
return DragVisualizationFrame(
|
|
x: x,
|
|
y: y,
|
|
x2: x2,
|
|
y2: y2,
|
|
referenceWidth: Double(referenceFrame.width),
|
|
referenceHeight: Double(referenceFrame.height)
|
|
)
|
|
}
|
|
|
|
func synthesizedCoordinateContext(
|
|
app: XCUIApplication,
|
|
policy: SynthesizedGesturePolicy
|
|
) -> SynthesizedCoordinateContext? {
|
|
#if os(iOS)
|
|
let health = runnerAccessibilityHealth
|
|
let orientation = Int(
|
|
RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)
|
|
)
|
|
guard let referenceFrame = orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: XCUIScreen.main.screenshot().image.size,
|
|
interfaceOrientation: orientation
|
|
) else {
|
|
return nil
|
|
}
|
|
return SynthesizedCoordinateContext(
|
|
referenceFrame: referenceFrame,
|
|
keyboardPolicy: policy.keyboardPolicy,
|
|
fallbackPolicy: policy.fallbackPolicy,
|
|
accessibilityHealth: health
|
|
)
|
|
#else
|
|
return nil
|
|
#endif
|
|
}
|
|
|
|
func orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: CGSize,
|
|
interfaceOrientation: Int
|
|
) -> CGRect? {
|
|
// Physical iOS screenshots can retain portrait dimensions after the interface rotates,
|
|
// while accessibility frames remain in the logical landscape coordinate space.
|
|
guard screenshotSize.width.isFinite, screenshotSize.height.isFinite,
|
|
screenshotSize.width > 0,
|
|
screenshotSize.height > 0
|
|
else {
|
|
return nil
|
|
}
|
|
let isLandscape = interfaceOrientation == RunnerInterfaceOrientation.landscapeLeft
|
|
|| interfaceOrientation == RunnerInterfaceOrientation.landscapeRight
|
|
let isPortrait = interfaceOrientation == RunnerInterfaceOrientation.portrait
|
|
|| interfaceOrientation == RunnerInterfaceOrientation.portraitUpsideDown
|
|
let width: CGFloat
|
|
let height: CGFloat
|
|
if isLandscape {
|
|
width = max(screenshotSize.width, screenshotSize.height)
|
|
height = min(screenshotSize.width, screenshotSize.height)
|
|
} else if isPortrait {
|
|
width = min(screenshotSize.width, screenshotSize.height)
|
|
height = max(screenshotSize.width, screenshotSize.height)
|
|
} else {
|
|
width = screenshotSize.width
|
|
height = screenshotSize.height
|
|
}
|
|
return CGRect(x: 0, y: 0, width: width, height: height)
|
|
}
|
|
|
|
func keyboardAvoidingSynthesizedDragPoints(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
context: SynthesizedCoordinateContext
|
|
) -> DragPoints {
|
|
#if os(iOS)
|
|
guard context.allowsKeyboardProbe else {
|
|
return DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
}
|
|
return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2)
|
|
#else
|
|
return DragPoints(x: x, y: y, x2: x2, y2: y2)
|
|
#endif
|
|
}
|
|
|
|
func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
|
|
if performTvRemoteSwipeIfAvailable(direction: direction) {
|
|
let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
|
|
let midX = frame.midX
|
|
let midY = frame.midY
|
|
return DragVisualizationFrame(
|
|
x: midX,
|
|
y: midY,
|
|
x2: midX,
|
|
y2: midY,
|
|
referenceWidth: frame.width,
|
|
referenceHeight: frame.height
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
|
|
switch direction {
|
|
case "up":
|
|
return pressTvRemote(.up)
|
|
case "down":
|
|
return pressTvRemote(.down)
|
|
case "left":
|
|
return pressTvRemote(.left)
|
|
case "right":
|
|
return pressTvRemote(.right)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? {
|
|
guard plan.topology == "single" || plan.topology == "two" else {
|
|
return "planned gesture topology must be single or two"
|
|
}
|
|
let supportedIntent = plan.topology == "single"
|
|
? plan.intent == "fling" || plan.intent == "pan"
|
|
: plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate"
|
|
|| plan.intent == "transform"
|
|
guard supportedIntent else { return "planned gesture has unsupported intent for its topology" }
|
|
if plan.topology == "single" {
|
|
guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else {
|
|
return "single-pointer gesture requires a supported execution profile"
|
|
}
|
|
} else if plan.executionProfile != nil {
|
|
return "multi-touch gesture cannot define a single-pointer execution profile"
|
|
}
|
|
guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else {
|
|
return "planned gesture durationMs must be between 16 and 10000"
|
|
}
|
|
let viewport = plan.viewport
|
|
guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite,
|
|
viewport.height.isFinite, viewport.width > 0, viewport.height > 0
|
|
else {
|
|
return "planned gesture viewport must be finite and positive"
|
|
}
|
|
let expectedPointerCount = plan.topology == "single" ? 1 : 2
|
|
guard plan.pointers.count == expectedPointerCount else {
|
|
return "planned gesture pointer count does not match topology"
|
|
}
|
|
for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index {
|
|
return "planned gesture requires ordered pointer ids"
|
|
}
|
|
let firstSamples = plan.pointers[0].samples
|
|
guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" }
|
|
for pointer in plan.pointers {
|
|
guard pointer.samples.count == firstSamples.count else {
|
|
return "planned pointer paths require matching samples"
|
|
}
|
|
var previousOffset = -1.0
|
|
for (index, sample) in pointer.samples.enumerated() {
|
|
guard sample.offsetMs.isFinite,
|
|
sample.offsetMs == firstSamples[index].offsetMs,
|
|
sample.offsetMs > previousOffset
|
|
else {
|
|
return "planned pointer sample offsets must match and strictly increase"
|
|
}
|
|
let point = sample.point
|
|
guard point.x.isFinite, point.y.isFinite,
|
|
point.x >= viewport.x,
|
|
point.x <= viewport.x + viewport.width,
|
|
point.y >= viewport.y,
|
|
point.y <= viewport.y + viewport.height
|
|
else {
|
|
return "planned pointer sample lies outside the viewport"
|
|
}
|
|
previousOffset = sample.offsetMs
|
|
}
|
|
guard pointer.samples.first?.offsetMs == 0,
|
|
pointer.samples.last?.offsetMs == plan.durationMs
|
|
else {
|
|
return "planned pointer paths must start at 0 and end at durationMs"
|
|
}
|
|
}
|
|
if plan.topology == "two" {
|
|
guard let firstStart = firstSamples.first?.point,
|
|
let secondStart = plan.pointers[1].samples.first?.point,
|
|
hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0
|
|
else {
|
|
return "planned pointer paths require a positive initial span"
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution {
|
|
plan.topology == "single" && plan.executionProfile == "endpoint-hold"
|
|
? .fastSwipe
|
|
: .sampled
|
|
}
|
|
|
|
func sampledPlannedGesture(
|
|
app: XCUIApplication,
|
|
plan: RunnerGesturePlan
|
|
) -> RunnerInteractionOutcome {
|
|
#if os(iOS)
|
|
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
// The portable planner and validation use this exact viewport. Using app.frame here can
|
|
// diverge when XCTest unions transformed/off-screen descendants into the application frame.
|
|
let frame = CGRect(
|
|
x: plan.viewport.x,
|
|
y: plan.viewport.y,
|
|
width: plan.viewport.width,
|
|
height: plan.viewport.height
|
|
)
|
|
let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
|
|
pointer.samples.map { sample in
|
|
let point = nativeSynthesizedPoint(
|
|
orientedX: sample.point.x,
|
|
orientedY: sample.point.y,
|
|
in: frame,
|
|
interfaceOrientation: orientation
|
|
)
|
|
return [
|
|
"x": NSNumber(value: Double(point.x)),
|
|
"y": NSNumber(value: Double(point.y)),
|
|
"offsetMs": NSNumber(value: sample.offsetMs),
|
|
]
|
|
}
|
|
}
|
|
if let message = RunnerSynthesizedGesture.synthesizeGesture(
|
|
withApplication: app,
|
|
pointerSamples: pointerSamples
|
|
) {
|
|
return .unsupported(
|
|
message: message,
|
|
hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists."
|
|
)
|
|
}
|
|
return .performed
|
|
#elseif os(tvOS)
|
|
return .unsupported(
|
|
message: "two-finger gestures are not supported on tvOS",
|
|
hint: "tvOS has no touch input; use remote-driven navigation."
|
|
)
|
|
#elseif os(visionOS)
|
|
return .unsupported(
|
|
message: "two-finger touch gestures are not supported on visionOS",
|
|
hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only."
|
|
)
|
|
#else
|
|
return .unsupported(
|
|
message: "two-finger gestures are not supported on macOS",
|
|
hint: "macOS automation has no multi-touch input; run on an iOS simulator."
|
|
)
|
|
#endif
|
|
}
|
|
|
|
private func interactionRoot(app: XCUIApplication) -> XCUIElement {
|
|
let windows = app.windows.allElementsBoundByIndex
|
|
if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {
|
|
return window
|
|
}
|
|
return app
|
|
}
|
|
|
|
private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
|
|
#if os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
|
|
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
|
|
)
|
|
#else
|
|
interactionCoordinate(app: app, x: x, y: y).tap()
|
|
return .performed
|
|
#endif
|
|
}
|
|
|
|
private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
|
|
#if os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
|
|
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then select it."
|
|
)
|
|
#else
|
|
interactionCoordinate(app: app, x: x, y: y).doubleTap()
|
|
return .performed
|
|
#endif
|
|
}
|
|
|
|
private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
|
|
#if os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element",
|
|
hint: "tvOS has no coordinate input; move focus with swipe/scroll to the target, then long-select it."
|
|
)
|
|
#else
|
|
interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration)
|
|
return .performed
|
|
#endif
|
|
}
|
|
|
|
private func performCoordinateDrag(
|
|
app: XCUIApplication,
|
|
x: Double,
|
|
y: Double,
|
|
x2: Double,
|
|
y2: Double,
|
|
holdDuration: TimeInterval
|
|
) -> RunnerInteractionOutcome {
|
|
#if os(tvOS)
|
|
return .unsupported(
|
|
message: "coordinate drag is not supported on tvOS",
|
|
hint: "tvOS has no coordinate input; use remote-driven swipe/scroll to move focus instead."
|
|
)
|
|
#else
|
|
let start = interactionCoordinate(app: app, x: x, y: y)
|
|
let end = interactionCoordinate(app: app, x: x2, y: y2)
|
|
start.press(forDuration: holdDuration, thenDragTo: end)
|
|
return .performed
|
|
#endif
|
|
}
|
|
|
|
#if !os(tvOS)
|
|
private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
|
|
#if os(iOS)
|
|
let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
return origin.withOffset(CGVector(dx: x, dy: y))
|
|
#else
|
|
let root = interactionRoot(app: app)
|
|
let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
|
|
let rootFrame = root.frame
|
|
let offsetX = x - Double(rootFrame.origin.x)
|
|
let offsetY = y - Double(rootFrame.origin.y)
|
|
return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
#if AGENT_DEVICE_RUNNER_UNIT_TESTS
|
|
// Identity in portrait/unknown, 90° per landscape, 180° upside-down.
|
|
func testNativeSynthesizedPointRotatesByInterfaceOrientation() {
|
|
let portrait = CGRect(x: 0, y: 0, width: 834, height: 1210)
|
|
let landscape = CGRect(x: 0, y: 0, width: 1210, height: 834)
|
|
let offsetLandscape = CGRect(x: 10, y: 20, width: 1210, height: 834)
|
|
// (frame, UIInterfaceOrientation, expected native point) for a tap at (170, 268).
|
|
let cases: [(CGRect, Int, CGPoint)] = [
|
|
(portrait, RunnerInterfaceOrientation.portrait, CGPoint(x: 170, y: 268)),
|
|
(landscape, RunnerInterfaceOrientation.landscapeRight, CGPoint(x: 566, y: 170)),
|
|
(landscape, RunnerInterfaceOrientation.landscapeLeft, CGPoint(x: 268, y: 1040)),
|
|
(portrait, RunnerInterfaceOrientation.portraitUpsideDown, CGPoint(x: 664, y: 942)),
|
|
(portrait, RunnerInterfaceOrientation.unknown, CGPoint(x: 170, y: 268)),
|
|
]
|
|
for (frame, orientation, expected) in cases {
|
|
XCTAssertEqual(
|
|
nativeSynthesizedPoint(orientedX: 170, orientedY: 268, in: frame, interfaceOrientation: orientation),
|
|
expected,
|
|
"interfaceOrientation \(orientation)"
|
|
)
|
|
}
|
|
XCTAssertEqual(
|
|
nativeSynthesizedPoint(
|
|
orientedX: 180,
|
|
orientedY: 288,
|
|
in: offsetLandscape,
|
|
interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft
|
|
),
|
|
CGPoint(x: 268, y: 1040),
|
|
"non-zero frame origin is localized before rotation"
|
|
)
|
|
}
|
|
|
|
func testNativeSynthesizedVectorRotatesByInterfaceOrientation() {
|
|
let cases: [(Int, CGVector)] = [
|
|
(RunnerInterfaceOrientation.portrait, CGVector(dx: 40, dy: -20)),
|
|
(RunnerInterfaceOrientation.landscapeRight, CGVector(dx: 20, dy: 40)),
|
|
(RunnerInterfaceOrientation.landscapeLeft, CGVector(dx: -20, dy: -40)),
|
|
(RunnerInterfaceOrientation.portraitUpsideDown, CGVector(dx: -40, dy: 20)),
|
|
(RunnerInterfaceOrientation.unknown, CGVector(dx: 40, dy: -20)),
|
|
]
|
|
for (orientation, expected) in cases {
|
|
let vector = nativeSynthesizedVector(orientedDx: 40, orientedDy: -20, interfaceOrientation: orientation)
|
|
XCTAssertEqual(vector.dx, expected.dx, "dx interfaceOrientation \(orientation)")
|
|
XCTAssertEqual(vector.dy, expected.dy, "dy interfaceOrientation \(orientation)")
|
|
}
|
|
}
|
|
|
|
func testSynthesizedScreenshotReferenceFrameUsesScreenshotSize() throws {
|
|
let resolved = try XCTUnwrap(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: CGSize(width: 430, height: 932),
|
|
interfaceOrientation: RunnerInterfaceOrientation.portrait
|
|
)
|
|
)
|
|
|
|
XCTAssertEqual(resolved, CGRect(x: 0, y: 0, width: 430, height: 932))
|
|
}
|
|
|
|
func testOrientedSynthesizedScreenshotReferenceFrameUsesLandscapeLogicalDimensions() {
|
|
let portraitCapture = CGSize(width: 430, height: 932)
|
|
let landscapeCapture = CGSize(width: 932, height: 430)
|
|
|
|
for orientation in [
|
|
RunnerInterfaceOrientation.landscapeLeft,
|
|
RunnerInterfaceOrientation.landscapeRight,
|
|
] {
|
|
XCTAssertEqual(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: portraitCapture,
|
|
interfaceOrientation: orientation
|
|
),
|
|
CGRect(x: 0, y: 0, width: 932, height: 430)
|
|
)
|
|
XCTAssertEqual(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: landscapeCapture,
|
|
interfaceOrientation: orientation
|
|
),
|
|
CGRect(x: 0, y: 0, width: 932, height: 430)
|
|
)
|
|
}
|
|
|
|
for orientation in [
|
|
RunnerInterfaceOrientation.portrait,
|
|
RunnerInterfaceOrientation.portraitUpsideDown,
|
|
] {
|
|
XCTAssertEqual(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: portraitCapture,
|
|
interfaceOrientation: orientation
|
|
),
|
|
CGRect(x: 0, y: 0, width: 430, height: 932)
|
|
)
|
|
XCTAssertEqual(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: landscapeCapture,
|
|
interfaceOrientation: orientation
|
|
),
|
|
CGRect(x: 0, y: 0, width: 430, height: 932)
|
|
)
|
|
}
|
|
|
|
XCTAssertEqual(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: landscapeCapture,
|
|
interfaceOrientation: RunnerInterfaceOrientation.unknown
|
|
),
|
|
CGRect(x: 0, y: 0, width: 932, height: 430)
|
|
)
|
|
}
|
|
|
|
func testSynthesizedScreenshotReferenceFrameRejectsInvalidSize() {
|
|
XCTAssertNil(
|
|
orientedSynthesizedScreenshotReferenceFrame(
|
|
screenshotSize: CGSize(width: CGFloat.infinity, height: 932),
|
|
interfaceOrientation: RunnerInterfaceOrientation.portrait
|
|
)
|
|
)
|
|
}
|
|
|
|
func testPlannedMultiTouchGestureAcceptsMatchingInBoundsTrajectories() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"two","intent":"pan","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":16,"point":{"x":90,"y":85}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":16,"point":{"x":90,"y":125}},{"offsetMs":32,"point":{"x":100,"y":130}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertNil(plannedGestureValidationError(plan))
|
|
XCTAssertEqual(plannedGestureExecution(for: plan), .sampled)
|
|
}
|
|
|
|
func testPlannedMultiTouchGestureRejectsMismatchedOffsets() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"two","intent":"transform","durationMs":32,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":80,"y":80}},{"offsetMs":32,"point":{"x":100,"y":90}}]},{"pointerId":1,"samples":[{"offsetMs":0,"point":{"x":80,"y":120}},{"offsetMs":31,"point":{"x":100,"y":130}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertEqual(
|
|
plannedGestureValidationError(plan),
|
|
"planned pointer sample offsets must match and strictly increase"
|
|
)
|
|
}
|
|
|
|
func testSinglePointerFlingUsesFastSwipeExecution() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"single","intent":"fling","executionProfile":"endpoint-hold","durationMs":100,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":100,"point":{"x":40,"y":150}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe)
|
|
}
|
|
|
|
func testSinglePointerTimedPanUsesSampledExecution() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"single","intent":"pan","executionProfile":"timed-pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":250,"point":{"x":100,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertEqual(plannedGestureExecution(for: plan), .sampled)
|
|
}
|
|
|
|
func testSinglePointerEndpointHoldUsesFastSwipeExecution() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"single","intent":"pan","executionProfile":"endpoint-hold","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertNil(plannedGestureValidationError(plan))
|
|
XCTAssertEqual(plannedGestureExecution(for: plan), .fastSwipe)
|
|
}
|
|
|
|
func testSinglePointerGestureRejectsMissingExecutionProfile() throws {
|
|
let plan = try JSONDecoder().decode(
|
|
RunnerGesturePlan.self,
|
|
from: Data(
|
|
#"{"topology":"single","intent":"pan","durationMs":500,"viewport":{"x":0,"y":0,"width":200,"height":300},"pointers":[{"pointerId":0,"samples":[{"offsetMs":0,"point":{"x":160,"y":150}},{"offsetMs":500,"point":{"x":40,"y":150}}]}]}"#.utf8
|
|
)
|
|
)
|
|
|
|
XCTAssertEqual(
|
|
plannedGestureValidationError(plan),
|
|
"single-pointer gesture requires a supported execution profile"
|
|
)
|
|
}
|
|
|
|
func testDesktopScrollWheelDeltasMapDirections() {
|
|
XCTAssertEqual(desktopScrollWheelDeltas(direction: .up, pixels: 120).vertical, 120)
|
|
XCTAssertEqual(desktopScrollWheelDeltas(direction: .down, pixels: 120).vertical, -120)
|
|
XCTAssertEqual(desktopScrollWheelDeltas(direction: .left, pixels: 120).horizontal, 120)
|
|
XCTAssertEqual(desktopScrollWheelDeltas(direction: .right, pixels: 120).horizontal, -120)
|
|
}
|
|
|
|
func testDesktopScrollWheelDeltaEventsHonorDurationAndPreservePixels() {
|
|
let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 50)
|
|
XCTAssertEqual(events.count, 4)
|
|
XCTAssertEqual(events.map(\.vertical).reduce(0, +), -200)
|
|
XCTAssertEqual(events.map(\.horizontal).reduce(0, +), 0)
|
|
XCTAssertEqual(desktopScrollEventIntervalSeconds(durationMs: 50, eventCount: events.count), 0.05 / 3.0)
|
|
}
|
|
|
|
func testDesktopScrollWheelDeltaEventsKeepInstantScrollSingleEvent() {
|
|
let events = desktopScrollWheelDeltaEvents(direction: .down, pixels: 200, durationMs: 0)
|
|
XCTAssertEqual(events.count, 1)
|
|
XCTAssertEqual(events.first?.vertical, -200)
|
|
}
|
|
#endif
|
|
}
|