From 9b559299623a3aa57ae68633f52dce5230438c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 13 May 2026 13:14:57 +0200 Subject: [PATCH] fix: preserve iOS tab button tap centers (#508) --- src/__tests__/runtime-interactions.test.ts | 162 +++++++++++++++++++++ src/commands/interaction-targeting.ts | 31 +++- 2 files changed, 192 insertions(+), 1 deletion(-) diff --git a/src/__tests__/runtime-interactions.test.ts b/src/__tests__/runtime-interactions.test.ts index 5f30a265b..8444b5e78 100644 --- a/src/__tests__/runtime-interactions.test.ts +++ b/src/__tests__/runtime-interactions.test.ts @@ -67,6 +67,61 @@ test('runtime press resolves selector targets to the actionable node center', as assert.deepEqual(result.backendResult, { ok: true }); }); +test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(iosTabBarSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const refResult = await device.interactions.click(ref('@e4'), { + session: 'default', + }); + const selectorResult = await device.interactions.click(selector('label=Settings'), { + session: 'default', + }); + + assert.deepEqual(calls, [ + { x: 166, y: 822 }, + { x: 257, y: 822 }, + ]); + assert.equal(refResult.kind, 'ref'); + assert.equal(refResult.node?.label, 'Library'); + assert.equal(selectorResult.kind, 'selector'); + assert.equal(selectorResult.node?.label, 'Settings'); +}); + +test('runtime click keeps non-button semantic targets at their own center', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(nonHittableCellSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await clickRefE2(device); + + assert.deepEqual(calls, [{ x: 70, y: 30 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.label, 'Account'); +}); + +test('runtime click still promotes non-touchable nodes to hittable ancestors', async () => { + const calls: Point[] = []; + const device = createInteractionDevice(nonTouchableGroupSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + }); + + const result = await clickRefE2(device); + + assert.deepEqual(calls, [{ x: 160, y: 60 }]); + assert.equal(result.kind, 'ref'); + assert.equal(result.node?.label, 'Clickable group'); +}); + test('runtime fill resolves refs and forwards text to the backend primitive', async () => { const calls: Array<{ point: Point; text: string; delayMs?: number }> = []; const device = createInteractionDevice(fillableSnapshot(), { @@ -422,6 +477,107 @@ function fillableSnapshot(): SnapshotState { ]); } +function iosTabBarSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeApplication', + label: 'TabRepro', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: false, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeTabBar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + hittable: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Home', + rect: { x: 30, y: 800, width: 91, height: 44 }, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Library', + rect: { x: 120, y: 800, width: 92, height: 44 }, + hittable: false, + }, + { + index: 4, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Settings', + rect: { x: 211, y: 800, width: 91, height: 44 }, + hittable: false, + }, + { + index: 5, + depth: 2, + parentIndex: 1, + type: 'XCUIElementTypeButton', + label: 'Search', + rect: { x: 304, y: 800, width: 92, height: 44 }, + hittable: false, + }, + ]); +} + +function nonHittableCellSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Settings list', + rect: { x: 10, y: 20, width: 300, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + label: 'Account', + rect: { x: 20, y: 10, width: 100, height: 40 }, + hittable: false, + }, + ]); +} + +function nonTouchableGroupSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'Clickable group', + rect: { x: 10, y: 20, width: 300, height: 80 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeOther', + label: 'Decorative group', + rect: { x: 30, y: 40, width: 60, height: 20 }, + hittable: false, + }, + ]); +} + function snapshotWithOffscreenContent(): SnapshotState { return makeSnapshotState([ { @@ -495,3 +651,9 @@ function createInteractionDevice( policy: localCommandPolicy(), }); } + +async function clickRefE2(device: ReturnType) { + return await device.interactions.click(ref('@e2'), { + session: 'default', + }); +} diff --git a/src/commands/interaction-targeting.ts b/src/commands/interaction-targeting.ts index ab4b4e379..e39bfc46e 100644 --- a/src/commands/interaction-targeting.ts +++ b/src/commands/interaction-targeting.ts @@ -1,9 +1,23 @@ import type { Rect, SnapshotNode } from '../utils/snapshot.ts'; import { centerOfRect } from '../utils/snapshot.ts'; import { containsPoint, pickLargestRect } from '../utils/rect-visibility.ts'; -import { findNearestHittableAncestor } from '../utils/snapshot-processing.ts'; +import { findNearestHittableAncestor, normalizeType } from '../utils/snapshot-processing.ts'; import { normalizeRect, resolveRectCenter } from '../utils/rect-center.ts'; +const SEMANTIC_TOUCH_ROLE_FRAGMENTS = [ + 'button', + 'link', + 'menuitem', + 'tabitem', + 'textfield', + 'searchfield', + 'securetextfield', + 'checkbox', + 'radio', + 'switch', + 'cell', +]; + export function resolveActionableTouchNode( nodes: SnapshotNode[], node: SnapshotNode, @@ -12,6 +26,9 @@ export function resolveActionableTouchNode( if (descendant?.rect && resolveRectCenter(descendant.rect)) { return descendant; } + if (isSemanticallyTouchableNode(node) && node.rect && resolveRectCenter(node.rect)) { + return node; + } const ancestor = findNearestHittableAncestor(nodes, node); if (ancestor?.rect && resolveRectCenter(ancestor.rect)) { if (isOverlyBroadAncestor(node, ancestor, nodes)) { @@ -49,6 +66,18 @@ function findPreferredActionableDescendant( return current === node ? null : current; } +function isSemanticallyTouchableNode(node: SnapshotNode): boolean { + const roles = [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')); + return roles.some(isSemanticTouchRole); +} + +function isSemanticTouchRole(role: string): boolean { + // Match Tab exactly so broad roles like Table/TabBar do not become touch targets. + return ( + role === 'tab' || SEMANTIC_TOUCH_ROLE_FRAGMENTS.some((fragment) => role.includes(fragment)) + ); +} + function areRectsApproximatelyEqual(left: Rect, right: Rect): boolean { const tolerance = 0.5; return (