diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..7bedd3adc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,6 @@ +.claude/** +dist/** +node_modules/** +**/*.md +scripts/maestro-conformance/corpus/** +fallow-baselines/** diff --git a/packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts b/packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts index e8bf2c765..3a3d9a7aa 100644 --- a/packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts +++ b/packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts @@ -213,8 +213,8 @@ test('iOS target resolution preserves distinct nested controls matched by one ex ).toMatchObject({ ok: true, node: { index: 1 }, matches: 2 }); }); -test('iOS target resolution keeps raw geometry bound to the canonical source node', () => { - const interactionSnapshot = makeSnapshot([ +test('iOS target resolution keeps semantic identity bound to presented geometry', () => { + const semanticSnapshot = makeSnapshot([ { index: 0, type: 'Application', @@ -235,31 +235,85 @@ test('iOS target resolution keeps raw geometry bound to the canonical source nod rect: { x: 200, y: 100, width: 80, height: 48 }, }, ]); - const canonicalSnapshot = makeSnapshot([ - interactionSnapshot.nodes[0]!, - { ...interactionSnapshot.nodes[2]!, index: 1 }, + const presentationSnapshot = makeSnapshot([ + semanticSnapshot.nodes[0]!, + { ...semanticSnapshot.nodes[2]!, index: 1 }, ]); expect( resolveMaestroTargetFromSnapshot( - canonicalSnapshot, + semanticSnapshot, { selector: { text: 'Save' }, allowAtomicSelectorDispatch: true }, 'ios', { interactiveBounds: true, - interaction: { - snapshot: interactionSnapshot, + presentation: { + snapshot: presentationSnapshot, sourceIndexes: new Map([ [0, 0], [1, 2], ]), + presentedIndexesBySourceIndex: new Map([ + [0, [0]], + [1, []], + [2, [1]], + ]), + }, + }, + ), + ).toMatchObject({ + ok: true, + node: { index: 2 }, + rect: { x: 200, y: 100, width: 80, height: 48 }, + dispatchCandidates: 1, + }); +}); + +test('iOS target resolution uses one presented candidate universe for geometry and dispatch', () => { + const semanticSnapshot = makeSnapshot([ + { + index: 0, + type: 'Button', + identifier: 'save', + rect: { x: 0, y: 100, width: 80, height: 48 }, + }, + { + index: 1, + type: 'Button', + identifier: 'save', + rect: { x: 200, y: 100, width: 80, height: 48 }, + }, + ]); + const presentationSnapshot = makeSnapshot([ + { + ...semanticSnapshot.nodes[1]!, + index: 0, + rect: { x: 220, y: 120, width: 80, height: 48 }, + }, + ]); + + expect( + resolveMaestroTargetFromSnapshot( + semanticSnapshot, + { selector: { id: 'save' }, allowAtomicSelectorDispatch: true }, + 'ios', + { + interactiveBounds: true, + presentation: { + snapshot: presentationSnapshot, + sourceIndexes: new Map([[0, 1]]), + presentedIndexesBySourceIndex: new Map([ + [0, []], + [1, [0]], + ]), }, }, ), ).toMatchObject({ ok: true, node: { index: 1 }, - rect: { x: 200, y: 100, width: 80, height: 48 }, - dispatchCandidates: 2, + rect: { x: 220, y: 120, width: 80, height: 48 }, + matches: 1, + dispatchCandidates: 0, }); }); diff --git a/packages/maestro/src/internal/runtime-target-ranking.ts b/packages/maestro/src/internal/runtime-target-ranking.ts index dea59f81b..776ab2a6f 100644 --- a/packages/maestro/src/internal/runtime-target-ranking.ts +++ b/packages/maestro/src/internal/runtime-target-ranking.ts @@ -17,28 +17,37 @@ export type MaestroRankedCandidates = { readonly parentMatched: boolean; }; +export type MaestroCandidateMatches = Pick; + export function rankMaestroCandidates( snapshot: SnapshotState, selector: MaestroSelector, platform: MaestroPlatform, childOf?: MaestroSelector, ): MaestroRankedCandidates { - const matches = snapshot.nodes.filter((node) => matchesMaestroTypedSelector(node, selector)); - const scoped = scopeMatchesByAncestor(snapshot, matches, childOf); + const scoped = matchMaestroCandidates(snapshot, selector, childOf); const visible = filterVisibleMaestroMatches({ nodes: snapshot.nodes, matches: scoped.matches, platform, }); return { - matches: scoped.matches, + ...scoped, visible, - ranked: normalizeMaestroSnapshotMatches(snapshot.nodes, visible, selector, platform), - parentMatched: scoped.parentMatched, + ranked: rankVisibleMaestroMatches(snapshot.nodes, visible, selector, platform), }; } -function normalizeMaestroSnapshotMatches( +export function matchMaestroCandidates( + snapshot: SnapshotState, + selector: MaestroSelector, + childOf?: MaestroSelector, +): MaestroCandidateMatches { + const matches = snapshot.nodes.filter((node) => matchesMaestroTypedSelector(node, selector)); + return scopeMatchesByAncestor(snapshot, matches, childOf); +} + +export function rankVisibleMaestroMatches( nodes: SnapshotNode[], matches: SnapshotNode[], selector: MaestroSelector, diff --git a/packages/maestro/src/internal/runtime-targets.ts b/packages/maestro/src/internal/runtime-targets.ts index d01dcca77..c60ac6693 100644 --- a/packages/maestro/src/internal/runtime-targets.ts +++ b/packages/maestro/src/internal/runtime-targets.ts @@ -1,9 +1,14 @@ import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import type { MaestroSelector } from './program-ir.ts'; import type { MaestroPlatform } from './runtime-target-policy.ts'; -import { rankMaestroCandidates, selectMaestroSnapshotMatch } from './runtime-target-ranking.ts'; +import { + matchMaestroCandidates, + rankMaestroCandidates, + rankVisibleMaestroMatches, + selectMaestroSnapshotMatch, +} from './runtime-target-ranking.ts'; import { pointInsideRect, stripUndefined } from './shared.ts'; -import { isDescendantOfSnapshotNode } from './snapshot-policy.ts'; +import { isMaestroNodeVisible } from './snapshot-policy.ts'; import { buildSnapshotNodeMap } from '@agent-device/contracts/snapshot'; export type MaestroTargetQuery = { @@ -22,8 +27,9 @@ export type MaestroTargetEvidence = { ref?: string; }; -type MaestroInteractionProjection = { +type MaestroInteractivePresentation = { snapshot: SnapshotState; + presentedIndexesBySourceIndex: ReadonlyMap; sourceIndexes: ReadonlyMap; }; @@ -44,10 +50,15 @@ export function resolveMaestroTargetFromSnapshot( platform: MaestroPlatform, options: { interactiveBounds?: boolean; - interaction?: MaestroInteractionProjection; + presentation?: MaestroInteractivePresentation; } = {}, ): MaestroTargetResolution { - const candidates = rankMaestroCandidates(snapshot, query.selector, platform, query.childOf); + const presentedNodes = options.presentation + ? createPresentedNodeLookup(options.presentation) + : undefined; + const candidates = presentedNodes + ? rankPresentedMaestroCandidates(snapshot, query, platform, presentedNodes) + : rankMaestroCandidates(snapshot, query.selector, platform, query.childOf); if (!candidates.parentMatched) { return { ok: false, @@ -61,49 +72,58 @@ export function resolveMaestroTargetFromSnapshot( if (!target) { return failedTargetResolution(query, matches, rankedMatches, evidence); } - const interactionCandidates = resolveInteractionCandidates(target.node, query, platform, options); - const interactionTarget = interactionCandidates - ? selectMaestroSnapshotMatch(interactionCandidates.mapped, undefined) - : undefined; + const presentedTarget = presentedNodes + ? selectMaestroSnapshotMatch(presentedNodes.forSource(target.node), undefined) + : null; const rect = - options.interactiveBounds === true ? (interactionTarget?.rect ?? target.rect) : target.rect; + options.interactiveBounds === true ? (presentedTarget?.rect ?? target.rect) : target.rect; return { ok: true, node: target.node, rect, matches: rankedMatches.length, dispatchCandidates: - platform === 'ios' && query.allowAtomicSelectorDispatch && !query.childOf - ? countInteractionDispatchCandidates(target, interactionCandidates) + platform === 'ios' && query.allowAtomicSelectorDispatch && !query.childOf && presentedNodes + ? countInteractionDispatchCandidates(target, rankedMatches, presentedTarget) : 0, evidence, }; } -function resolveInteractionCandidates( - canonicalTarget: SnapshotNode, +function createPresentedNodeLookup(presentation: MaestroInteractivePresentation): { + isVisible: (node: SnapshotNode) => boolean; + forSource: (node: SnapshotNode) => SnapshotNode[]; +} { + const presentedByIndex = buildSnapshotNodeMap(presentation.snapshot.nodes); + const forSource = (semanticNode: SnapshotNode): SnapshotNode[] => { + return (presentation.presentedIndexesBySourceIndex.get(semanticNode.index) ?? []).flatMap( + (presentedIndex) => { + const node = presentedByIndex.get(presentedIndex); + return node ? [node] : []; + }, + ); + }; + return { + forSource, + isVisible: (node) => + forSource(node).some((presentedNode) => + isMaestroNodeVisible(presentedNode, presentation.snapshot.nodes, 'ios'), + ), + }; +} + +function rankPresentedMaestroCandidates( + snapshot: SnapshotState, query: MaestroTargetQuery, platform: MaestroPlatform, - options: { - interaction?: MaestroInteractionProjection; - }, -): { all: SnapshotNode[]; mapped: SnapshotNode[] } | undefined { - const interaction = options.interaction; - if (!interaction) return undefined; - const { snapshot, sourceIndexes } = interaction; - const sourceIndex = sourceIndexes.get(canonicalTarget.index); - if (sourceIndex === undefined) return undefined; - const byIndex = buildSnapshotNodeMap(snapshot.nodes); - const source = byIndex.get(sourceIndex); - if (!source) return undefined; - const all = rankMaestroCandidates(snapshot, query.selector, platform, query.childOf).ranked; + presentedNodes: ReturnType, +) { + const scoped = matchMaestroCandidates(snapshot, query.selector, query.childOf); + const visible = scoped.matches.filter(presentedNodes.isVisible); return { - all, - mapped: all.filter( - (candidate) => - candidate.index === source.index || - isDescendantOfSnapshotNode(snapshot.nodes, candidate, source, byIndex), - ), + ...scoped, + visible, + ranked: rankVisibleMaestroMatches(snapshot.nodes, visible, query.selector, platform), }; } @@ -126,14 +146,13 @@ function failedTargetResolution( function countInteractionDispatchCandidates( target: { node: SnapshotNode; rect: Rect }, - interactionCandidates: { all: SnapshotNode[]; mapped: SnapshotNode[] } | undefined, + rankedCandidates: SnapshotNode[], + presentedTarget: { node: SnapshotNode; rect: Rect } | null, ): number { - if (!interactionCandidates) return 0; - if (interactionCandidates.all.length !== 1) return interactionCandidates.all.length; - const interactionTarget = selectMaestroSnapshotMatch(interactionCandidates.mapped, undefined); - return interactionTarget && - interactionTarget.node.hittable !== false && - haveSameTapPoint(interactionTarget.rect, target.rect) + if (rankedCandidates.length !== 1) return rankedCandidates.length; + return presentedTarget && + presentedTarget.node.hittable !== false && + haveSameTapPoint(presentedTarget.rect, target.rect) ? 1 : 0; } diff --git a/packages/maestro/test/conformance/verify.test.ts b/packages/maestro/test/conformance/verify.test.ts index 48b13d829..e6254d934 100644 --- a/packages/maestro/test/conformance/verify.test.ts +++ b/packages/maestro/test/conformance/verify.test.ts @@ -10,6 +10,7 @@ import { CONFORMANCE_DATA_DIR, CORPUS_DIR, checkCoverage, + checkCorpusSeals, checkFixtureSeals, checkLayer2, classifyAllFlows, @@ -43,6 +44,15 @@ test('fixture content is sealed against hand editing', () => { } }); +test('vendored upstream corpus content matches its manifest sha256', () => { + const drifted = checkCorpusSeals().filter((result) => !result.sealed); + assert.deepEqual( + drifted, + [], + 'upstream corpus content drifted — restore the vendored bytes or regenerate from the pinned upstream commit', + ); +}); + // The seal is only worth having if it actually catches an edit. Prove it does, // rather than trusting that a hash comparison must work. test('the seal rejects an edited capture (proof the check has teeth)', () => { diff --git a/packages/maestro/test/conformance/verify.ts b/packages/maestro/test/conformance/verify.ts index 3a00693cd..0b40a1e9f 100644 --- a/packages/maestro/test/conformance/verify.ts +++ b/packages/maestro/test/conformance/verify.ts @@ -11,6 +11,7 @@ // flow our engine parses, or be explicitly listed as unverified. // - Bug classes: the four #1217 regressions each assert against their fixture. import fs from 'node:fs'; +import { createHash } from 'node:crypto'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { AppError } from '@agent-device/kernel/errors'; @@ -76,6 +77,13 @@ export function loadLayer2(): Layer2Fixture { export type SealResult = { file: string; sealed: boolean; expected: string; actual?: string }; +type CorpusManifest = { + flows: Array<{ + file: string; + origin: { kind: 'authored'; note: string } | { kind: 'upstream'; sha256: string }; + }>; +}; + /** * Recompute each fixture's content seal. This is what makes "generated from * upstream" an enforced property rather than a claim in a README: editing a @@ -89,6 +97,25 @@ export function checkFixtureSeals(): SealResult[] { }); } +/** Verify vendored upstream source bytes in normal, Java-free per-PR CI. */ +export function checkCorpusSeals(): SealResult[] { + const manifest = readJson(path.join(CORPUS_DIR, 'manifest.json')); + return manifest.flows.flatMap((flow) => { + if (flow.origin.kind !== 'upstream') return []; + const actual = createHash('sha256') + .update(fs.readFileSync(path.join(CORPUS_DIR, flow.file))) + .digest('hex'); + return [ + { + file: flow.file, + sealed: actual === flow.origin.sha256, + expected: flow.origin.sha256, + actual, + }, + ]; + }); +} + /** * Error codes that count as a deliberate parser rejection. Every rejection the * Maestro parser raises — unsupported command/option, bad value, and even a YAML diff --git a/scripts/maestro-conformance/corpus/authored/doubletap.yaml b/scripts/maestro-conformance/corpus/authored/doubletap.yaml index c587fe768..4e5c0e1ec 100644 --- a/scripts/maestro-conformance/corpus/authored/doubletap.yaml +++ b/scripts/maestro-conformance/corpus/authored/doubletap.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- doubleTapOn: 'Button' +- doubleTapOn: "Button" diff --git a/scripts/maestro-conformance/corpus/authored/extended-wait.yaml b/scripts/maestro-conformance/corpus/authored/extended-wait.yaml index f548f1a36..9008d1296 100644 --- a/scripts/maestro-conformance/corpus/authored/extended-wait.yaml +++ b/scripts/maestro-conformance/corpus/authored/extended-wait.yaml @@ -2,9 +2,9 @@ appId: com.example.app --- - extendedWaitUntil: visible: - id: 'Item' + id: "Item" timeout: 1000 - extendedWaitUntil: notVisible: - id: 'Another' + id: "Another" timeout: 1000 diff --git a/scripts/maestro-conformance/corpus/authored/repeat.yaml b/scripts/maestro-conformance/corpus/authored/repeat.yaml index 02a7ba32d..a94ff5842 100644 --- a/scripts/maestro-conformance/corpus/authored/repeat.yaml +++ b/scripts/maestro-conformance/corpus/authored/repeat.yaml @@ -3,4 +3,4 @@ appId: com.example.app - repeat: times: 3 commands: - - tapOn: 'Button' + - tapOn: "Button" diff --git a/scripts/maestro-conformance/corpus/authored/runflow-child.yaml b/scripts/maestro-conformance/corpus/authored/runflow-child.yaml index 988afa98e..9e122a166 100644 --- a/scripts/maestro-conformance/corpus/authored/runflow-child.yaml +++ b/scripts/maestro-conformance/corpus/authored/runflow-child.yaml @@ -2,4 +2,4 @@ appId: com.example.include --- - launchApp - tapOn: - id: 'included-button' + id: "included-button" diff --git a/scripts/maestro-conformance/corpus/authored/runflow-main.yaml b/scripts/maestro-conformance/corpus/authored/runflow-main.yaml index 3a4706b79..85f2abe85 100644 --- a/scripts/maestro-conformance/corpus/authored/runflow-main.yaml +++ b/scripts/maestro-conformance/corpus/authored/runflow-main.yaml @@ -1,5 +1,5 @@ appId: com.example.app --- -- tapOn: 'Before' +- tapOn: "Before" - runFlow: runflow-child.yaml -- tapOn: 'After' +- tapOn: "After" diff --git a/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml b/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml index 0d3a50db2..c578a3b9d 100644 --- a/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml +++ b/scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml @@ -2,6 +2,6 @@ appId: com.example.app --- - scrollUntilVisible: element: - text: 'Test' + text: "Test" direction: DOWN timeout: 10000 diff --git a/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml b/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml index 10bbbb6e6..b02bd06d8 100644 --- a/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml +++ b/scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml @@ -3,5 +3,5 @@ appId: com.example.app --- - swipe: - start: '50.5%, 50%' - end: '10%, 50%' + start: "50.5%, 50%" + end: "10%, 50%" diff --git a/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml b/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml index d15a68661..1fe6fa092 100644 --- a/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml +++ b/scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml @@ -6,4 +6,4 @@ appId: com.example.app - retry: maxRetries: 99 commands: - - tapOn: 'Retry' + - tapOn: "Retry" diff --git a/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml b/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml index 2bbc1e15a..05650fa66 100644 --- a/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml +++ b/scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml @@ -4,4 +4,4 @@ # same name (no reflectable upstream constant exists). appId: com.example.app --- -- tapOn: 'Submit' +- tapOn: "Submit" diff --git a/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml b/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml index b19fccc0f..a7398af0b 100644 --- a/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml +++ b/scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml @@ -4,4 +4,4 @@ appId: com.example.app --- - swipe: from: - id: 'row' + id: "row" diff --git a/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml b/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml index 709775b99..ae3b78c13 100644 --- a/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml +++ b/scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml @@ -1,4 +1,4 @@ # The command document must be a sequence. appId: com.example.app --- -tapOn: 'Button' +tapOn: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml b/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml index aeac7d86f..ecf2d9aa2 100644 --- a/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml +++ b/scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml @@ -2,4 +2,4 @@ appId: com.example.app --- - tapOn: - - text: 'Button' + - text: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml b/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml index 9affff913..2b6b7c1a1 100644 --- a/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml +++ b/scripts/maestro-conformance/corpus/invalid/unknown-command.yaml @@ -1,4 +1,4 @@ # Upstream rejects an unknown command name (typo of tapOn). appId: com.example.app --- -- tapOnn: 'Button' +- tapOnn: "Button" diff --git a/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml b/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml index 365a16ed2..e81afabd6 100644 --- a/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml +++ b/scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml @@ -2,5 +2,5 @@ appId: com.example.app --- - tapOn: - text: 'Button' + text: "Button" bogusField: true diff --git a/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml b/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml index bfc66a8ac..5552e4dba 100644 --- a/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml +++ b/scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - assertVisible: - id: 'element_id' + id: "element_id" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml b/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml index d7cf9a2b1..4d05bf058 100644 --- a/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml +++ b/scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - assertVisible: - text: 'Element Text' + text: "Element Text" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml b/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml index 33aa62537..259034234 100644 --- a/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml +++ b/scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - tapOn: - text: '.*button.*' + text: ".*button.*" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml b/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml index 9d01c737c..66ef9c30d 100644 --- a/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml +++ b/scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml @@ -1,8 +1,8 @@ appId: com.example.app --- - tapOn: - text: 'Optional Element' + text: "Optional Element" optional: true - assertVisible: - text: 'Non Optional' - optional: false + text: "Non Optional" + optional: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml b/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml index 92f0b14cc..bd91ecc5c 100644 --- a/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml +++ b/scripts/maestro-conformance/corpus/upstream/010_scroll.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- scroll +- scroll \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml b/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml index 26ee785a0..cd7d0c53d 100644 --- a/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml +++ b/scripts/maestro-conformance/corpus/upstream/011_back_press.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- back +- back \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml b/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml index 0c44ea92d..2ba2ee5e5 100644 --- a/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml +++ b/scripts/maestro-conformance/corpus/upstream/012_input_text.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- -- inputText: 'Hello World' -- inputText: user@example.com +- inputText: "Hello World" +- inputText: user@example.com \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml b/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml index 4a888fc01..e98c0c42b 100644 --- a/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml +++ b/scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- launchApp +- launchApp \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml b/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml index c75fb92de..061a370ab 100644 --- a/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml +++ b/scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - tapOn: - point: 100,200 + point: 100,200 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml b/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml index dc1063320..c358ecb69 100644 --- a/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml +++ b/scripts/maestro-conformance/corpus/upstream/017_swipe.yaml @@ -3,4 +3,4 @@ appId: com.example.app - swipe: start: 100,500 end: 100,200 - duration: 3000 + duration: 3000 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml b/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml index f849b74aa..46ce97e75 100644 --- a/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml +++ b/scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - launchApp: - clearState: true + clearState: true \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml b/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml index ecff4d90b..42cc23970 100644 --- a/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml +++ b/scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - assertNotVisible: - id: 'element_id' + id: "element_id" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml b/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml index 28b84ef71..b5c8d1b6b 100644 --- a/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml +++ b/scripts/maestro-conformance/corpus/upstream/027_open_link.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- openLink: https://example.com +- openLink: https://example.com \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml b/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml index 54de0fa45..440f73d7f 100644 --- a/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml +++ b/scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - longPressOn: - text: '.*button.*' + text: ".*button.*" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml b/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml index ac0543bbf..7e7b2abf3 100644 --- a/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml +++ b/scripts/maestro-conformance/corpus/upstream/032_element_index.yaml @@ -7,4 +7,4 @@ appId: com.example.app - tapOn: text: Item.* index: ${0 + 1} - retryTapIfNoChange: false + retryTapIfNoChange: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml b/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml index b2f50f7e0..8d72550e7 100644 --- a/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml +++ b/scripts/maestro-conformance/corpus/upstream/034_press_key.yaml @@ -29,3 +29,4 @@ appId: com.example.app - pressKey: TV Input HDMI 1 - pressKey: TV Input HDMI 2 - pressKey: TV Input HDMI 3 + diff --git a/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml b/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml index b8da72f9f..93ab4f21e 100644 --- a/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml +++ b/scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- hideKeyboard +- hideKeyboard \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml b/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml index debd13545..dd58226fa 100644 --- a/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml +++ b/scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml @@ -1,6 +1,6 @@ appId: com.example.app env: - TIMEOUT: 1000 + TIMEOUT: 1000 --- - extendedWaitUntil: visible: Item diff --git a/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml b/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml index 872b772d7..e2859cb86 100644 --- a/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml +++ b/scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml @@ -4,10 +4,10 @@ appId: com.other.app times: 3 commands: - tapOn: Button -- assertVisible: '3' +- assertVisible: "3" - evalScript: ${output.list = [1, 2, 3]} - repeat: times: ${output.list.length} commands: - tapOn: Button -- assertVisible: '6' +- assertVisible: "6" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml b/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml index 0766cd474..c548e5468 100644 --- a/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml +++ b/scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml @@ -2,4 +2,4 @@ appId: com.example.app --- - swipe: direction: RIGHT - duration: 500 + duration: 500 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml b/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml index 0b1fd853a..11e1a8348 100644 --- a/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml +++ b/scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - launchApp: - stopApp: false + stopApp: false \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml b/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml index c13139b73..4c05431f7 100644 --- a/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml +++ b/scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml @@ -1,5 +1,5 @@ appId: com.example.app --- - copyTextFrom: - id: 'myId' + id: "myId" - pasteText diff --git a/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml b/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml index ee6d31a41..abad0c509 100644 --- a/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml +++ b/scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml @@ -1,3 +1,3 @@ appId: com.example.app --- -- assertTrue: ${1+1} +- assertTrue: ${1+1} \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml b/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml index ec5173620..e8832c37a 100644 --- a/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml +++ b/scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml @@ -1,4 +1,4 @@ appId: com.example.app --- - waitForAnimationToEnd: - timeout: 500 + timeout: 500 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml b/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml index 0756e9c43..604241ce9 100644 --- a/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml +++ b/scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml @@ -3,4 +3,4 @@ appId: com.example.app - swipe: direction: RIGHT from: - text: 'swiping element' + text: "swiping element" diff --git a/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml b/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml index e6130d679..50329a922 100644 --- a/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml +++ b/scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml @@ -3,16 +3,16 @@ appId: com.example.app - scrollUntilVisible: timeout: 1 element: - id: 'not_found' + id: "not_found" optional: true - assertTrue: - condition: 'false' + condition: "false" optional: true - extendedWaitUntil: visible: - id: 'not_found' + id: "not_found" timeout: 1 optional: true - assertVisible: - text: 'Button' + text: "Button" optional: true diff --git a/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml b/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml index 8733876cb..18d45d3f8 100644 --- a/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml +++ b/scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml @@ -1,6 +1,6 @@ appId: com.example.app --- - swipe: - start: '50%,30%' - end: '50%,60%' - duration: 3000 + start: "50%,30%" + end: "50%,60%" + duration: 3000 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml b/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml index 5042b3054..539b380e3 100644 --- a/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml +++ b/scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml @@ -2,8 +2,8 @@ appId: com.example.app --- - scrollUntilVisible: element: - text: 'Test' + text: "Test" speed: 100 visibilityPercentage: 100 direction: DOWN - timeout: 10 + timeout: 10 \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml b/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml index 049d22bb4..3ab7d1906 100644 --- a/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml +++ b/scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml @@ -1,10 +1,10 @@ appId: com.example.app --- - assertVisible: - text: 'child_id' + text: "child_id" childOf: - text: 'parent_id_1' + text: "parent_id_1" - assertNotVisible: - text: 'child_id' + text: "child_id" childOf: - text: 'parent_id_3' + text: "parent_id_3" \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml b/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml index a6b6c0ccc..b6b57ebec 100644 --- a/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml +++ b/scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml @@ -1,5 +1,5 @@ appId: com.example.app --- - tapOn: - text: '.*button.*' - retryTapIfNoChange: true + text: ".*button.*" + retryTapIfNoChange: true \ No newline at end of file diff --git a/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml b/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml index 2ac17d675..483fc1a5c 100644 --- a/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml +++ b/scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml @@ -3,4 +3,4 @@ appId: com.example.app - setPermissions: permissions: all: deny - notifications: unset + notifications: unset \ No newline at end of file diff --git a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts index b9483bc11..0945a85b5 100644 --- a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts +++ b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts @@ -10,6 +10,7 @@ import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts'; import { MAESTRO_OBSERVATION_POLL_MS, maestroSnapshotSignature, + observeTypedMaestroCondition, resolveTypedMaestroTarget, waitForTypedSnapshotStability, } from '../daemon-runtime-port-observation.ts'; @@ -130,6 +131,76 @@ test('computes expensive target evidence only for the command policies that cons expect(atomicRetry.dispatchSelector).toEqual({ key: 'id', value: 'continue' }); }); +test('matches iOS Maestro ids on semantic nodes suppressed from interactive presentation', async () => { + const snapshot = makeSnapshot([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + parentIndex: 0, + type: 'Other', + identifier: 'inert-title', + rect: { x: 20, y: 100, width: 180, height: 32 }, + }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Inert surface', + rect: { x: 20, y: 100, width: 180, height: 32 }, + }, + ]); + + await expect( + observeTypedMaestroCondition({ + condition: { kind: 'visible', selector: { id: 'inert-title' } }, + timeoutMs: 0, + context: { generation: 0, env: {} }, + snapshot: async () => snapshot, + dependencies: makeDependencies(), + platform: 'ios', + }), + ).resolves.toMatchObject({ matched: true, visible: true, candidateCount: 1, ref: 'e2' }); +}); + +test('reports iOS Maestro evidence refs in semantic snapshot space after earlier suppression', async () => { + const snapshot = makeSnapshot([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { index: 1, parentIndex: 0, type: 'Other', identifier: 'discarded-wrapper' }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Earlier content', + rect: { x: 20, y: 40, width: 180, height: 32 }, + }, + { + index: 3, + parentIndex: 0, + type: 'Other', + identifier: 'target-wrapper', + rect: { x: 20, y: 100, width: 180, height: 32 }, + }, + { + index: 4, + parentIndex: 3, + type: 'StaticText', + label: 'Target content', + rect: { x: 20, y: 100, width: 180, height: 32 }, + }, + ]); + + await expect( + observeTypedMaestroCondition({ + condition: { kind: 'visible', selector: { id: 'target-wrapper' } }, + timeoutMs: 0, + context: { generation: 0, env: {} }, + snapshot: async () => snapshot, + dependencies: makeDependencies(), + platform: 'ios', + }), + ).resolves.toMatchObject({ matched: true, visible: true, candidateCount: 1, ref: 'e4' }); +}); + test('compares snapshots before sleeping and captures once beyond a zero settle budget', async () => { const clock = { value: 0 }; const captures = [ diff --git a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-target-geometry.test.ts b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-target-geometry.test.ts index 3ae6f7f94..54a27ae5f 100644 --- a/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-target-geometry.test.ts +++ b/src/daemon/adapters/maestro/__tests__/daemon-runtime-port-target-geometry.test.ts @@ -62,7 +62,7 @@ test('uses canonical iOS presentation only for atomic selector uniqueness', asyn expect(requests[1]?.positionals).toEqual(['text="First"']); }); -test('uses resolved iOS geometry when canonical presentation changes target bounds', async () => { +test('uses exact iOS representative geometry when presentation suppresses the semantic target', async () => { const requests: DaemonRequest[] = []; const port = createDaemonMaestroRuntimePort({ baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }), @@ -134,7 +134,7 @@ test('uses resolved iOS geometry when canonical presentation changes target boun expect(requests.map(({ command }) => command)).toEqual(['snapshot', 'click']); expect( requests.filter(({ command }) => command === 'click').map(({ positionals }) => positionals), - ).toEqual([['56', '121']]); + ).toEqual([['266', '121']]); }); test('uses the selected iOS node interactive bounds without changing raw target matching', async () => { diff --git a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts index 96a886f91..edcf1897e 100644 --- a/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts +++ b/src/daemon/adapters/maestro/daemon-runtime-port-observation.ts @@ -104,17 +104,21 @@ function resolveTargetFromSnapshot(params: { params.platform === 'ios' ? buildIosInteractiveSnapshotPresentation(params.snapshot.nodes) : undefined; - const canonicalSnapshot = presentation + const interactiveSnapshot = presentation ? { ...params.snapshot, nodes: attachRefs(presentation.nodes) } : params.snapshot; const resolution = resolveMaestroTargetFromSnapshot( - canonicalSnapshot, + params.snapshot, params.query, params.platform, { interactiveBounds: params.mode === 'tap', - interaction: presentation - ? { snapshot: params.snapshot, sourceIndexes: presentation.sourceIndexes } + presentation: presentation + ? { + snapshot: interactiveSnapshot, + sourceIndexes: presentation.sourceIndexes, + presentedIndexesBySourceIndex: presentation.presentedIndexesBySourceIndex, + } : undefined, }, ); diff --git a/src/daemon/snapshot-presentation/ios/index.ts b/src/daemon/snapshot-presentation/ios/index.ts index 41567fe29..2bf001dbb 100644 --- a/src/daemon/snapshot-presentation/ios/index.ts +++ b/src/daemon/snapshot-presentation/ios/index.ts @@ -26,6 +26,8 @@ export function presentIosInteractiveSnapshot(nodes: RawSnapshotNode[]): RawSnap export type IosInteractiveSnapshotPresentation = { nodes: RawSnapshotNode[]; + /** Presented node indexes for every source index; suppressed noise maps to an empty list. */ + presentedIndexesBySourceIndex: ReadonlyMap; sourceIndexes: ReadonlyMap; }; @@ -33,16 +35,18 @@ export function buildIosInteractiveSnapshotPresentation( nodes: RawSnapshotNode[], ): IosInteractiveSnapshotPresentation { if (nodes.length === 0) { - return { nodes, sourceIndexes: new Map() }; + return { nodes, presentedIndexesBySourceIndex: new Map(), sourceIndexes: new Map() }; } const sourceIndexes = new Map(nodes.map((node) => [node.index, node.index])); const replacements = new Map(); + const representativeSourceIndexesBySourceIndex = new Map>(); const semanticRepresentativeIndexes = new Set(); const sourceNodesByIndex = new Map(nodes.map((node) => [node.index, node])); const suppressedIndexes = new Set(); const ruleContext: SnapshotTreeRuleContext = { replacements, + representativeSourceIndexesBySourceIndex, semanticRepresentativeIndexes, sourceNodesByIndex, suppressedIndexes, @@ -53,7 +57,11 @@ export function buildIosInteractiveSnapshotPresentation( } if (suppressedIndexes.size === 0 && replacements.size === 0) { - return { nodes, sourceIndexes }; + return { + nodes, + presentedIndexesBySourceIndex: new Map(nodes.map((node) => [node.index, [node.index]])), + sourceIndexes, + }; } const presentedSourceNodes = nodes @@ -64,13 +72,47 @@ export function buildIosInteractiveSnapshotPresentation( suppressedIndexes, nodes, ); + const presentedIndexBySourceIndex = new Map( + presentedSourceNodes.map((node, position) => [node.index, presentedNodes[position]!.index]), + ); return { nodes: presentedNodes, - sourceIndexes: new Map( - presentedNodes.map((node, position) => [ + presentedIndexesBySourceIndex: new Map( + nodes.map((node) => [ node.index, - sourceIndexes.get(presentedSourceNodes[position]!.index)!, + resolvePresentedIndexes( + node.index, + presentedIndexBySourceIndex, + representativeSourceIndexesBySourceIndex, + ), ]), ), + sourceIndexes: new Map( + presentedNodes.map((node, position) => [node.index, presentedSourceNodes[position]!.index]), + ), }; } + +function resolvePresentedIndexes( + sourceIndex: number, + presentedIndexBySourceIndex: ReadonlyMap, + representativesBySourceIndex: ReadonlyMap>, + visited = new Set(), +): number[] { + const direct = presentedIndexBySourceIndex.get(sourceIndex); + if (direct !== undefined) return [direct]; + if (visited.has(sourceIndex)) return []; + visited.add(sourceIndex); + const resolved = new Set(); + for (const representative of representativesBySourceIndex.get(sourceIndex) ?? []) { + for (const presentedIndex of resolvePresentedIndexes( + representative, + presentedIndexBySourceIndex, + representativesBySourceIndex, + visited, + )) { + resolved.add(presentedIndex); + } + } + return [...resolved].sort((left, right) => left - right); +} diff --git a/src/daemon/snapshot-presentation/ios/mapping.test.ts b/src/daemon/snapshot-presentation/ios/mapping.test.ts new file mode 100644 index 000000000..9191320c9 --- /dev/null +++ b/src/daemon/snapshot-presentation/ios/mapping.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { buildIosInteractiveSnapshotPresentation } from './index.ts'; + +test('publishes an exact representative for every semantic source index', () => { + const nodes: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 320, height: 640 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Other', + identifier: 'semantic-wrapper', + rect: { x: 20, y: 80, width: 120, height: 40 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Visible child', + rect: { x: 20, y: 80, width: 120, height: 40 }, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'StaticText', + identifier: 'semantic-wrapper', + label: 'Unrelated', + rect: { x: 20, y: 180, width: 120, height: 40 }, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'StaticText', + identifier: 'shared-but-not-a-representative', + rect: { x: 20, y: 240, width: 120, height: 40 }, + }, + { + index: 5, + depth: 2, + parentIndex: 4, + type: 'Other', + identifier: 'shared-but-not-a-representative', + rect: { x: 20, y: 240, width: 120, height: 40 }, + }, + ]; + + const presentation = buildIosInteractiveSnapshotPresentation(nodes); + + expect([...presentation.presentedIndexesBySourceIndex]).toEqual([ + [0, [0]], + [1, [1]], + [2, [1]], + [3, [2]], + [4, [3]], + [5, []], + ]); + expect(presentation.sourceIndexes).toEqual( + new Map([ + [0, 0], + [1, 2], + [2, 3], + [3, 4], + ]), + ); +}); diff --git a/src/daemon/snapshot-presentation/ios/noise.ts b/src/daemon/snapshot-presentation/ios/noise.ts index 6b2292465..c692fb052 100644 --- a/src/daemon/snapshot-presentation/ios/noise.ts +++ b/src/daemon/snapshot-presentation/ios/noise.ts @@ -10,6 +10,8 @@ import { normalizeType } from '@agent-device/contracts/snapshot'; import { collectIosScrollIndicatorPresentation } from './scroll.ts'; import { areRectsApproximatelyEqual, + associateSnapshotPresentation, + collectDescendantsByParentIndex, findDescendant, findLargestViewportRect, forEachDescendant, @@ -26,17 +28,13 @@ export function collectIosPresentationNoiseSuppression( ): void { const { suppressedIndexes } = context; collectIosOffscreenKeyboardSuppression(nodes, context.sourceNodesByIndex, suppressedIndexes); - collectIosStructuralIdentifierSuppression(nodes, suppressedIndexes); + collectIosStructuralIdentifierSuppression(nodes, context); collectIosScrollIndicatorPresentation(nodes, context); - collectIosSearchToolbarSuppression(nodes, context.sourceNodesByIndex, suppressedIndexes); - collectIosActionWrapperSuppression(nodes, suppressedIndexes); + collectIosSearchToolbarSuppression(nodes, context); + collectIosActionWrapperSuppression(nodes, context); collectIosReactNativeOverlayActionPresentation(nodes, context.replacements); - collectIosReactNativeOverlayWrapperSuppression(nodes, suppressedIndexes); - collectIosRepeatedStaticSuppression( - nodes, - suppressedIndexes, - context.semanticRepresentativeIndexes, - ); + collectIosReactNativeOverlayWrapperSuppression(nodes, context); + collectIosRepeatedStaticSuppression(nodes, context); } function collectIosReactNativeOverlayActionPresentation( @@ -111,7 +109,7 @@ function remainingHorizontalPartition( function collectIosReactNativeOverlayWrapperSuppression( nodes: RawSnapshotNode[], - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, ): void { forEachOtherNodeWithLabel(nodes, (node, _nodeLabel, position) => { if (!isReactNativeCollapsedWarningWrapperCandidate(node)) return; @@ -121,7 +119,10 @@ function collectIosReactNativeOverlayWrapperSuppression( collectDescendantNodes(nodes, position), ) ) { - suppressedIndexes.add(node.index); + context.suppressedIndexes.add(node.index); + for (const descendant of collectDescendantNodes(nodes, position)) { + associateSnapshotPresentation(context, node, descendant); + } } }); } @@ -136,24 +137,16 @@ function collectDescendantNodes(nodes: RawSnapshotNode[], position: number): Raw function collectIosRepeatedStaticSuppression( nodes: RawSnapshotNode[], - suppressedIndexes: Set, - semanticRepresentativeIndexes: ReadonlySet, + context: SnapshotTreeRuleContext, ): void { for (let position = 0; position < nodes.length; position += 1) { const node = nodes[position]; const nodeLabel = node?.label?.trim(); - if (!node || suppressedIndexes.has(node.index) || !nodeLabel) { + if (!node || context.suppressedIndexes.has(node.index) || !nodeLabel) { continue; } - collectRepeatedStaticSuppressionForNode( - nodes, - position, - node, - nodeLabel, - suppressedIndexes, - semanticRepresentativeIndexes, - ); + collectRepeatedStaticSuppressionForNode(nodes, position, node, nodeLabel, context); } } @@ -162,72 +155,60 @@ function collectRepeatedStaticSuppressionForNode( position: number, node: RawSnapshotNode, nodeLabel: string, - suppressedIndexes: Set, - semanticRepresentativeIndexes: ReadonlySet, + context: SnapshotTreeRuleContext, ): void { const type = normalizeType(node.type ?? ''); if (type === 'statictext' || type === 'link') { - suppressRepeatedStaticDescendants( - nodes, - position, - nodeLabel, - suppressedIndexes, - semanticRepresentativeIndexes, - ); + suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); return; } if (type !== 'other') { return; } - if (hasEquivalentSemanticDescendant(nodes, position, nodeLabel)) { - suppressedIndexes.add(node.index); + const semanticDescendant = findEquivalentSemanticDescendant(nodes, position, nodeLabel); + if (semanticDescendant) { + context.suppressedIndexes.add(node.index); + associateSnapshotPresentation(context, node, semanticDescendant); return; } - suppressRepeatedStaticDescendants( - nodes, - position, - nodeLabel, - suppressedIndexes, - semanticRepresentativeIndexes, - ); + suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context); } -function hasEquivalentSemanticDescendant( +function findEquivalentSemanticDescendant( nodes: RawSnapshotNode[], position: number, nodeLabel: string, -): boolean { - return Boolean( - findDescendant(nodes, position, (descendant) => { - const type = normalizeType(descendant.type ?? ''); - return ( - (type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) && - descendant.label?.trim() === nodeLabel - ); - }), - ); +): RawSnapshotNode | undefined { + return findDescendant(nodes, position, (descendant) => { + const type = normalizeType(descendant.type ?? ''); + return ( + (type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) && + descendant.label?.trim() === nodeLabel + ); + }); } function suppressRepeatedStaticDescendants( nodes: RawSnapshotNode[], position: number, label: string, - suppressedIndexes: Set, - semanticRepresentativeIndexes: ReadonlySet, + representative: RawSnapshotNode, + context: SnapshotTreeRuleContext, ): void { forEachDescendant(nodes, position, (descendant) => { if ( - !semanticRepresentativeIndexes.has(descendant.index) && + !context.semanticRepresentativeIndexes.has(descendant.index) && isRepeatedStaticNode(descendant, label) ) { - suppressedIndexes.add(descendant.index); + context.suppressedIndexes.add(descendant.index); + associateSnapshotPresentation(context, descendant, representative); } }); } function collectIosActionWrapperSuppression( nodes: RawSnapshotNode[], - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, ): void { forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => { const semanticDescendant = findDescendant(nodes, position, (descendant) => { @@ -239,7 +220,8 @@ function collectIosActionWrapperSuppression( ); }); if (semanticDescendant) { - suppressedIndexes.add(node.index); + context.suppressedIndexes.add(node.index); + associateSnapshotPresentation(context, node, semanticDescendant); } }); } @@ -330,7 +312,7 @@ function suppressOffscreenKeyboardAncestors( function collectIosStructuralIdentifierSuppression( nodes: RawSnapshotNode[], - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, ): void { for (const node of nodes) { if (normalizeType(node.type ?? '') !== 'other') { @@ -342,20 +324,22 @@ function collectIosStructuralIdentifierSuppression( if (!node.identifier?.trim()) { continue; } - suppressedIndexes.add(node.index); + context.suppressedIndexes.add(node.index); + for (const descendant of collectDescendantsByParentIndex(nodes, node.index)) { + associateSnapshotPresentation(context, node, descendant); + } } } function collectIosSearchToolbarSuppression( nodes: RawSnapshotNode[], - sourceNodesByIndex: ReadonlyMap, - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, ): void { for (let position = 0; position < nodes.length; position += 1) { const node = nodes[position]; if (!node) continue; if (isExposedSearchField(node)) { - suppressSearchToolbarDescendants(nodes, position, null, suppressedIndexes); + suppressSearchToolbarDescendants(nodes, position, node, context); continue; } if (!isSearchToolbar(node)) continue; @@ -370,9 +354,10 @@ function collectIosSearchToolbarSuppression( continue; } - suppressedIndexes.add(node.index); - suppressToolbarAncestors(node, sourceNodesByIndex, suppressedIndexes); - suppressSearchToolbarDescendants(nodes, position, innerSearch.index, suppressedIndexes); + context.suppressedIndexes.add(node.index); + associateSnapshotPresentation(context, node, innerSearch); + suppressToolbarAncestors(node, innerSearch, context); + suppressSearchToolbarDescendants(nodes, position, innerSearch, context); } } @@ -388,31 +373,33 @@ function isSearchToolbar(node: RawSnapshotNode): boolean { function suppressSearchToolbarDescendants( nodes: RawSnapshotNode[], position: number, - keptSearchIndex: number | null, - suppressedIndexes: Set, + keptSearch: RawSnapshotNode, + context: SnapshotTreeRuleContext, ): void { forEachDescendant(nodes, position, (descendant) => { - if (descendant.index === keptSearchIndex) { + if (descendant.index === keptSearch.index) { return; } if (shouldSuppressIosSearchToolbarDescendant(descendant)) { - suppressedIndexes.add(descendant.index); + context.suppressedIndexes.add(descendant.index); + associateSnapshotPresentation(context, descendant, keptSearch); } }); } function suppressToolbarAncestors( node: RawSnapshotNode, - sourceNodesByIndex: ReadonlyMap, - suppressedIndexes: Set, + representative: RawSnapshotNode, + context: SnapshotTreeRuleContext, ): void { let current = node; while (typeof current.parentIndex === 'number') { - const parent = sourceNodesByIndex.get(current.parentIndex); + const parent = context.sourceNodesByIndex.get(current.parentIndex); if (!parent || parent.label !== 'Toolbar') { return; } - suppressedIndexes.add(parent.index); + context.suppressedIndexes.add(parent.index); + associateSnapshotPresentation(context, parent, representative); current = parent; } } diff --git a/src/daemon/snapshot-presentation/ios/rows.ts b/src/daemon/snapshot-presentation/ios/rows.ts index 34b54ca6e..7ea3b3971 100644 --- a/src/daemon/snapshot-presentation/ios/rows.ts +++ b/src/daemon/snapshot-presentation/ios/rows.ts @@ -3,6 +3,7 @@ import { normalizeType } from '@agent-device/contracts/snapshot'; import { isSystemScrollIndicatorLabel } from '../../../utils/scroll-indicator.ts'; import { areRectsApproximatelyEqual, + associateSnapshotPresentation, collectDescendants, isDisabledChevronButton, mergeReplacement, @@ -47,6 +48,7 @@ function resolveIosRowLabel( } mergeReplacement(context.replacements, row, { label: title }); context.suppressedIndexes.add(titleNode.index); + associateSnapshotPresentation(context, titleNode, row); return title; } @@ -73,7 +75,7 @@ function collectIosRowPresentationForNode( const rowType = normalizeType(row.type ?? ''); if (rowType === 'button') { const descendants = collectDescendants(nodes, position); - suppressRepeatedRowDescendants(descendants, rowLabel, context.suppressedIndexes, row); + suppressRepeatedRowDescendants(descendants, rowLabel, context, row); return; } if (rowType !== 'cell') { @@ -108,13 +110,8 @@ function collectSwitchRowPresentation( mergeReplacement(context.replacements, switchControl, { identifier: promotedIdentifier }); } context.suppressedIndexes.add(row.index); - suppressSwitchRowDescendants( - descendants, - row, - rowLabel, - switchControl, - context.suppressedIndexes, - ); + associateSnapshotPresentation(context, row, switchControl); + suppressSwitchRowDescendants(descendants, row, rowLabel, switchControl, context); return true; } @@ -129,7 +126,7 @@ function collectButtonRowPresentation( ); if (!rowButton) { if (descendants.some(isDisabledChevronButton)) { - suppressRepeatedRowDescendants(descendants, rowLabel, context.suppressedIndexes, row); + suppressRepeatedRowDescendants(descendants, rowLabel, context, row); } return; } @@ -139,10 +136,11 @@ function collectButtonRowPresentation( } context.suppressedIndexes.add(rowButton.index); + associateSnapshotPresentation(context, rowButton, row); suppressRepeatedRowDescendants( descendants.filter((descendant) => descendant.index !== rowButton.index), rowLabel, - context.suppressedIndexes, + context, row, ); } @@ -152,7 +150,7 @@ function suppressSwitchRowDescendants( row: RawSnapshotNode, rowLabel: string, switchControl: RawSnapshotNode, - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, ): void { for (const descendant of descendants) { if (descendant.index === switchControl.index) { @@ -164,7 +162,8 @@ function suppressSwitchRowDescendants( isIosSwitchValueDescendant(descendant, switchControl) || shouldSuppressRepeatedTextDescendant(descendant, rowLabel) ) { - suppressedIndexes.add(descendant.index); + context.suppressedIndexes.add(descendant.index); + associateSnapshotPresentation(context, descendant, switchControl); } } } @@ -172,7 +171,7 @@ function suppressSwitchRowDescendants( function suppressRepeatedRowDescendants( descendants: RawSnapshotNode[], rowLabel: string, - suppressedIndexes: Set, + context: SnapshotTreeRuleContext, row?: RawSnapshotNode, ): void { for (const descendant of descendants) { @@ -180,7 +179,8 @@ function suppressRepeatedRowDescendants( shouldSuppressRepeatedTextDescendant(descendant, rowLabel) || (row && isEmptyRowButtonWrapper(descendant, row)) ) { - suppressedIndexes.add(descendant.index); + context.suppressedIndexes.add(descendant.index); + if (row) associateSnapshotPresentation(context, descendant, row); } } } diff --git a/src/daemon/snapshot-presentation/ios/scroll.ts b/src/daemon/snapshot-presentation/ios/scroll.ts index 5d8048a74..4956e6d43 100644 --- a/src/daemon/snapshot-presentation/ios/scroll.ts +++ b/src/daemon/snapshot-presentation/ios/scroll.ts @@ -4,6 +4,7 @@ import { isSystemScrollIndicatorLabel, } from '../../../utils/scroll-indicator.ts'; import { + associateSnapshotPresentation, findNearestScrollableContainer, isScrollableSnapshotType, updateReplacement, @@ -47,6 +48,10 @@ function collectIosScrollIndicatorNodePresentation( return; } + if (context.suppressedIndexes.has(node.index)) { + associateSnapshotPresentation(context, node, container); + } + applyScrollIndicatorReplacement(context, container, node, directions); } diff --git a/src/daemon/snapshot-presentation/ios/transitions.ts b/src/daemon/snapshot-presentation/ios/transitions.ts index a221e3a19..697b09c3d 100644 --- a/src/daemon/snapshot-presentation/ios/transitions.ts +++ b/src/daemon/snapshot-presentation/ios/transitions.ts @@ -2,6 +2,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; import { rectContains } from '@agent-device/kernel/rect'; import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; import { + associateSnapshotPresentation, collectChildrenByParent, mergeReplacement, type SnapshotTreeRuleContext, @@ -41,6 +42,8 @@ function collectNavigationTitleAffordances( context.semanticRepresentativeIndexes.add(field.index); context.suppressedIndexes.add(title.index); context.suppressedIndexes.add(image.index); + associateSnapshotPresentation(context, title, field); + associateSnapshotPresentation(context, image, field); } } diff --git a/src/daemon/snapshot-presentation/tree.ts b/src/daemon/snapshot-presentation/tree.ts index a1656a068..5947640e7 100644 --- a/src/daemon/snapshot-presentation/tree.ts +++ b/src/daemon/snapshot-presentation/tree.ts @@ -3,6 +3,8 @@ import { normalizeType } from '@agent-device/contracts/snapshot'; export { areRectsApproximatelyEqual } from '../../utils/rect-center.ts'; export type SnapshotTreeRuleContext = { + /** Exact semantic-source relationships declared by the presentation rule that owns them. */ + representativeSourceIndexesBySourceIndex: Map>; replacements: Map; /** Projected label owners that repeated-text suppression must retain. */ semanticRepresentativeIndexes: Set; @@ -10,6 +12,17 @@ export type SnapshotTreeRuleContext = { suppressedIndexes: Set; }; +export function associateSnapshotPresentation( + context: SnapshotTreeRuleContext, + source: RawSnapshotNode, + representative: RawSnapshotNode, +): void { + const representatives = + context.representativeSourceIndexesBySourceIndex.get(source.index) ?? new Set(); + representatives.add(representative.index); + context.representativeSourceIndexesBySourceIndex.set(source.index, representatives); +} + export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map { const childrenByParent = new Map(); for (const node of nodes) { @@ -33,6 +46,23 @@ export function collectDescendants( return nodes.slice(startPosition + 1, endPosition); } +export function collectDescendantsByParentIndex( + nodes: RawSnapshotNode[], + ancestorIndex: number, +): RawSnapshotNode[] { + const byIndex = new Map(nodes.map((node) => [node.index, node])); + return nodes.filter((node) => { + let parentIndex = node.parentIndex; + const visited = new Set(); + while (typeof parentIndex === 'number' && !visited.has(parentIndex)) { + if (parentIndex === ancestorIndex) return true; + visited.add(parentIndex); + parentIndex = byIndex.get(parentIndex)?.parentIndex; + } + return false; + }); +} + export function findDescendant( nodes: RawSnapshotNode[], startPosition: number,