skills/callstack__agent-device
1
0
Fork 0
mirror of https://github.com/callstack/agent-device.git synced 2026-09-14 20:06:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(maestro): restore conformance invariants (#1889)

Browse Source 
* fix(maestro): restore conformance invariants

* fix(maestro): use exact iOS presentation mappings
This commit is contained in:
Michał Pierzchała
2026-08-20 12:51:43 +02:00
committed by GitHub
parent e65443d774
commit 2d7a310dd0
61 changed files with 552 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.prettierignore
+6
View File
@@ -0,0 +1,6 @@
.claude/**
dist/**
node_modules/**
**/*.md
scripts/maestro-conformance/corpus/**
fallow-baselines/**
packages/maestro/src/internal/__tests__/runtime-targets-typed.test.ts
+64 -10
View File
@@ -213,8 +213,8 @@ test('iOS target resolution preserves distinct nested controls matched by one ex
).toMatchObject({ ok: true, node: { index: 1 }, matches: 2 });
});
test('iOS target resolution keeps raw geometry bound to the canonical source node', () => {
const interactionSnapshot = makeSnapshot([
test('iOS target resolution keeps semantic identity bound to presented geometry', () => {
const semanticSnapshot = makeSnapshot([
{
index: 0,
type: 'Application',
@@ -235,31 +235,85 @@ test('iOS target resolution keeps raw geometry bound to the canonical source nod
rect: { x: 200, y: 100, width: 80, height: 48 },
},
]);
const canonicalSnapshot = makeSnapshot([
interactionSnapshot.nodes[0]!,
{ ...interactionSnapshot.nodes[2]!, index: 1 },
const presentationSnapshot = makeSnapshot([
semanticSnapshot.nodes[0]!,
{ ...semanticSnapshot.nodes[2]!, index: 1 },
]);
expect(
resolveMaestroTargetFromSnapshot(
canonicalSnapshot,
semanticSnapshot,
{ selector: { text: 'Save' }, allowAtomicSelectorDispatch: true },
'ios',
{
interactiveBounds: true,
interaction: {
snapshot: interactionSnapshot,
presentation: {
snapshot: presentationSnapshot,
sourceIndexes: new Map([
[0, 0],
[1, 2],
]),
presentedIndexesBySourceIndex: new Map([
[0, [0]],
[1, []],
[2, [1]],
]),
},
},
),
).toMatchObject({
ok: true,
node: { index: 2 },
rect: { x: 200, y: 100, width: 80, height: 48 },
dispatchCandidates: 1,
});
});
test('iOS target resolution uses one presented candidate universe for geometry and dispatch', () => {
const semanticSnapshot = makeSnapshot([
{
index: 0,
type: 'Button',
identifier: 'save',
rect: { x: 0, y: 100, width: 80, height: 48 },
},
{
index: 1,
type: 'Button',
identifier: 'save',
rect: { x: 200, y: 100, width: 80, height: 48 },
},
]);
const presentationSnapshot = makeSnapshot([
{
...semanticSnapshot.nodes[1]!,
index: 0,
rect: { x: 220, y: 120, width: 80, height: 48 },
},
]);
expect(
resolveMaestroTargetFromSnapshot(
semanticSnapshot,
{ selector: { id: 'save' }, allowAtomicSelectorDispatch: true },
'ios',
{
interactiveBounds: true,
presentation: {
snapshot: presentationSnapshot,
sourceIndexes: new Map([[0, 1]]),
presentedIndexesBySourceIndex: new Map([
[0, []],
[1, [0]],
]),
},
},
),
).toMatchObject({
ok: true,
node: { index: 1 },
rect: { x: 200, y: 100, width: 80, height: 48 },
dispatchCandidates: 2,
rect: { x: 220, y: 120, width: 80, height: 48 },
matches: 1,
dispatchCandidates: 0,
});
});
packages/maestro/src/internal/runtime-target-ranking.ts
+15 -6
View File
@@ -17,28 +17,37 @@ export type MaestroRankedCandidates = {
readonly parentMatched: boolean;
};
export type MaestroCandidateMatches = Pick<MaestroRankedCandidates, 'matches' | 'parentMatched'>;
export function rankMaestroCandidates(
snapshot: SnapshotState,
selector: MaestroSelector,
platform: MaestroPlatform,
childOf?: MaestroSelector,
): MaestroRankedCandidates {
const matches = snapshot.nodes.filter((node) => matchesMaestroTypedSelector(node, selector));
const scoped = scopeMatchesByAncestor(snapshot, matches, childOf);
const scoped = matchMaestroCandidates(snapshot, selector, childOf);
const visible = filterVisibleMaestroMatches({
nodes: snapshot.nodes,
matches: scoped.matches,
platform,
});
return {
matches: scoped.matches,
...scoped,
visible,
ranked: normalizeMaestroSnapshotMatches(snapshot.nodes, visible, selector, platform),
parentMatched: scoped.parentMatched,
ranked: rankVisibleMaestroMatches(snapshot.nodes, visible, selector, platform),
};
}
function normalizeMaestroSnapshotMatches(
export function matchMaestroCandidates(
snapshot: SnapshotState,
selector: MaestroSelector,
childOf?: MaestroSelector,
): MaestroCandidateMatches {
const matches = snapshot.nodes.filter((node) => matchesMaestroTypedSelector(node, selector));
return scopeMatchesByAncestor(snapshot, matches, childOf);
}
export function rankVisibleMaestroMatches(
nodes: SnapshotNode[],
matches: SnapshotNode[],
selector: MaestroSelector,
packages/maestro/src/internal/runtime-targets.ts
+59 -40
View File
@@ -1,9 +1,14 @@
import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
import type { MaestroSelector } from './program-ir.ts';
import type { MaestroPlatform } from './runtime-target-policy.ts';
import { rankMaestroCandidates, selectMaestroSnapshotMatch } from './runtime-target-ranking.ts';
import {
matchMaestroCandidates,
rankMaestroCandidates,
rankVisibleMaestroMatches,
selectMaestroSnapshotMatch,
} from './runtime-target-ranking.ts';
import { pointInsideRect, stripUndefined } from './shared.ts';
import { isDescendantOfSnapshotNode } from './snapshot-policy.ts';
import { isMaestroNodeVisible } from './snapshot-policy.ts';
import { buildSnapshotNodeMap } from '@agent-device/contracts/snapshot';
export type MaestroTargetQuery = {
@@ -22,8 +27,9 @@ export type MaestroTargetEvidence = {
ref?: string;
};
type MaestroInteractionProjection = {
type MaestroInteractivePresentation = {
snapshot: SnapshotState;
presentedIndexesBySourceIndex: ReadonlyMap<number, number[]>;
sourceIndexes: ReadonlyMap<number, number>;
};
@@ -44,10 +50,15 @@ export function resolveMaestroTargetFromSnapshot(
platform: MaestroPlatform,
options: {
interactiveBounds?: boolean;
interaction?: MaestroInteractionProjection;
presentation?: MaestroInteractivePresentation;
} = {},
): MaestroTargetResolution {
const candidates = rankMaestroCandidates(snapshot, query.selector, platform, query.childOf);
const presentedNodes = options.presentation
? createPresentedNodeLookup(options.presentation)
: undefined;
const candidates = presentedNodes
? rankPresentedMaestroCandidates(snapshot, query, platform, presentedNodes)
: rankMaestroCandidates(snapshot, query.selector, platform, query.childOf);
if (!candidates.parentMatched) {
return {
ok: false,
@@ -61,49 +72,58 @@ export function resolveMaestroTargetFromSnapshot(
if (!target) {
return failedTargetResolution(query, matches, rankedMatches, evidence);
}
const interactionCandidates = resolveInteractionCandidates(target.node, query, platform, options);
const interactionTarget = interactionCandidates
? selectMaestroSnapshotMatch(interactionCandidates.mapped, undefined)
: undefined;
const presentedTarget = presentedNodes
? selectMaestroSnapshotMatch(presentedNodes.forSource(target.node), undefined)
: null;
const rect =
options.interactiveBounds === true ? (interactionTarget?.rect ?? target.rect) : target.rect;
options.interactiveBounds === true ? (presentedTarget?.rect ?? target.rect) : target.rect;
return {
ok: true,
node: target.node,
rect,
matches: rankedMatches.length,
dispatchCandidates:
platform === 'ios' && query.allowAtomicSelectorDispatch && !query.childOf
? countInteractionDispatchCandidates(target, interactionCandidates)
platform === 'ios' && query.allowAtomicSelectorDispatch && !query.childOf && presentedNodes
? countInteractionDispatchCandidates(target, rankedMatches, presentedTarget)
: 0,
evidence,
};
}
function resolveInteractionCandidates(
canonicalTarget: SnapshotNode,
function createPresentedNodeLookup(presentation: MaestroInteractivePresentation): {
isVisible: (node: SnapshotNode) => boolean;
forSource: (node: SnapshotNode) => SnapshotNode[];
} {
const presentedByIndex = buildSnapshotNodeMap(presentation.snapshot.nodes);
const forSource = (semanticNode: SnapshotNode): SnapshotNode[] => {
return (presentation.presentedIndexesBySourceIndex.get(semanticNode.index) ?? []).flatMap(
(presentedIndex) => {
const node = presentedByIndex.get(presentedIndex);
return node ? [node] : [];
},
);
};
return {
forSource,
isVisible: (node) =>
forSource(node).some((presentedNode) =>
isMaestroNodeVisible(presentedNode, presentation.snapshot.nodes, 'ios'),
),
};
}
function rankPresentedMaestroCandidates(
snapshot: SnapshotState,
query: MaestroTargetQuery,
platform: MaestroPlatform,
options: {
interaction?: MaestroInteractionProjection;
},
): { all: SnapshotNode[]; mapped: SnapshotNode[] } | undefined {
const interaction = options.interaction;
if (!interaction) return undefined;
const { snapshot, sourceIndexes } = interaction;
const sourceIndex = sourceIndexes.get(canonicalTarget.index);
if (sourceIndex === undefined) return undefined;
const byIndex = buildSnapshotNodeMap(snapshot.nodes);
const source = byIndex.get(sourceIndex);
if (!source) return undefined;
const all = rankMaestroCandidates(snapshot, query.selector, platform, query.childOf).ranked;
presentedNodes: ReturnType<typeof createPresentedNodeLookup>,
) {
const scoped = matchMaestroCandidates(snapshot, query.selector, query.childOf);
const visible = scoped.matches.filter(presentedNodes.isVisible);
return {
all,
mapped: all.filter(
(candidate) =>
candidate.index === source.index ||
isDescendantOfSnapshotNode(snapshot.nodes, candidate, source, byIndex),
),
...scoped,
visible,
ranked: rankVisibleMaestroMatches(snapshot.nodes, visible, query.selector, platform),
};
}
@@ -126,14 +146,13 @@ function failedTargetResolution(
function countInteractionDispatchCandidates(
target: { node: SnapshotNode; rect: Rect },
interactionCandidates: { all: SnapshotNode[]; mapped: SnapshotNode[] } | undefined,
rankedCandidates: SnapshotNode[],
presentedTarget: { node: SnapshotNode; rect: Rect } | null,
): number {
if (!interactionCandidates) return 0;
if (interactionCandidates.all.length !== 1) return interactionCandidates.all.length;
const interactionTarget = selectMaestroSnapshotMatch(interactionCandidates.mapped, undefined);
return interactionTarget &&
interactionTarget.node.hittable !== false &&
haveSameTapPoint(interactionTarget.rect, target.rect)
if (rankedCandidates.length !== 1) return rankedCandidates.length;
return presentedTarget &&
presentedTarget.node.hittable !== false &&
haveSameTapPoint(presentedTarget.rect, target.rect)
? 1
: 0;
}
packages/maestro/test/conformance/verify.test.ts
+10
View File
@@ -10,6 +10,7 @@ import {
CONFORMANCE_DATA_DIR,
CORPUS_DIR,
checkCoverage,
checkCorpusSeals,
checkFixtureSeals,
checkLayer2,
classifyAllFlows,
@@ -43,6 +44,15 @@ test('fixture content is sealed against hand editing', () => {
}
});
test('vendored upstream corpus content matches its manifest sha256', () => {
const drifted = checkCorpusSeals().filter((result) => !result.sealed);
assert.deepEqual(
drifted,
[],
'upstream corpus content drifted — restore the vendored bytes or regenerate from the pinned upstream commit',
);
});
// The seal is only worth having if it actually catches an edit. Prove it does,
// rather than trusting that a hash comparison must work.
test('the seal rejects an edited capture (proof the check has teeth)', () => {
packages/maestro/test/conformance/verify.ts
+27
View File
@@ -11,6 +11,7 @@
// flow our engine parses, or be explicitly listed as unverified.
// - Bug classes: the four #1217 regressions each assert against their fixture.
import fs from 'node:fs';
import { createHash } from 'node:crypto';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { AppError } from '@agent-device/kernel/errors';
@@ -76,6 +77,13 @@ export function loadLayer2(): Layer2Fixture {
export type SealResult = { file: string; sealed: boolean; expected: string; actual?: string };
type CorpusManifest = {
flows: Array<{
file: string;
origin: { kind: 'authored'; note: string } | { kind: 'upstream'; sha256: string };
}>;
};
/**
* Recompute each fixture's content seal. This is what makes "generated from
* upstream" an enforced property rather than a claim in a README: editing a
@@ -89,6 +97,25 @@ export function checkFixtureSeals(): SealResult[] {
});
}
/** Verify vendored upstream source bytes in normal, Java-free per-PR CI. */
export function checkCorpusSeals(): SealResult[] {
const manifest = readJson<CorpusManifest>(path.join(CORPUS_DIR, 'manifest.json'));
return manifest.flows.flatMap((flow) => {
if (flow.origin.kind !== 'upstream') return [];
const actual = createHash('sha256')
.update(fs.readFileSync(path.join(CORPUS_DIR, flow.file)))
.digest('hex');
return [
{
file: flow.file,
sealed: actual === flow.origin.sha256,
expected: flow.origin.sha256,
actual,
},
];
});
}
/**
* Error codes that count as a deliberate parser rejection. Every rejection the
* Maestro parser raises — unsupported command/option, bad value, and even a YAML
scripts/maestro-conformance/corpus/authored/doubletap.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- doubleTapOn: 'Button'
- doubleTapOn: "Button"
scripts/maestro-conformance/corpus/authored/extended-wait.yaml
+2 -2
View File
@@ -2,9 +2,9 @@ appId: com.example.app
---
- extendedWaitUntil:
visible:
id: 'Item'
id: "Item"
timeout: 1000
- extendedWaitUntil:
notVisible:
id: 'Another'
id: "Another"
timeout: 1000
scripts/maestro-conformance/corpus/authored/repeat.yaml
+1 -1
View File
@@ -3,4 +3,4 @@ appId: com.example.app
- repeat:
times: 3
commands:
- tapOn: 'Button'
- tapOn: "Button"
scripts/maestro-conformance/corpus/authored/runflow-child.yaml
+1 -1
View File
@@ -2,4 +2,4 @@ appId: com.example.include
---
- launchApp
- tapOn:
id: 'included-button'
id: "included-button"
scripts/maestro-conformance/corpus/authored/runflow-main.yaml
+2 -2
View File
@@ -1,5 +1,5 @@
appId: com.example.app
---
- tapOn: 'Before'
- tapOn: "Before"
- runFlow: runflow-child.yaml
- tapOn: 'After'
- tapOn: "After"
scripts/maestro-conformance/corpus/authored/scroll-until-visible.yaml
+1 -1
View File
@@ -2,6 +2,6 @@ appId: com.example.app
---
- scrollUntilVisible:
element:
text: 'Test'
text: "Test"
direction: DOWN
timeout: 10000
scripts/maestro-conformance/corpus/bug-classes/percent-decimal-swipe.yaml
+2 -2
View File
@@ -3,5 +3,5 @@
appId: com.example.app
---
- swipe:
start: '50.5%, 50%'
end: '10%, 50%'
start: "50.5%, 50%"
end: "10%, 50%"
scripts/maestro-conformance/corpus/bug-classes/retry-over-cap.yaml
+1 -1
View File
@@ -6,4 +6,4 @@ appId: com.example.app
- retry:
maxRetries: 99
commands:
- tapOn: 'Retry'
- tapOn: "Retry"
scripts/maestro-conformance/corpus/bug-classes/settle-after-tap.yaml
+1 -1
View File
@@ -4,4 +4,4 @@
# same name (no reflectable upstream constant exists).
appId: com.example.app
---
- tapOn: 'Submit'
- tapOn: "Submit"
scripts/maestro-conformance/corpus/bug-classes/target-swipe-missing-direction.yaml
+1 -1
View File
@@ -4,4 +4,4 @@ appId: com.example.app
---
- swipe:
from:
id: 'row'
id: "row"
scripts/maestro-conformance/corpus/invalid/commands-not-a-list.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# The command document must be a sequence.
appId: com.example.app
---
tapOn: 'Button'
tapOn: "Button"
scripts/maestro-conformance/corpus/invalid/malformed-selector.yaml
+1 -1
View File
@@ -2,4 +2,4 @@
appId: com.example.app
---
- tapOn:
- text: 'Button'
- text: "Button"
scripts/maestro-conformance/corpus/invalid/unknown-command.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
# Upstream rejects an unknown command name (typo of tapOn).
appId: com.example.app
---
- tapOnn: 'Button'
- tapOnn: "Button"
scripts/maestro-conformance/corpus/invalid/unknown-selector-field.yaml
+1 -1
View File
@@ -2,5 +2,5 @@
appId: com.example.app
---
- tapOn:
text: 'Button'
text: "Button"
bogusField: true
scripts/maestro-conformance/corpus/upstream/001_assert_visible_by_id.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- assertVisible:
id: 'element_id'
id: "element_id"
scripts/maestro-conformance/corpus/upstream/002_assert_visible_by_text.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- assertVisible:
text: 'Element Text'
text: "Element Text"
scripts/maestro-conformance/corpus/upstream/008_tap_on_element.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- tapOn:
text: '.*button.*'
text: ".*button.*"
scripts/maestro-conformance/corpus/upstream/009_skip_optional_elements.yaml
+3 -3
View File
@@ -1,8 +1,8 @@
appId: com.example.app
---
- tapOn:
text: 'Optional Element'
text: "Optional Element"
optional: true
- assertVisible:
text: 'Non Optional'
optional: false
text: "Non Optional"
optional: false
scripts/maestro-conformance/corpus/upstream/010_scroll.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- scroll
- scroll
scripts/maestro-conformance/corpus/upstream/011_back_press.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- back
- back
scripts/maestro-conformance/corpus/upstream/012_input_text.yaml
+2 -2
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- inputText: 'Hello World'
- inputText: user@example.com
- inputText: "Hello World"
- inputText: user@example.com
scripts/maestro-conformance/corpus/upstream/013_launch_app.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- launchApp
- launchApp
scripts/maestro-conformance/corpus/upstream/014_tap_on_point.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- tapOn:
point: 100,200
point: 100,200
scripts/maestro-conformance/corpus/upstream/017_swipe.yaml
+1 -1
View File
@@ -3,4 +3,4 @@ appId: com.example.app
- swipe:
start: 100,500
end: 100,200
duration: 3000
duration: 3000
scripts/maestro-conformance/corpus/upstream/021_launch_app_with_clear_state.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- launchApp:
clearState: true
clearState: true
scripts/maestro-conformance/corpus/upstream/026_assert_not_visible.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- assertNotVisible:
id: 'element_id'
id: "element_id"
scripts/maestro-conformance/corpus/upstream/027_open_link.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- openLink: https://example.com
- openLink: https://example.com
scripts/maestro-conformance/corpus/upstream/029_long_press_on_element.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- longPressOn:
text: '.*button.*'
text: ".*button.*"
scripts/maestro-conformance/corpus/upstream/032_element_index.yaml
+1 -1
View File
@@ -7,4 +7,4 @@ appId: com.example.app
- tapOn:
text: Item.*
index: ${0 + 1}
retryTapIfNoChange: false
retryTapIfNoChange: false
scripts/maestro-conformance/corpus/upstream/034_press_key.yaml
+1
View File
@@ -29,3 +29,4 @@ appId: com.example.app
- pressKey: TV Input HDMI 1
- pressKey: TV Input HDMI 2
- pressKey: TV Input HDMI 3
scripts/maestro-conformance/corpus/upstream/039_hide_keyboard.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- hideKeyboard
- hideKeyboard
scripts/maestro-conformance/corpus/upstream/042_extended_wait.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
appId: com.example.app
env:
TIMEOUT: 1000
TIMEOUT: 1000
---
- extendedWaitUntil:
visible: Item
scripts/maestro-conformance/corpus/upstream/053_repeat_times.yaml
+2 -2
View File
@@ -4,10 +4,10 @@ appId: com.other.app
times: 3
commands:
- tapOn: Button
- assertVisible: '3'
- assertVisible: "3"
- evalScript: ${output.list = [1, 2, 3]}
- repeat:
times: ${output.list.length}
commands:
- tapOn: Button
- assertVisible: '6'
- assertVisible: "6"
scripts/maestro-conformance/corpus/upstream/059_directional_swipe_command.yaml
+1 -1
View File
@@ -2,4 +2,4 @@ appId: com.example.app
---
- swipe:
direction: RIGHT
duration: 500
duration: 500
scripts/maestro-conformance/corpus/upstream/061_launchApp_withoutStopping.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- launchApp:
stopApp: false
stopApp: false
scripts/maestro-conformance/corpus/upstream/062_copy_paste_text.yaml
+1 -1
View File
@@ -1,5 +1,5 @@
appId: com.example.app
---
- copyTextFrom:
id: 'myId'
id: "myId"
- pasteText
scripts/maestro-conformance/corpus/upstream/067_assertTrue_pass.yaml
+1 -1
View File
@@ -1,3 +1,3 @@
appId: com.example.app
---
- assertTrue: ${1+1}
- assertTrue: ${1+1}
scripts/maestro-conformance/corpus/upstream/069_wait_for_animation_to_end.yaml
+1 -1
View File
@@ -1,4 +1,4 @@
appId: com.example.app
---
- waitForAnimationToEnd:
timeout: 500
timeout: 500
scripts/maestro-conformance/corpus/upstream/074_directional_swipe_element.yaml
+1 -1
View File
@@ -3,4 +3,4 @@ appId: com.example.app
- swipe:
direction: RIGHT
from:
text: 'swiping element'
text: "swiping element"
scripts/maestro-conformance/corpus/upstream/076_optional_assertion.yaml
+4 -4
View File
@@ -3,16 +3,16 @@ appId: com.example.app
- scrollUntilVisible:
timeout: 1
element:
id: 'not_found'
id: "not_found"
optional: true
- assertTrue:
condition: 'false'
condition: "false"
optional: true
- extendedWaitUntil:
visible:
id: 'not_found'
id: "not_found"
timeout: 1
optional: true
- assertVisible:
text: 'Button'
text: "Button"
optional: true
scripts/maestro-conformance/corpus/upstream/078_swipe_relative.yaml
+3 -3
View File
@@ -1,6 +1,6 @@
appId: com.example.app
---
- swipe:
start: '50%,30%'
end: '50%,60%'
duration: 3000
start: "50%,30%"
end: "50%,60%"
duration: 3000
scripts/maestro-conformance/corpus/upstream/079_scroll_until_visible.yaml
+2 -2
View File
@@ -2,8 +2,8 @@ appId: com.example.app
---
- scrollUntilVisible:
element:
text: 'Test'
text: "Test"
speed: 100
visibilityPercentage: 100
direction: DOWN
timeout: 10
timeout: 10
scripts/maestro-conformance/corpus/upstream/114_child_of_selector.yaml
+4 -4
View File
@@ -1,10 +1,10 @@
appId: com.example.app
---
- assertVisible:
text: 'child_id'
text: "child_id"
childOf:
text: 'parent_id_1'
text: "parent_id_1"
- assertNotVisible:
text: 'child_id'
text: "child_id"
childOf:
text: 'parent_id_3'
text: "parent_id_3"
scripts/maestro-conformance/corpus/upstream/120_tap_on_element_retryTapIfNoChange.yaml
+2 -2
View File
@@ -1,5 +1,5 @@
appId: com.example.app
---
- tapOn:
text: '.*button.*'
retryTapIfNoChange: true
text: ".*button.*"
retryTapIfNoChange: true
scripts/maestro-conformance/corpus/upstream/131_setPermissions.yaml
+1 -1
View File
@@ -3,4 +3,4 @@ appId: com.example.app
- setPermissions:
permissions:
all: deny
notifications: unset
notifications: unset
src/daemon/adapters/maestro/__tests__/daemon-runtime-port-observation.test.ts
+71
View File
@@ -10,6 +10,7 @@ import { createDaemonMaestroRuntimePort } from '../daemon-runtime-port.ts';
import {
MAESTRO_OBSERVATION_POLL_MS,
maestroSnapshotSignature,
observeTypedMaestroCondition,
resolveTypedMaestroTarget,
waitForTypedSnapshotStability,
} from '../daemon-runtime-port-observation.ts';
@@ -130,6 +131,76 @@ test('computes expensive target evidence only for the command policies that cons
expect(atomicRetry.dispatchSelector).toEqual({ key: 'id', value: 'continue' });
});
test('matches iOS Maestro ids on semantic nodes suppressed from interactive presentation', async () => {
const snapshot = makeSnapshot([
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
{
index: 1,
parentIndex: 0,
type: 'Other',
identifier: 'inert-title',
rect: { x: 20, y: 100, width: 180, height: 32 },
},
{
index: 2,
parentIndex: 1,
type: 'StaticText',
label: 'Inert surface',
rect: { x: 20, y: 100, width: 180, height: 32 },
},
]);
await expect(
observeTypedMaestroCondition({
condition: { kind: 'visible', selector: { id: 'inert-title' } },
timeoutMs: 0,
context: { generation: 0, env: {} },
snapshot: async () => snapshot,
dependencies: makeDependencies(),
platform: 'ios',
}),
).resolves.toMatchObject({ matched: true, visible: true, candidateCount: 1, ref: 'e2' });
});
test('reports iOS Maestro evidence refs in semantic snapshot space after earlier suppression', async () => {
const snapshot = makeSnapshot([
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
{ index: 1, parentIndex: 0, type: 'Other', identifier: 'discarded-wrapper' },
{
index: 2,
parentIndex: 1,
type: 'StaticText',
label: 'Earlier content',
rect: { x: 20, y: 40, width: 180, height: 32 },
},
{
index: 3,
parentIndex: 0,
type: 'Other',
identifier: 'target-wrapper',
rect: { x: 20, y: 100, width: 180, height: 32 },
},
{
index: 4,
parentIndex: 3,
type: 'StaticText',
label: 'Target content',
rect: { x: 20, y: 100, width: 180, height: 32 },
},
]);
await expect(
observeTypedMaestroCondition({
condition: { kind: 'visible', selector: { id: 'target-wrapper' } },
timeoutMs: 0,
context: { generation: 0, env: {} },
snapshot: async () => snapshot,
dependencies: makeDependencies(),
platform: 'ios',
}),
).resolves.toMatchObject({ matched: true, visible: true, candidateCount: 1, ref: 'e4' });
});
test('compares snapshots before sleeping and captures once beyond a zero settle budget', async () => {
const clock = { value: 0 };
const captures = [
src/daemon/adapters/maestro/__tests__/daemon-runtime-port-target-geometry.test.ts
+2 -2
View File
@@ -62,7 +62,7 @@ test('uses canonical iOS presentation only for atomic selector uniqueness', asyn
expect(requests[1]?.positionals).toEqual(['text="First"']);
});
test('uses resolved iOS geometry when canonical presentation changes target bounds', async () => {
test('uses exact iOS representative geometry when presentation suppresses the semantic target', async () => {
const requests: DaemonRequest[] = [];
const port = createDaemonMaestroRuntimePort({
baseReq: makeBaseRequest({ flags: { platform: 'ios', replayBackend: 'maestro' } }),
@@ -134,7 +134,7 @@ test('uses resolved iOS geometry when canonical presentation changes target boun
expect(requests.map(({ command }) => command)).toEqual(['snapshot', 'click']);
expect(
requests.filter(({ command }) => command === 'click').map(({ positionals }) => positionals),
).toEqual([['56', '121']]);
).toEqual([['266', '121']]);
});
test('uses the selected iOS node interactive bounds without changing raw target matching', async () => {
src/daemon/adapters/maestro/daemon-runtime-port-observation.ts
+8 -4
View File
@@ -104,17 +104,21 @@ function resolveTargetFromSnapshot(params: {
params.platform === 'ios'
? buildIosInteractiveSnapshotPresentation(params.snapshot.nodes)
: undefined;
const canonicalSnapshot = presentation
const interactiveSnapshot = presentation
? { ...params.snapshot, nodes: attachRefs(presentation.nodes) }
: params.snapshot;
const resolution = resolveMaestroTargetFromSnapshot(
canonicalSnapshot,
params.snapshot,
params.query,
params.platform,
{
interactiveBounds: params.mode === 'tap',
interaction: presentation
? { snapshot: params.snapshot, sourceIndexes: presentation.sourceIndexes }
presentation: presentation
? {
snapshot: interactiveSnapshot,
sourceIndexes: presentation.sourceIndexes,
presentedIndexesBySourceIndex: presentation.presentedIndexesBySourceIndex,
}
: undefined,
},
);
src/daemon/snapshot-presentation/ios/index.ts
+47 -5
View File
@@ -26,6 +26,8 @@ export function presentIosInteractiveSnapshot(nodes: RawSnapshotNode[]): RawSnap
export type IosInteractiveSnapshotPresentation = {
nodes: RawSnapshotNode[];
/** Presented node indexes for every source index; suppressed noise maps to an empty list. */
presentedIndexesBySourceIndex: ReadonlyMap<number, number[]>;
sourceIndexes: ReadonlyMap<number, number>;
};
@@ -33,16 +35,18 @@ export function buildIosInteractiveSnapshotPresentation(
nodes: RawSnapshotNode[],
): IosInteractiveSnapshotPresentation {
if (nodes.length === 0) {
return { nodes, sourceIndexes: new Map() };
return { nodes, presentedIndexesBySourceIndex: new Map(), sourceIndexes: new Map() };
}
const sourceIndexes = new Map(nodes.map((node) => [node.index, node.index]));
const replacements = new Map<number, RawSnapshotNode>();
const representativeSourceIndexesBySourceIndex = new Map<number, Set<number>>();
const semanticRepresentativeIndexes = new Set<number>();
const sourceNodesByIndex = new Map(nodes.map((node) => [node.index, node]));
const suppressedIndexes = new Set<number>();
const ruleContext: SnapshotTreeRuleContext = {
replacements,
representativeSourceIndexesBySourceIndex,
semanticRepresentativeIndexes,
sourceNodesByIndex,
suppressedIndexes,
@@ -53,7 +57,11 @@ export function buildIosInteractiveSnapshotPresentation(
}
if (suppressedIndexes.size === 0 && replacements.size === 0) {
return { nodes, sourceIndexes };
return {
nodes,
presentedIndexesBySourceIndex: new Map(nodes.map((node) => [node.index, [node.index]])),
sourceIndexes,
};
}
const presentedSourceNodes = nodes
@@ -64,13 +72,47 @@ export function buildIosInteractiveSnapshotPresentation(
suppressedIndexes,
nodes,
);
const presentedIndexBySourceIndex = new Map(
presentedSourceNodes.map((node, position) => [node.index, presentedNodes[position]!.index]),
);
return {
nodes: presentedNodes,
sourceIndexes: new Map(
presentedNodes.map((node, position) => [
presentedIndexesBySourceIndex: new Map(
nodes.map((node) => [
node.index,
sourceIndexes.get(presentedSourceNodes[position]!.index)!,
resolvePresentedIndexes(
node.index,
presentedIndexBySourceIndex,
representativeSourceIndexesBySourceIndex,
),
]),
),
sourceIndexes: new Map(
presentedNodes.map((node, position) => [node.index, presentedSourceNodes[position]!.index]),
),
};
}
function resolvePresentedIndexes(
sourceIndex: number,
presentedIndexBySourceIndex: ReadonlyMap<number, number>,
representativesBySourceIndex: ReadonlyMap<number, ReadonlySet<number>>,
visited = new Set<number>(),
): number[] {
const direct = presentedIndexBySourceIndex.get(sourceIndex);
if (direct !== undefined) return [direct];
if (visited.has(sourceIndex)) return [];
visited.add(sourceIndex);
const resolved = new Set<number>();
for (const representative of representativesBySourceIndex.get(sourceIndex) ?? []) {
for (const presentedIndex of resolvePresentedIndexes(
representative,
presentedIndexBySourceIndex,
representativesBySourceIndex,
visited,
)) {
resolved.add(presentedIndex);
}
}
return [...resolved].sort((left, right) => left - right);
}
src/daemon/snapshot-presentation/ios/mapping.test.ts
+69
View File
@@ -0,0 +1,69 @@
import { expect, test } from 'vitest';
import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import { buildIosInteractiveSnapshotPresentation } from './index.ts';
test('publishes an exact representative for every semantic source index', () => {
const nodes: RawSnapshotNode[] = [
{ index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 320, height: 640 } },
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Other',
identifier: 'semantic-wrapper',
rect: { x: 20, y: 80, width: 120, height: 40 },
},
{
index: 2,
depth: 2,
parentIndex: 1,
type: 'StaticText',
label: 'Visible child',
rect: { x: 20, y: 80, width: 120, height: 40 },
},
{
index: 3,
depth: 1,
parentIndex: 0,
type: 'StaticText',
identifier: 'semantic-wrapper',
label: 'Unrelated',
rect: { x: 20, y: 180, width: 120, height: 40 },
},
{
index: 4,
depth: 1,
parentIndex: 0,
type: 'StaticText',
identifier: 'shared-but-not-a-representative',
rect: { x: 20, y: 240, width: 120, height: 40 },
},
{
index: 5,
depth: 2,
parentIndex: 4,
type: 'Other',
identifier: 'shared-but-not-a-representative',
rect: { x: 20, y: 240, width: 120, height: 40 },
},
];
const presentation = buildIosInteractiveSnapshotPresentation(nodes);
expect([...presentation.presentedIndexesBySourceIndex]).toEqual([
[0, [0]],
[1, [1]],
[2, [1]],
[3, [2]],
[4, [3]],
[5, []],
]);
expect(presentation.sourceIndexes).toEqual(
new Map([
[0, 0],
[1, 2],
[2, 3],
[3, 4],
]),
);
});
src/daemon/snapshot-presentation/ios/noise.ts
+60 -73
View File
@@ -10,6 +10,8 @@ import { normalizeType } from '@agent-device/contracts/snapshot';
import { collectIosScrollIndicatorPresentation } from './scroll.ts';
import {
areRectsApproximatelyEqual,
associateSnapshotPresentation,
collectDescendantsByParentIndex,
findDescendant,
findLargestViewportRect,
forEachDescendant,
@@ -26,17 +28,13 @@ export function collectIosPresentationNoiseSuppression(
): void {
const { suppressedIndexes } = context;
collectIosOffscreenKeyboardSuppression(nodes, context.sourceNodesByIndex, suppressedIndexes);
collectIosStructuralIdentifierSuppression(nodes, suppressedIndexes);
collectIosStructuralIdentifierSuppression(nodes, context);
collectIosScrollIndicatorPresentation(nodes, context);
collectIosSearchToolbarSuppression(nodes, context.sourceNodesByIndex, suppressedIndexes);
collectIosActionWrapperSuppression(nodes, suppressedIndexes);
collectIosSearchToolbarSuppression(nodes, context);
collectIosActionWrapperSuppression(nodes, context);
collectIosReactNativeOverlayActionPresentation(nodes, context.replacements);
collectIosReactNativeOverlayWrapperSuppression(nodes, suppressedIndexes);
collectIosRepeatedStaticSuppression(
nodes,
suppressedIndexes,
context.semanticRepresentativeIndexes,
);
collectIosReactNativeOverlayWrapperSuppression(nodes, context);
collectIosRepeatedStaticSuppression(nodes, context);
}
function collectIosReactNativeOverlayActionPresentation(
@@ -111,7 +109,7 @@ function remainingHorizontalPartition(
function collectIosReactNativeOverlayWrapperSuppression(
nodes: RawSnapshotNode[],
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
): void {
forEachOtherNodeWithLabel(nodes, (node, _nodeLabel, position) => {
if (!isReactNativeCollapsedWarningWrapperCandidate(node)) return;
@@ -121,7 +119,10 @@ function collectIosReactNativeOverlayWrapperSuppression(
collectDescendantNodes(nodes, position),
)
) {
suppressedIndexes.add(node.index);
context.suppressedIndexes.add(node.index);
for (const descendant of collectDescendantNodes(nodes, position)) {
associateSnapshotPresentation(context, node, descendant);
}
}
});
}
@@ -136,24 +137,16 @@ function collectDescendantNodes(nodes: RawSnapshotNode[], position: number): Raw
function collectIosRepeatedStaticSuppression(
nodes: RawSnapshotNode[],
suppressedIndexes: Set<number>,
semanticRepresentativeIndexes: ReadonlySet<number>,
context: SnapshotTreeRuleContext,
): void {
for (let position = 0; position < nodes.length; position += 1) {
const node = nodes[position];
const nodeLabel = node?.label?.trim();
if (!node || suppressedIndexes.has(node.index) || !nodeLabel) {
if (!node || context.suppressedIndexes.has(node.index) || !nodeLabel) {
continue;
}
collectRepeatedStaticSuppressionForNode(
nodes,
position,
node,
nodeLabel,
suppressedIndexes,
semanticRepresentativeIndexes,
);
collectRepeatedStaticSuppressionForNode(nodes, position, node, nodeLabel, context);
}
}
@@ -162,72 +155,60 @@ function collectRepeatedStaticSuppressionForNode(
position: number,
node: RawSnapshotNode,
nodeLabel: string,
suppressedIndexes: Set<number>,
semanticRepresentativeIndexes: ReadonlySet<number>,
context: SnapshotTreeRuleContext,
): void {
const type = normalizeType(node.type ?? '');
if (type === 'statictext' || type === 'link') {
suppressRepeatedStaticDescendants(
nodes,
position,
nodeLabel,
suppressedIndexes,
semanticRepresentativeIndexes,
);
suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context);
return;
}
if (type !== 'other') {
return;
}
if (hasEquivalentSemanticDescendant(nodes, position, nodeLabel)) {
suppressedIndexes.add(node.index);
const semanticDescendant = findEquivalentSemanticDescendant(nodes, position, nodeLabel);
if (semanticDescendant) {
context.suppressedIndexes.add(node.index);
associateSnapshotPresentation(context, node, semanticDescendant);
return;
}
suppressRepeatedStaticDescendants(
nodes,
position,
nodeLabel,
suppressedIndexes,
semanticRepresentativeIndexes,
);
suppressRepeatedStaticDescendants(nodes, position, nodeLabel, node, context);
}
function hasEquivalentSemanticDescendant(
function findEquivalentSemanticDescendant(
nodes: RawSnapshotNode[],
position: number,
nodeLabel: string,
): boolean {
return Boolean(
findDescendant(nodes, position, (descendant) => {
const type = normalizeType(descendant.type ?? '');
return (
(type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) &&
descendant.label?.trim() === nodeLabel
);
}),
);
): RawSnapshotNode | undefined {
return findDescendant(nodes, position, (descendant) => {
const type = normalizeType(descendant.type ?? '');
return (
(type === 'link' || type === 'searchfield' || isScrollableSnapshotType(descendant.type)) &&
descendant.label?.trim() === nodeLabel
);
});
}
function suppressRepeatedStaticDescendants(
nodes: RawSnapshotNode[],
position: number,
label: string,
suppressedIndexes: Set<number>,
semanticRepresentativeIndexes: ReadonlySet<number>,
representative: RawSnapshotNode,
context: SnapshotTreeRuleContext,
): void {
forEachDescendant(nodes, position, (descendant) => {
if (
!semanticRepresentativeIndexes.has(descendant.index) &&
!context.semanticRepresentativeIndexes.has(descendant.index) &&
isRepeatedStaticNode(descendant, label)
) {
suppressedIndexes.add(descendant.index);
context.suppressedIndexes.add(descendant.index);
associateSnapshotPresentation(context, descendant, representative);
}
});
}
function collectIosActionWrapperSuppression(
nodes: RawSnapshotNode[],
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
): void {
forEachOtherNodeWithLabel(nodes, (node, nodeLabel, position) => {
const semanticDescendant = findDescendant(nodes, position, (descendant) => {
@@ -239,7 +220,8 @@ function collectIosActionWrapperSuppression(
);
});
if (semanticDescendant) {
suppressedIndexes.add(node.index);
context.suppressedIndexes.add(node.index);
associateSnapshotPresentation(context, node, semanticDescendant);
}
});
}
@@ -330,7 +312,7 @@ function suppressOffscreenKeyboardAncestors(
function collectIosStructuralIdentifierSuppression(
nodes: RawSnapshotNode[],
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
): void {
for (const node of nodes) {
if (normalizeType(node.type ?? '') !== 'other') {
@@ -342,20 +324,22 @@ function collectIosStructuralIdentifierSuppression(
if (!node.identifier?.trim()) {
continue;
}
suppressedIndexes.add(node.index);
context.suppressedIndexes.add(node.index);
for (const descendant of collectDescendantsByParentIndex(nodes, node.index)) {
associateSnapshotPresentation(context, node, descendant);
}
}
}
function collectIosSearchToolbarSuppression(
nodes: RawSnapshotNode[],
sourceNodesByIndex: ReadonlyMap<number, RawSnapshotNode>,
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
): void {
for (let position = 0; position < nodes.length; position += 1) {
const node = nodes[position];
if (!node) continue;
if (isExposedSearchField(node)) {
suppressSearchToolbarDescendants(nodes, position, null, suppressedIndexes);
suppressSearchToolbarDescendants(nodes, position, node, context);
continue;
}
if (!isSearchToolbar(node)) continue;
@@ -370,9 +354,10 @@ function collectIosSearchToolbarSuppression(
continue;
}
suppressedIndexes.add(node.index);
suppressToolbarAncestors(node, sourceNodesByIndex, suppressedIndexes);
suppressSearchToolbarDescendants(nodes, position, innerSearch.index, suppressedIndexes);
context.suppressedIndexes.add(node.index);
associateSnapshotPresentation(context, node, innerSearch);
suppressToolbarAncestors(node, innerSearch, context);
suppressSearchToolbarDescendants(nodes, position, innerSearch, context);
}
}
@@ -388,31 +373,33 @@ function isSearchToolbar(node: RawSnapshotNode): boolean {
function suppressSearchToolbarDescendants(
nodes: RawSnapshotNode[],
position: number,
keptSearchIndex: number | null,
suppressedIndexes: Set<number>,
keptSearch: RawSnapshotNode,
context: SnapshotTreeRuleContext,
): void {
forEachDescendant(nodes, position, (descendant) => {
if (descendant.index === keptSearchIndex) {
if (descendant.index === keptSearch.index) {
return;
}
if (shouldSuppressIosSearchToolbarDescendant(descendant)) {
suppressedIndexes.add(descendant.index);
context.suppressedIndexes.add(descendant.index);
associateSnapshotPresentation(context, descendant, keptSearch);
}
});
}
function suppressToolbarAncestors(
node: RawSnapshotNode,
sourceNodesByIndex: ReadonlyMap<number, RawSnapshotNode>,
suppressedIndexes: Set<number>,
representative: RawSnapshotNode,
context: SnapshotTreeRuleContext,
): void {
let current = node;
while (typeof current.parentIndex === 'number') {
const parent = sourceNodesByIndex.get(current.parentIndex);
const parent = context.sourceNodesByIndex.get(current.parentIndex);
if (!parent || parent.label !== 'Toolbar') {
return;
}
suppressedIndexes.add(parent.index);
context.suppressedIndexes.add(parent.index);
associateSnapshotPresentation(context, parent, representative);
current = parent;
}
}
src/daemon/snapshot-presentation/ios/rows.ts
+14 -14
View File
@@ -3,6 +3,7 @@ import { normalizeType } from '@agent-device/contracts/snapshot';
import { isSystemScrollIndicatorLabel } from '../../../utils/scroll-indicator.ts';
import {
areRectsApproximatelyEqual,
associateSnapshotPresentation,
collectDescendants,
isDisabledChevronButton,
mergeReplacement,
@@ -47,6 +48,7 @@ function resolveIosRowLabel(
}
mergeReplacement(context.replacements, row, { label: title });
context.suppressedIndexes.add(titleNode.index);
associateSnapshotPresentation(context, titleNode, row);
return title;
}
@@ -73,7 +75,7 @@ function collectIosRowPresentationForNode(
const rowType = normalizeType(row.type ?? '');
if (rowType === 'button') {
const descendants = collectDescendants(nodes, position);
suppressRepeatedRowDescendants(descendants, rowLabel, context.suppressedIndexes, row);
suppressRepeatedRowDescendants(descendants, rowLabel, context, row);
return;
}
if (rowType !== 'cell') {
@@ -108,13 +110,8 @@ function collectSwitchRowPresentation(
mergeReplacement(context.replacements, switchControl, { identifier: promotedIdentifier });
}
context.suppressedIndexes.add(row.index);
suppressSwitchRowDescendants(
descendants,
row,
rowLabel,
switchControl,
context.suppressedIndexes,
);
associateSnapshotPresentation(context, row, switchControl);
suppressSwitchRowDescendants(descendants, row, rowLabel, switchControl, context);
return true;
}
@@ -129,7 +126,7 @@ function collectButtonRowPresentation(
);
if (!rowButton) {
if (descendants.some(isDisabledChevronButton)) {
suppressRepeatedRowDescendants(descendants, rowLabel, context.suppressedIndexes, row);
suppressRepeatedRowDescendants(descendants, rowLabel, context, row);
}
return;
}
@@ -139,10 +136,11 @@ function collectButtonRowPresentation(
}
context.suppressedIndexes.add(rowButton.index);
associateSnapshotPresentation(context, rowButton, row);
suppressRepeatedRowDescendants(
descendants.filter((descendant) => descendant.index !== rowButton.index),
rowLabel,
context.suppressedIndexes,
context,
row,
);
}
@@ -152,7 +150,7 @@ function suppressSwitchRowDescendants(
row: RawSnapshotNode,
rowLabel: string,
switchControl: RawSnapshotNode,
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
): void {
for (const descendant of descendants) {
if (descendant.index === switchControl.index) {
@@ -164,7 +162,8 @@ function suppressSwitchRowDescendants(
isIosSwitchValueDescendant(descendant, switchControl) ||
shouldSuppressRepeatedTextDescendant(descendant, rowLabel)
) {
suppressedIndexes.add(descendant.index);
context.suppressedIndexes.add(descendant.index);
associateSnapshotPresentation(context, descendant, switchControl);
}
}
}
@@ -172,7 +171,7 @@ function suppressSwitchRowDescendants(
function suppressRepeatedRowDescendants(
descendants: RawSnapshotNode[],
rowLabel: string,
suppressedIndexes: Set<number>,
context: SnapshotTreeRuleContext,
row?: RawSnapshotNode,
): void {
for (const descendant of descendants) {
@@ -180,7 +179,8 @@ function suppressRepeatedRowDescendants(
shouldSuppressRepeatedTextDescendant(descendant, rowLabel) ||
(row && isEmptyRowButtonWrapper(descendant, row))
) {
suppressedIndexes.add(descendant.index);
context.suppressedIndexes.add(descendant.index);
if (row) associateSnapshotPresentation(context, descendant, row);
}
}
}
src/daemon/snapshot-presentation/ios/scroll.ts
+5
View File
@@ -4,6 +4,7 @@ import {
isSystemScrollIndicatorLabel,
} from '../../../utils/scroll-indicator.ts';
import {
associateSnapshotPresentation,
findNearestScrollableContainer,
isScrollableSnapshotType,
updateReplacement,
@@ -47,6 +48,10 @@ function collectIosScrollIndicatorNodePresentation(
return;
}
if (context.suppressedIndexes.has(node.index)) {
associateSnapshotPresentation(context, node, container);
}
applyScrollIndicatorReplacement(context, container, node, directions);
}
src/daemon/snapshot-presentation/ios/transitions.ts
+3
View File
@@ -2,6 +2,7 @@ import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
import { rectContains } from '@agent-device/kernel/rect';
import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot';
import {
associateSnapshotPresentation,
collectChildrenByParent,
mergeReplacement,
type SnapshotTreeRuleContext,
@@ -41,6 +42,8 @@ function collectNavigationTitleAffordances(
context.semanticRepresentativeIndexes.add(field.index);
context.suppressedIndexes.add(title.index);
context.suppressedIndexes.add(image.index);
associateSnapshotPresentation(context, title, field);
associateSnapshotPresentation(context, image, field);
}
}
src/daemon/snapshot-presentation/tree.ts
+30
View File
@@ -3,6 +3,8 @@ import { normalizeType } from '@agent-device/contracts/snapshot';
export { areRectsApproximatelyEqual } from '../../utils/rect-center.ts';
export type SnapshotTreeRuleContext = {
/** Exact semantic-source relationships declared by the presentation rule that owns them. */
representativeSourceIndexesBySourceIndex: Map<number, Set<number>>;
replacements: Map<number, RawSnapshotNode>;
/** Projected label owners that repeated-text suppression must retain. */
semanticRepresentativeIndexes: Set<number>;
@@ -10,6 +12,17 @@ export type SnapshotTreeRuleContext = {
suppressedIndexes: Set<number>;
};
export function associateSnapshotPresentation(
context: SnapshotTreeRuleContext,
source: RawSnapshotNode,
representative: RawSnapshotNode,
): void {
const representatives =
context.representativeSourceIndexesBySourceIndex.get(source.index) ?? new Set<number>();
representatives.add(representative.index);
context.representativeSourceIndexesBySourceIndex.set(source.index, representatives);
}
export function collectChildrenByParent(nodes: RawSnapshotNode[]): Map<number, RawSnapshotNode[]> {
const childrenByParent = new Map<number, RawSnapshotNode[]>();
for (const node of nodes) {
@@ -33,6 +46,23 @@ export function collectDescendants(
return nodes.slice(startPosition + 1, endPosition);
}
export function collectDescendantsByParentIndex(
nodes: RawSnapshotNode[],
ancestorIndex: number,
): RawSnapshotNode[] {
const byIndex = new Map(nodes.map((node) => [node.index, node]));
return nodes.filter((node) => {
let parentIndex = node.parentIndex;
const visited = new Set<number>();
while (typeof parentIndex === 'number' && !visited.has(parentIndex)) {
if (parentIndex === ancestorIndex) return true;
visited.add(parentIndex);
parentIndex = byIndex.get(parentIndex)?.parentIndex;
}
return false;
});
}
export function findDescendant(
nodes: RawSnapshotNode[],
startPosition: number,