fix: recover Android snapshots from system-only helper output (#861)

* fix: recover Android snapshots from system-only helper output

* fix: tighten Android snapshot recovery follow-up

* fix: preserve Android foreground container pruning
This commit is contained in:
Michał Pierzchała
2026-06-25 07:37:23 +02:00
committed by GitHub
parent d8e6bb7aa7
commit df490ee859
12 changed files with 1715 additions and 1716 deletions
-2
View File
@@ -5,7 +5,6 @@
"version": "1.0.0",
"orientation": "default",
"userInterfaceStyle": "automatic",
"newArchEnabled": true,
"plugins": ["expo-router"],
"ios": {
"supportsTablet": true,
@@ -13,7 +12,6 @@
},
"android": {
"package": "com.callstack.agentdevicelab",
"edgeToEdgeEnabled": true,
"predictiveBackGestureEnabled": false
}
}
+1 -1
View File
@@ -1,5 +1,5 @@
import { ThemeProvider } from '@react-navigation/native';
import { Stack } from 'expo-router';
import { ThemeProvider } from 'expo-router/react-navigation';
import { StatusBar } from 'expo-status-bar';
import { StyleSheet } from 'react-native';
import { GestureHandlerRootView } from 'react-native-gesture-handler';
+11 -12
View File
@@ -10,21 +10,20 @@
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@expo/metro-runtime": "~55.0.9",
"@react-navigation/native": "^7.2.2",
"expo": "~55.0.12",
"expo-constants": "55.0.12",
"expo-linking": "55.0.11",
"expo-router": "~55.0.11",
"expo-status-bar": "~55.0.5",
"react": "19.2.0",
"react-native": "0.83.4",
"@expo/metro-runtime": "~56.0.15",
"expo": "~56.0.12",
"expo-constants": "56.0.18",
"expo-linking": "56.0.14",
"expo-router": "~56.2.11",
"expo-status-bar": "~56.0.4",
"react": "19.2.3",
"react-native": "0.85.3",
"react-native-gesture-handler": "^2.31.2",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0"
"react-native-safe-area-context": "~5.7.0",
"react-native-screens": "~4.25.2"
},
"devDependencies": {
"@types/react": "~19.2.2",
"typescript": "~5.9.2"
"typescript": "~6.0.3"
}
}
+1119 -1681
View File
File diff suppressed because it is too large Load Diff
@@ -95,7 +95,7 @@ export function SettingsScreen(props: SettingsScreenProps) {
{props.diagnosticsExpanded ? (
<View style={styles.diagnosticsBody} testID="diagnostics-body">
<Text style={styles.diagnosticsText}>Build: expo-sdk-55 / lab-fixture-1</Text>
<Text style={styles.diagnosticsText}>Build: expo-sdk-56 / lab-fixture-1</Text>
<Text style={styles.diagnosticsText}>API mode: mock network with retry simulation</Text>
<Text style={styles.diagnosticsText}>
Device target hint: use this accordion for get-text and exists assertions
+1 -1
View File
@@ -1,4 +1,4 @@
import { DarkTheme, DefaultTheme, type Theme } from '@react-navigation/native';
import { DarkTheme, DefaultTheme, type Theme } from 'expo-router/react-navigation';
import { useColorScheme, type ColorSchemeName } from 'react-native';
export interface AppColors {
+1
View File
@@ -67,6 +67,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
'snapshot_capture',
async () =>
await snapshotAndroid(device, {
appBundleId: options?.appBundleId,
interactiveOnly: options?.interactiveOnly,
depth: options?.depth,
scope: options?.scope,
@@ -479,6 +479,34 @@ test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible ov
);
});
test('parseUiHierarchy keeps React Native content under a transparent Expo tools overlay', () => {
const xml = `<hierarchy>
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
<node class="android.widget.TextView" text="Agent Device Tester" bounds="[24,80][280,140]" enabled="true" visible-to-user="true" drawing-order="1"/>
<node class="android.widget.Button" text="Gesture lab" bounds="[24,180][280,240]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
</node>
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
<node class="android.widget.ImageView" content-desc="Tools" bounds="[320,80][360,120]" enabled="true" visible-to-user="true" drawing-order="1"/>
</node>
</node>
</hierarchy>`;
const result = parseUiHierarchy(xml, 800, { raw: true });
assert.equal(
result.nodes.some((node) => node.label === 'Agent Device Tester'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Gesture lab'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Tools'),
true,
);
});
test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
const xml =
"<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";
@@ -77,8 +77,15 @@ const installedHelperProbe = {
stderr: '',
};
function snapshotAndroidWithHelper(helperAdb: AndroidAdbExecutor) {
function snapshotAndroidWithHelper(
helperAdb: AndroidAdbExecutor,
options: Omit<
NonNullable<Parameters<typeof snapshotAndroid>[1]>,
'helperAdb' | 'helperArtifact'
> = {},
) {
return snapshotAndroid(device, {
...options,
helperAdb,
helperArtifact,
});
@@ -316,10 +323,11 @@ test('screenshotAndroid throws when PNG payload is truncated', async () => {
function helperOutput(
xml: string,
options: { truncated?: boolean; nodeCount?: number } = {},
options: { truncated?: boolean; nodeCount?: number; windowCount?: number } = {},
): string {
const truncated = options.truncated ?? false;
const nodeCount = options.nodeCount ?? 1;
const windowCount = options.windowCount ?? 1;
return [
'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1',
'INSTRUMENTATION_STATUS: helperApiVersion=1',
@@ -338,7 +346,7 @@ function helperOutput(
'INSTRUMENTATION_RESULT: maxNodes=5000',
'INSTRUMENTATION_RESULT: rootPresent=true',
'INSTRUMENTATION_RESULT: captureMode=interactive-windows',
'INSTRUMENTATION_RESULT: windowCount=1',
`INSTRUMENTATION_RESULT: windowCount=${windowCount}`,
`INSTRUMENTATION_RESULT: nodeCount=${nodeCount}`,
`INSTRUMENTATION_RESULT: truncated=${truncated}`,
'INSTRUMENTATION_RESULT: elapsedMs=12',
@@ -346,6 +354,61 @@ function helperOutput(
].join('\n');
}
function androidSystemWindowOnlyXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="3" window-layer="30" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node content-desc="Back" class="android.widget.ImageButton" package="com.android.systemui" bounds="[0,792][96,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' <node content-desc="Home" class="android.widget.ImageButton" package="com.android.systemui" bounds="[147,792][243,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}
function androidFabricAppXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node index="0" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true">',
' <node index="0" text="Fabric dashboard" resource-id="io.example.fabric:id/title" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][280,140]" enabled="true" />',
' <node index="1" text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}
function androidContentPoorFabricAppWindowXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
' </node>',
' <node window-index="1" window-type="3" window-layer="30" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}
function androidContentPoorExpoToolsOverlayXml(): string {
return [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node index="0" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node text="7:52" resource-id="com.android.systemui:id/clock" class="android.widget.TextView" package="com.android.systemui" bounds="[12,4][54,20]" enabled="true" visible-to-user="true" />',
' <node content-desc="Battery 100 percent." resource-id="com.android.systemui:id/battery" class="android.widget.LinearLayout" package="com.android.systemui" bounds="[340,4][380,20]" enabled="true" visible-to-user="true" />',
' </node>',
' <node index="1" class="android.widget.FrameLayout" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
' <node index="1" text="Agent Device Tester" class="android.widget.TextView" package="host.exp.exponent" bounds="[0,0][0,0]" enabled="true" visible-to-user="false" />',
' <node index="1" text="Tools" class="android.widget.ImageView" package="host.exp.exponent" bounds="[20,760][64,804]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
}
function mockScreenshotEvents(events: string[]): void {
mockRunCmd.mockImplementation(async (_cmd, args) => {
if (args.includes('exec-out')) {
@@ -730,6 +793,176 @@ test('snapshotAndroid falls back to stock uiautomator when helper fails', async
assert.equal(mockRunCmd.mock.calls.length, 0);
});
test('snapshotAndroid falls back to stock uiautomator when helper returns only system windows', async () => {
const adbCalls: string[][] = [];
const helperXml = androidSystemWindowOnlyXml();
const stockXml = androidFabricAppXml();
const helperAdb: AndroidAdbExecutor = async (args) => {
adbCalls.push(args);
if (args.includes('--show-versioncode')) return installedHelperProbe;
if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') {
return { exitCode: 0, stdout: '', stderr: '' };
}
if (args.includes('instrument')) {
return { exitCode: 0, stdout: helperOutput(helperXml, { nodeCount: 3 }), stderr: '' };
}
if (args.includes('exec-out')) {
return { exitCode: 0, stdout: stockXml, stderr: '' };
}
throw new Error(`unexpected helper adb args: ${args.join(' ')}`);
};
const result = await snapshotAndroidWithHelper(helperAdb);
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned only non-application windows',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
assert.equal(
adbCalls.some(
(args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper',
),
true,
);
assert.equal(
adbCalls.some((args) => args.includes('exec-out')),
true,
);
});
test('snapshotAndroid falls back to stock uiautomator when helper returns no nodes', async () => {
const helperXml = '<?xml version="1.0" encoding="UTF-8"?><hierarchy rotation="0"></hierarchy>';
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 0 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});
const result = await snapshotAndroidWithHelper(helperAdb);
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned no accessibility nodes',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
});
test('snapshotAndroid falls back to stock uiautomator when foreground app window lacks content', async () => {
const helperXml = androidContentPoorFabricAppWindowXml();
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});
const result = await snapshotAndroidWithHelper(helperAdb, {
appBundleId: 'io.example.fabric',
});
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned insufficient foreground app content',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
});
test('snapshotAndroid falls back to stock uiautomator when standalone helper sees only an app overlay', async () => {
const helperXml = androidContentPoorExpoToolsOverlayXml();
const stockXml = androidFabricAppXml();
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
stderr: '',
}),
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
});
const result = await snapshotAndroidWithHelper(helperAdb);
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
assert.equal(
result.androidSnapshot.fallbackReason,
'Android snapshot helper returned insufficient application window content',
);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(
result.nodes.some((node) => node.label === 'Open details'),
true,
);
});
test('snapshotAndroid keeps helper output when application and system windows are both present', async () => {
let stockAttempted = false;
const helperXml = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<hierarchy rotation="0">',
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
' <node text="Fabric dashboard" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][260,140]" enabled="true" visible-to-user="true" />',
' <node text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
' </node>',
' <node window-index="1" window-type="3" window-layer="20" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
' </node>',
'</hierarchy>',
].join('\n');
const helperAdb = createHelperAdb({
instrument: async () => ({
exitCode: 0,
stdout: helperOutput(helperXml, { nodeCount: 4 }),
stderr: '',
}),
stock: async () => {
stockAttempted = true;
throw new Error('stock fallback should not run');
},
});
const result = await snapshotAndroidWithHelper(helperAdb, {
appBundleId: 'io.example.fabric',
});
assert.equal(result.androidSnapshot.backend, 'android-helper');
assert.equal(result.androidSnapshot.fallbackReason, undefined);
assert.equal(
result.nodes.some((node) => node.label === 'Fabric dashboard'),
true,
);
assert.equal(stockAttempted, false);
});
test('snapshotAndroid emits fallback and stock capture diagnostics', async () => {
const stockXml =
'<?xml version="1.0" encoding="UTF-8"?><hierarchy><node text="stock" bounds="[0,0][10,10]" /></hierarchy>';
@@ -0,0 +1,256 @@
import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts';
import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts';
const ANDROID_WINDOW_TYPE_APPLICATION = 1;
const MAX_REPORTED_WINDOW_TYPES = 8;
const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2;
const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']);
export type AndroidHelperContentRecoveryDecision = {
reason: 'empty-helper-output' | 'system-window-only' | 'content-poor-app-window';
fallbackReason: string;
diagnostics: {
helperNodeCount: number;
helperWindowRootCount: number;
helperApplicationWindowRootCount: number;
helperMeaningfulNodeCount: number;
helperApplicationMeaningfulNodeCount: number;
helperNonSystemMeaningfulNodeCount: number;
helperForegroundAppMeaningfulNodeCount?: number;
helperForegroundAppPackage?: string;
helperForegroundAppMeaningfulNodeThreshold?: number;
helperWindowTypes: number[];
helperCaptureMode?: AndroidSnapshotBackendMetadata['captureMode'];
};
};
export function classifyAndroidHelperContentRecovery(
xml: string,
metadata: AndroidSnapshotBackendMetadata,
options: { foregroundAppPackage?: string } = {},
): AndroidHelperContentRecoveryDecision | undefined {
if (metadata.backend !== 'android-helper') return undefined;
const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage);
if (summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false) {
return buildRecoveryDecision(
summary,
metadata,
'empty-helper-output',
'Android snapshot helper returned no accessibility nodes',
);
}
const foregroundAppMeaningfulNodeCount = summary.foregroundAppMeaningfulNodeCount;
if (
foregroundAppMeaningfulNodeCount !== undefined &&
(foregroundAppMeaningfulNodeCount === 0 ||
(foregroundAppMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES &&
summary.meaningfulNodeCount > foregroundAppMeaningfulNodeCount))
) {
return buildRecoveryDecision(
summary,
metadata,
'content-poor-app-window',
'Android snapshot helper returned insufficient foreground app content',
);
}
if (
foregroundAppMeaningfulNodeCount === undefined &&
summary.applicationWindowRootCount > 0 &&
summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES
) {
return buildRecoveryDecision(
summary,
metadata,
'content-poor-app-window',
'Android snapshot helper returned insufficient application window content',
);
}
if (
foregroundAppMeaningfulNodeCount === undefined &&
summary.windowRootCount === 0 &&
(metadata.windowCount ?? 0) > 1 &&
summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES
) {
return buildRecoveryDecision(
summary,
metadata,
'content-poor-app-window',
'Android snapshot helper returned insufficient application window content',
);
}
if (summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0) {
return buildRecoveryDecision(
summary,
metadata,
'system-window-only',
'Android snapshot helper returned only non-application windows',
);
}
return undefined;
}
function buildRecoveryDecision(
summary: AndroidHelperXmlSummary,
metadata: AndroidSnapshotBackendMetadata,
reason: AndroidHelperContentRecoveryDecision['reason'],
fallbackReason: string,
): AndroidHelperContentRecoveryDecision {
return {
reason,
fallbackReason,
diagnostics: buildRecoveryDiagnostics(summary, metadata),
};
}
type AndroidHelperXmlSummary = {
nodeCount: number;
windowRootCount: number;
applicationWindowRootCount: number;
meaningfulNodeCount: number;
applicationMeaningfulNodeCount: number;
nonSystemMeaningfulNodeCount: number;
foregroundAppPackage?: string;
foregroundAppMeaningfulNodeCount?: number;
windowTypes: number[];
};
type AndroidHelperXmlSummaryState = Omit<AndroidHelperXmlSummary, 'windowTypes'> & {
currentWindowType?: number;
windowTypes: Set<number>;
};
function summarizeAndroidHelperXml(
xml: string,
foregroundAppPackage: string | undefined,
): AndroidHelperXmlSummary {
const summary = createAndroidHelperXmlSummaryState(foregroundAppPackage);
for (const node of androidUiNodes(xml)) {
recordAndroidHelperSummaryNode(summary, node);
}
return finalizeAndroidHelperXmlSummary(summary);
}
function createAndroidHelperXmlSummaryState(
foregroundAppPackage: string | undefined,
): AndroidHelperXmlSummaryState {
return {
nodeCount: 0,
windowRootCount: 0,
applicationWindowRootCount: 0,
meaningfulNodeCount: 0,
applicationMeaningfulNodeCount: 0,
nonSystemMeaningfulNodeCount: 0,
...(foregroundAppPackage !== undefined
? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 }
: {}),
windowTypes: new Set<number>(),
};
}
function recordAndroidHelperSummaryNode(
summary: AndroidHelperXmlSummaryState,
node: AndroidUiNodeMetadata,
): void {
summary.nodeCount += 1;
recordAndroidHelperWindowNode(summary, node);
recordAndroidHelperMeaningfulNode(summary, node);
}
function recordAndroidHelperWindowNode(
summary: AndroidHelperXmlSummaryState,
node: AndroidUiNodeMetadata,
): void {
if (node.windowType === undefined) return;
summary.currentWindowType = node.windowType;
summary.windowRootCount += 1;
summary.windowTypes.add(node.windowType);
if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) {
summary.applicationWindowRootCount += 1;
}
}
function recordAndroidHelperMeaningfulNode(
summary: AndroidHelperXmlSummaryState,
node: AndroidUiNodeMetadata,
): void {
if (!isMeaningfulContentNode(node)) return;
summary.meaningfulNodeCount += 1;
if (summary.currentWindowType === ANDROID_WINDOW_TYPE_APPLICATION) {
summary.applicationMeaningfulNodeCount += 1;
}
if (!isAndroidSystemPackage(node.packageName)) {
summary.nonSystemMeaningfulNodeCount += 1;
}
if (
summary.foregroundAppPackage !== undefined &&
node.packageName === summary.foregroundAppPackage
) {
summary.foregroundAppMeaningfulNodeCount = (summary.foregroundAppMeaningfulNodeCount ?? 0) + 1;
}
}
function finalizeAndroidHelperXmlSummary(
summary: AndroidHelperXmlSummaryState,
): AndroidHelperXmlSummary {
return {
nodeCount: summary.nodeCount,
windowRootCount: summary.windowRootCount,
applicationWindowRootCount: summary.applicationWindowRootCount,
meaningfulNodeCount: summary.meaningfulNodeCount,
applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount,
nonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount,
...(summary.foregroundAppPackage !== undefined
? {
foregroundAppPackage: summary.foregroundAppPackage,
foregroundAppMeaningfulNodeCount: summary.foregroundAppMeaningfulNodeCount ?? 0,
}
: {}),
windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES),
};
}
function isMeaningfulContentNode(node: AndroidUiNodeMetadata): boolean {
if (node.visibleToUser === false) return false;
return hasText(node.text) || hasText(node.desc) || hasText(node.resourceId);
}
function hasText(value: string | null): boolean {
return value !== null && value.trim() !== '';
}
function isAndroidSystemPackage(packageName: string | null): boolean {
return packageName === null || ANDROID_SYSTEM_PACKAGES.has(packageName);
}
function buildRecoveryDiagnostics(
summary: AndroidHelperXmlSummary,
metadata: AndroidSnapshotBackendMetadata,
): AndroidHelperContentRecoveryDecision['diagnostics'] {
return {
helperNodeCount: summary.nodeCount,
helperWindowRootCount: summary.windowRootCount,
helperApplicationWindowRootCount: summary.applicationWindowRootCount,
helperMeaningfulNodeCount: summary.meaningfulNodeCount,
helperApplicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount,
helperNonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount,
...(summary.foregroundAppPackage !== undefined
? {
helperForegroundAppPackage: summary.foregroundAppPackage,
helperForegroundAppMeaningfulNodeCount: summary.foregroundAppMeaningfulNodeCount,
helperForegroundAppMeaningfulNodeThreshold: MIN_FOREGROUND_APP_MEANINGFUL_NODES,
}
: {}),
helperWindowTypes: summary.windowTypes,
...(metadata.captureMode ? { helperCaptureMode: metadata.captureMode } : {}),
};
}
+43 -1
View File
@@ -44,6 +44,10 @@ import {
type AndroidSnapshotHelperOutput,
} from './snapshot-helper.ts';
import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts';
import {
classifyAndroidHelperContentRecovery,
type AndroidHelperContentRecoveryDecision,
} from './snapshot-content-recovery.ts';
const UI_HIERARCHY_DUMP_TIMEOUT_MS = 8_000;
const HELPER_INSTALL_TIMEOUT_MS = 30_000;
@@ -62,6 +66,7 @@ const RETRYABLE_ADB_STDERR_PATTERNS = [
] as const;
export type AndroidSnapshotOptions = SnapshotOptions & {
appBundleId?: string;
helperArtifact?: AndroidSnapshotHelperArtifact;
helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy;
helperSessionScope?: 'command' | 'daemon-session';
@@ -230,7 +235,20 @@ async function captureAndroidUiHierarchyWithHelper(
artifact,
helperDeviceKey,
});
return formatAndroidHelperCaptureResult(capture, artifact, install.reason);
const helperCapture = formatAndroidHelperCaptureResult(capture, artifact, install.reason);
const contentRecovery = classifyAndroidHelperContentRecovery(
helperCapture.xml,
helperCapture.metadata,
{ foregroundAppPackage: options.appBundleId },
);
if (!contentRecovery) return helperCapture;
return await recoverAndroidHelperContentUnavailable({
contentRecovery,
helperDeviceKey,
artifact,
device,
adb,
});
} catch (error) {
return await recoverAndroidHelperCaptureFailure({
error,
@@ -364,6 +382,30 @@ function formatAndroidHelperCaptureResult(
};
}
async function recoverAndroidHelperContentUnavailable(params: {
contentRecovery: AndroidHelperContentRecoveryDecision;
helperDeviceKey: string;
artifact: AndroidSnapshotHelperArtifact;
device: DeviceInfo;
adb: AndroidAdbExecutor;
}): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> {
emitDiagnostic({
level: 'warn',
phase: 'android_snapshot_helper_content_fallback',
data: {
reason: params.contentRecovery.reason,
fallbackReason: params.contentRecovery.fallbackReason,
...params.contentRecovery.diagnostics,
},
});
await resetAndroidSnapshotHelperRuntime(params.adb, params.artifact.manifest.packageName);
return await captureStockUiHierarchy(
params.device,
params.contentRecovery.fallbackReason,
params.adb,
);
}
async function recoverAndroidHelperCaptureFailure(params: {
error: unknown;
helperDeviceKey: string;
+18 -14
View File
@@ -460,7 +460,7 @@ type AndroidNodeInclusionInfo = {
};
type AndroidTreePruneState = {
agentVisibleContentMemo: WeakMap<AndroidNode, boolean>;
actionableContentMemo: WeakMap<AndroidNode, boolean>;
};
type AndroidCoveringCandidate = AndroidNode & {
@@ -528,7 +528,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
discardInactiveAndroidApplicationWindows(root);
// UiAutomation can expose covered React Native navigation surfaces in the same accessibility
// window. If a higher drawing-order sibling covers them, agents should see the foreground surface.
pruneAndroidCoveredSubtrees(root, { agentVisibleContentMemo: new WeakMap() });
pruneAndroidCoveredSubtrees(root, { actionableContentMemo: new WeakMap() });
applyAndroidScrollActionHints(root);
return root;
}
@@ -588,27 +588,31 @@ function canCoverSibling(
node.visibleToUser !== false &&
node.drawingOrder !== undefined &&
hasPositiveRect(node) &&
hasAgentVisibleContent(node, state)
(hasOwnAgentVisibleContent(node) || hasActionableDescendant(node, state))
);
}
function hasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
const cached = state.agentVisibleContentMemo.get(node);
if (cached !== undefined) return cached;
const result = computeHasAgentVisibleContent(node, state);
state.agentVisibleContentMemo.set(node, result);
return result;
}
function computeHasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
function hasOwnAgentVisibleContent(node: AndroidNode): boolean {
if (node.visibleToUser === false) return false;
if (node.hittable) return true;
const label = node.label?.trim() ?? '';
if (label && !isGenericAndroidId(label)) return true;
const identifier = node.identifier?.trim() ?? '';
if (identifier && !isGenericAndroidId(identifier)) return true;
return node.children.some((child) => hasAgentVisibleContent(child, state));
return false;
}
function hasActionableDescendant(node: AndroidNode, state: AndroidTreePruneState): boolean {
const cached = state.actionableContentMemo.get(node);
if (cached !== undefined) return cached;
const result = node.children.some(
(child) =>
child.visibleToUser !== false &&
(Boolean(child.hittable) || hasActionableDescendant(child, state)),
);
state.actionableContentMemo.set(node, result);
return result;
}
function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } {