mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
fix: recover Android snapshots from system-only helper output (#861)
* fix: recover Android snapshots from system-only helper output * fix: tighten Android snapshot recovery follow-up * fix: preserve Android foreground container pruning
This commit is contained in:
committed by
GitHub
parent
d8e6bb7aa7
commit
df490ee859
@@ -5,7 +5,6 @@
|
||||
"version": "1.0.0",
|
||||
"orientation": "default",
|
||||
"userInterfaceStyle": "automatic",
|
||||
"newArchEnabled": true,
|
||||
"plugins": ["expo-router"],
|
||||
"ios": {
|
||||
"supportsTablet": true,
|
||||
@@ -13,7 +12,6 @@
|
||||
},
|
||||
"android": {
|
||||
"package": "com.callstack.agentdevicelab",
|
||||
"edgeToEdgeEnabled": true,
|
||||
"predictiveBackGestureEnabled": false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ThemeProvider } from '@react-navigation/native';
|
||||
import { Stack } from 'expo-router';
|
||||
import { ThemeProvider } from 'expo-router/react-navigation';
|
||||
import { StatusBar } from 'expo-status-bar';
|
||||
import { StyleSheet } from 'react-native';
|
||||
import { GestureHandlerRootView } from 'react-native-gesture-handler';
|
||||
|
||||
@@ -10,21 +10,20 @@
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@expo/metro-runtime": "~55.0.9",
|
||||
"@react-navigation/native": "^7.2.2",
|
||||
"expo": "~55.0.12",
|
||||
"expo-constants": "55.0.12",
|
||||
"expo-linking": "55.0.11",
|
||||
"expo-router": "~55.0.11",
|
||||
"expo-status-bar": "~55.0.5",
|
||||
"react": "19.2.0",
|
||||
"react-native": "0.83.4",
|
||||
"@expo/metro-runtime": "~56.0.15",
|
||||
"expo": "~56.0.12",
|
||||
"expo-constants": "56.0.18",
|
||||
"expo-linking": "56.0.14",
|
||||
"expo-router": "~56.2.11",
|
||||
"expo-status-bar": "~56.0.4",
|
||||
"react": "19.2.3",
|
||||
"react-native": "0.85.3",
|
||||
"react-native-gesture-handler": "^2.31.2",
|
||||
"react-native-safe-area-context": "~5.6.2",
|
||||
"react-native-screens": "~4.23.0"
|
||||
"react-native-safe-area-context": "~5.7.0",
|
||||
"react-native-screens": "~4.25.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "~19.2.2",
|
||||
"typescript": "~5.9.2"
|
||||
"typescript": "~6.0.3"
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+1119
-1681
File diff suppressed because it is too large
Load Diff
@@ -95,7 +95,7 @@ export function SettingsScreen(props: SettingsScreenProps) {
|
||||
|
||||
{props.diagnosticsExpanded ? (
|
||||
<View style={styles.diagnosticsBody} testID="diagnostics-body">
|
||||
<Text style={styles.diagnosticsText}>Build: expo-sdk-55 / lab-fixture-1</Text>
|
||||
<Text style={styles.diagnosticsText}>Build: expo-sdk-56 / lab-fixture-1</Text>
|
||||
<Text style={styles.diagnosticsText}>API mode: mock network with retry simulation</Text>
|
||||
<Text style={styles.diagnosticsText}>
|
||||
Device target hint: use this accordion for get-text and exists assertions
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { DarkTheme, DefaultTheme, type Theme } from '@react-navigation/native';
|
||||
import { DarkTheme, DefaultTheme, type Theme } from 'expo-router/react-navigation';
|
||||
import { useColorScheme, type ColorSchemeName } from 'react-native';
|
||||
|
||||
export interface AppColors {
|
||||
|
||||
@@ -67,6 +67,7 @@ export function createAndroidInteractor(device: DeviceInfo): Interactor {
|
||||
'snapshot_capture',
|
||||
async () =>
|
||||
await snapshotAndroid(device, {
|
||||
appBundleId: options?.appBundleId,
|
||||
interactiveOnly: options?.interactiveOnly,
|
||||
depth: options?.depth,
|
||||
scope: options?.scope,
|
||||
|
||||
@@ -479,6 +479,34 @@ test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible ov
|
||||
);
|
||||
});
|
||||
|
||||
test('parseUiHierarchy keeps React Native content under a transparent Expo tools overlay', () => {
|
||||
const xml = `<hierarchy>
|
||||
<node class="android.widget.FrameLayout" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="0">
|
||||
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="1">
|
||||
<node class="android.widget.TextView" text="Agent Device Tester" bounds="[24,80][280,140]" enabled="true" visible-to-user="true" drawing-order="1"/>
|
||||
<node class="android.widget.Button" text="Gesture lab" bounds="[24,180][280,240]" clickable="true" enabled="true" visible-to-user="true" drawing-order="2"/>
|
||||
</node>
|
||||
<node class="android.view.ViewGroup" bounds="[0,0][390,844]" visible-to-user="true" drawing-order="2">
|
||||
<node class="android.widget.ImageView" content-desc="Tools" bounds="[320,80][360,120]" enabled="true" visible-to-user="true" drawing-order="1"/>
|
||||
</node>
|
||||
</node>
|
||||
</hierarchy>`;
|
||||
|
||||
const result = parseUiHierarchy(xml, 800, { raw: true });
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Agent Device Tester'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Gesture lab'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Tools'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('parseUiHierarchy ignores attribute-name prefix spoofing', () => {
|
||||
const xml =
|
||||
"<hierarchy><node class='android.widget.TextView' hint-text='Spoofed' text='Actual' bounds='[10,20][110,60]'/></hierarchy>";
|
||||
|
||||
@@ -77,8 +77,15 @@ const installedHelperProbe = {
|
||||
stderr: '',
|
||||
};
|
||||
|
||||
function snapshotAndroidWithHelper(helperAdb: AndroidAdbExecutor) {
|
||||
function snapshotAndroidWithHelper(
|
||||
helperAdb: AndroidAdbExecutor,
|
||||
options: Omit<
|
||||
NonNullable<Parameters<typeof snapshotAndroid>[1]>,
|
||||
'helperAdb' | 'helperArtifact'
|
||||
> = {},
|
||||
) {
|
||||
return snapshotAndroid(device, {
|
||||
...options,
|
||||
helperAdb,
|
||||
helperArtifact,
|
||||
});
|
||||
@@ -316,10 +323,11 @@ test('screenshotAndroid throws when PNG payload is truncated', async () => {
|
||||
|
||||
function helperOutput(
|
||||
xml: string,
|
||||
options: { truncated?: boolean; nodeCount?: number } = {},
|
||||
options: { truncated?: boolean; nodeCount?: number; windowCount?: number } = {},
|
||||
): string {
|
||||
const truncated = options.truncated ?? false;
|
||||
const nodeCount = options.nodeCount ?? 1;
|
||||
const windowCount = options.windowCount ?? 1;
|
||||
return [
|
||||
'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1',
|
||||
'INSTRUMENTATION_STATUS: helperApiVersion=1',
|
||||
@@ -338,7 +346,7 @@ function helperOutput(
|
||||
'INSTRUMENTATION_RESULT: maxNodes=5000',
|
||||
'INSTRUMENTATION_RESULT: rootPresent=true',
|
||||
'INSTRUMENTATION_RESULT: captureMode=interactive-windows',
|
||||
'INSTRUMENTATION_RESULT: windowCount=1',
|
||||
`INSTRUMENTATION_RESULT: windowCount=${windowCount}`,
|
||||
`INSTRUMENTATION_RESULT: nodeCount=${nodeCount}`,
|
||||
`INSTRUMENTATION_RESULT: truncated=${truncated}`,
|
||||
'INSTRUMENTATION_RESULT: elapsedMs=12',
|
||||
@@ -346,6 +354,61 @@ function helperOutput(
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function androidSystemWindowOnlyXml(): string {
|
||||
return [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<hierarchy rotation="0">',
|
||||
' <node window-index="0" window-type="3" window-layer="30" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
|
||||
' <node content-desc="Back" class="android.widget.ImageButton" package="com.android.systemui" bounds="[0,792][96,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
|
||||
' <node content-desc="Home" class="android.widget.ImageButton" package="com.android.systemui" bounds="[147,792][243,844]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
'</hierarchy>',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function androidFabricAppXml(): string {
|
||||
return [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<hierarchy rotation="0">',
|
||||
' <node index="0" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true">',
|
||||
' <node index="0" text="Fabric dashboard" resource-id="io.example.fabric:id/title" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][280,140]" enabled="true" />',
|
||||
' <node index="1" text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" />',
|
||||
' </node>',
|
||||
'</hierarchy>',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function androidContentPoorFabricAppWindowXml(): string {
|
||||
return [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<hierarchy rotation="0">',
|
||||
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
|
||||
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
' <node window-index="1" window-type="3" window-layer="30" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
|
||||
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
'</hierarchy>',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function androidContentPoorExpoToolsOverlayXml(): string {
|
||||
return [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<hierarchy rotation="0">',
|
||||
' <node index="0" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
|
||||
' <node text="7:52" resource-id="com.android.systemui:id/clock" class="android.widget.TextView" package="com.android.systemui" bounds="[12,4][54,20]" enabled="true" visible-to-user="true" />',
|
||||
' <node content-desc="Battery 100 percent." resource-id="com.android.systemui:id/battery" class="android.widget.LinearLayout" package="com.android.systemui" bounds="[340,4][380,20]" enabled="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
' <node index="1" class="android.widget.FrameLayout" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
|
||||
' <node index="0" class="androidx.compose.ui.platform.ComposeView" package="host.exp.exponent" bounds="[0,0][390,844]" enabled="true" visible-to-user="true" />',
|
||||
' <node index="1" text="Agent Device Tester" class="android.widget.TextView" package="host.exp.exponent" bounds="[0,0][0,0]" enabled="true" visible-to-user="false" />',
|
||||
' <node index="1" text="Tools" class="android.widget.ImageView" package="host.exp.exponent" bounds="[20,760][64,804]" enabled="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
'</hierarchy>',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function mockScreenshotEvents(events: string[]): void {
|
||||
mockRunCmd.mockImplementation(async (_cmd, args) => {
|
||||
if (args.includes('exec-out')) {
|
||||
@@ -730,6 +793,176 @@ test('snapshotAndroid falls back to stock uiautomator when helper fails', async
|
||||
assert.equal(mockRunCmd.mock.calls.length, 0);
|
||||
});
|
||||
|
||||
test('snapshotAndroid falls back to stock uiautomator when helper returns only system windows', async () => {
|
||||
const adbCalls: string[][] = [];
|
||||
const helperXml = androidSystemWindowOnlyXml();
|
||||
const stockXml = androidFabricAppXml();
|
||||
const helperAdb: AndroidAdbExecutor = async (args) => {
|
||||
adbCalls.push(args);
|
||||
if (args.includes('--show-versioncode')) return installedHelperProbe;
|
||||
if (args[0] === 'shell' && args[1] === 'am' && args[2] === 'force-stop') {
|
||||
return { exitCode: 0, stdout: '', stderr: '' };
|
||||
}
|
||||
if (args.includes('instrument')) {
|
||||
return { exitCode: 0, stdout: helperOutput(helperXml, { nodeCount: 3 }), stderr: '' };
|
||||
}
|
||||
if (args.includes('exec-out')) {
|
||||
return { exitCode: 0, stdout: stockXml, stderr: '' };
|
||||
}
|
||||
throw new Error(`unexpected helper adb args: ${args.join(' ')}`);
|
||||
};
|
||||
|
||||
const result = await snapshotAndroidWithHelper(helperAdb);
|
||||
|
||||
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
|
||||
assert.equal(
|
||||
result.androidSnapshot.fallbackReason,
|
||||
'Android snapshot helper returned only non-application windows',
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Fabric dashboard'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Open details'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
adbCalls.some(
|
||||
(args) => args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
adbCalls.some((args) => args.includes('exec-out')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('snapshotAndroid falls back to stock uiautomator when helper returns no nodes', async () => {
|
||||
const helperXml = '<?xml version="1.0" encoding="UTF-8"?><hierarchy rotation="0"></hierarchy>';
|
||||
const stockXml = androidFabricAppXml();
|
||||
const helperAdb = createHelperAdb({
|
||||
instrument: async () => ({
|
||||
exitCode: 0,
|
||||
stdout: helperOutput(helperXml, { nodeCount: 0 }),
|
||||
stderr: '',
|
||||
}),
|
||||
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
|
||||
});
|
||||
|
||||
const result = await snapshotAndroidWithHelper(helperAdb);
|
||||
|
||||
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
|
||||
assert.equal(
|
||||
result.androidSnapshot.fallbackReason,
|
||||
'Android snapshot helper returned no accessibility nodes',
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Fabric dashboard'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('snapshotAndroid falls back to stock uiautomator when foreground app window lacks content', async () => {
|
||||
const helperXml = androidContentPoorFabricAppWindowXml();
|
||||
const stockXml = androidFabricAppXml();
|
||||
const helperAdb = createHelperAdb({
|
||||
instrument: async () => ({
|
||||
exitCode: 0,
|
||||
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
|
||||
stderr: '',
|
||||
}),
|
||||
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
|
||||
});
|
||||
|
||||
const result = await snapshotAndroidWithHelper(helperAdb, {
|
||||
appBundleId: 'io.example.fabric',
|
||||
});
|
||||
|
||||
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
|
||||
assert.equal(
|
||||
result.androidSnapshot.fallbackReason,
|
||||
'Android snapshot helper returned insufficient foreground app content',
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Fabric dashboard'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Open details'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('snapshotAndroid falls back to stock uiautomator when standalone helper sees only an app overlay', async () => {
|
||||
const helperXml = androidContentPoorExpoToolsOverlayXml();
|
||||
const stockXml = androidFabricAppXml();
|
||||
const helperAdb = createHelperAdb({
|
||||
instrument: async () => ({
|
||||
exitCode: 0,
|
||||
stdout: helperOutput(helperXml, { nodeCount: 4, windowCount: 2 }),
|
||||
stderr: '',
|
||||
}),
|
||||
stock: async () => ({ exitCode: 0, stdout: stockXml, stderr: '' }),
|
||||
});
|
||||
|
||||
const result = await snapshotAndroidWithHelper(helperAdb);
|
||||
|
||||
assert.equal(result.androidSnapshot.backend, 'uiautomator-dump');
|
||||
assert.equal(
|
||||
result.androidSnapshot.fallbackReason,
|
||||
'Android snapshot helper returned insufficient application window content',
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Fabric dashboard'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Open details'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('snapshotAndroid keeps helper output when application and system windows are both present', async () => {
|
||||
let stockAttempted = false;
|
||||
const helperXml = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<hierarchy rotation="0">',
|
||||
' <node window-index="0" window-type="1" window-layer="10" window-active="true" window-focused="true" class="android.widget.FrameLayout" package="io.example.fabric" bounds="[0,0][390,844]" enabled="true" visible-to-user="true">',
|
||||
' <node text="Fabric dashboard" class="android.widget.TextView" package="io.example.fabric" bounds="[24,96][260,140]" enabled="true" visible-to-user="true" />',
|
||||
' <node text="Open details" class="android.widget.Button" package="io.example.fabric" bounds="[24,180][220,236]" clickable="true" enabled="true" focusable="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
' <node window-index="1" window-type="3" window-layer="20" window-active="false" window-focused="false" class="android.widget.FrameLayout" package="com.android.systemui" bounds="[0,0][390,24]" enabled="true" visible-to-user="true">',
|
||||
' <node content-desc="Battery" class="android.widget.ImageView" package="com.android.systemui" bounds="[340,4][370,20]" enabled="true" visible-to-user="true" />',
|
||||
' </node>',
|
||||
'</hierarchy>',
|
||||
].join('\n');
|
||||
const helperAdb = createHelperAdb({
|
||||
instrument: async () => ({
|
||||
exitCode: 0,
|
||||
stdout: helperOutput(helperXml, { nodeCount: 4 }),
|
||||
stderr: '',
|
||||
}),
|
||||
stock: async () => {
|
||||
stockAttempted = true;
|
||||
throw new Error('stock fallback should not run');
|
||||
},
|
||||
});
|
||||
|
||||
const result = await snapshotAndroidWithHelper(helperAdb, {
|
||||
appBundleId: 'io.example.fabric',
|
||||
});
|
||||
|
||||
assert.equal(result.androidSnapshot.backend, 'android-helper');
|
||||
assert.equal(result.androidSnapshot.fallbackReason, undefined);
|
||||
assert.equal(
|
||||
result.nodes.some((node) => node.label === 'Fabric dashboard'),
|
||||
true,
|
||||
);
|
||||
assert.equal(stockAttempted, false);
|
||||
});
|
||||
|
||||
test('snapshotAndroid emits fallback and stock capture diagnostics', async () => {
|
||||
const stockXml =
|
||||
'<?xml version="1.0" encoding="UTF-8"?><hierarchy><node text="stock" bounds="[0,0][10,10]" /></hierarchy>';
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts';
|
||||
import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts';
|
||||
|
||||
const ANDROID_WINDOW_TYPE_APPLICATION = 1;
|
||||
const MAX_REPORTED_WINDOW_TYPES = 8;
|
||||
const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2;
|
||||
const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']);
|
||||
|
||||
export type AndroidHelperContentRecoveryDecision = {
|
||||
reason: 'empty-helper-output' | 'system-window-only' | 'content-poor-app-window';
|
||||
fallbackReason: string;
|
||||
diagnostics: {
|
||||
helperNodeCount: number;
|
||||
helperWindowRootCount: number;
|
||||
helperApplicationWindowRootCount: number;
|
||||
helperMeaningfulNodeCount: number;
|
||||
helperApplicationMeaningfulNodeCount: number;
|
||||
helperNonSystemMeaningfulNodeCount: number;
|
||||
helperForegroundAppMeaningfulNodeCount?: number;
|
||||
helperForegroundAppPackage?: string;
|
||||
helperForegroundAppMeaningfulNodeThreshold?: number;
|
||||
helperWindowTypes: number[];
|
||||
helperCaptureMode?: AndroidSnapshotBackendMetadata['captureMode'];
|
||||
};
|
||||
};
|
||||
|
||||
export function classifyAndroidHelperContentRecovery(
|
||||
xml: string,
|
||||
metadata: AndroidSnapshotBackendMetadata,
|
||||
options: { foregroundAppPackage?: string } = {},
|
||||
): AndroidHelperContentRecoveryDecision | undefined {
|
||||
if (metadata.backend !== 'android-helper') return undefined;
|
||||
|
||||
const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage);
|
||||
if (summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false) {
|
||||
return buildRecoveryDecision(
|
||||
summary,
|
||||
metadata,
|
||||
'empty-helper-output',
|
||||
'Android snapshot helper returned no accessibility nodes',
|
||||
);
|
||||
}
|
||||
|
||||
const foregroundAppMeaningfulNodeCount = summary.foregroundAppMeaningfulNodeCount;
|
||||
if (
|
||||
foregroundAppMeaningfulNodeCount !== undefined &&
|
||||
(foregroundAppMeaningfulNodeCount === 0 ||
|
||||
(foregroundAppMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES &&
|
||||
summary.meaningfulNodeCount > foregroundAppMeaningfulNodeCount))
|
||||
) {
|
||||
return buildRecoveryDecision(
|
||||
summary,
|
||||
metadata,
|
||||
'content-poor-app-window',
|
||||
'Android snapshot helper returned insufficient foreground app content',
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
foregroundAppMeaningfulNodeCount === undefined &&
|
||||
summary.applicationWindowRootCount > 0 &&
|
||||
summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES
|
||||
) {
|
||||
return buildRecoveryDecision(
|
||||
summary,
|
||||
metadata,
|
||||
'content-poor-app-window',
|
||||
'Android snapshot helper returned insufficient application window content',
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
foregroundAppMeaningfulNodeCount === undefined &&
|
||||
summary.windowRootCount === 0 &&
|
||||
(metadata.windowCount ?? 0) > 1 &&
|
||||
summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES
|
||||
) {
|
||||
return buildRecoveryDecision(
|
||||
summary,
|
||||
metadata,
|
||||
'content-poor-app-window',
|
||||
'Android snapshot helper returned insufficient application window content',
|
||||
);
|
||||
}
|
||||
|
||||
if (summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0) {
|
||||
return buildRecoveryDecision(
|
||||
summary,
|
||||
metadata,
|
||||
'system-window-only',
|
||||
'Android snapshot helper returned only non-application windows',
|
||||
);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function buildRecoveryDecision(
|
||||
summary: AndroidHelperXmlSummary,
|
||||
metadata: AndroidSnapshotBackendMetadata,
|
||||
reason: AndroidHelperContentRecoveryDecision['reason'],
|
||||
fallbackReason: string,
|
||||
): AndroidHelperContentRecoveryDecision {
|
||||
return {
|
||||
reason,
|
||||
fallbackReason,
|
||||
diagnostics: buildRecoveryDiagnostics(summary, metadata),
|
||||
};
|
||||
}
|
||||
|
||||
type AndroidHelperXmlSummary = {
|
||||
nodeCount: number;
|
||||
windowRootCount: number;
|
||||
applicationWindowRootCount: number;
|
||||
meaningfulNodeCount: number;
|
||||
applicationMeaningfulNodeCount: number;
|
||||
nonSystemMeaningfulNodeCount: number;
|
||||
foregroundAppPackage?: string;
|
||||
foregroundAppMeaningfulNodeCount?: number;
|
||||
windowTypes: number[];
|
||||
};
|
||||
|
||||
type AndroidHelperXmlSummaryState = Omit<AndroidHelperXmlSummary, 'windowTypes'> & {
|
||||
currentWindowType?: number;
|
||||
windowTypes: Set<number>;
|
||||
};
|
||||
|
||||
function summarizeAndroidHelperXml(
|
||||
xml: string,
|
||||
foregroundAppPackage: string | undefined,
|
||||
): AndroidHelperXmlSummary {
|
||||
const summary = createAndroidHelperXmlSummaryState(foregroundAppPackage);
|
||||
|
||||
for (const node of androidUiNodes(xml)) {
|
||||
recordAndroidHelperSummaryNode(summary, node);
|
||||
}
|
||||
|
||||
return finalizeAndroidHelperXmlSummary(summary);
|
||||
}
|
||||
|
||||
function createAndroidHelperXmlSummaryState(
|
||||
foregroundAppPackage: string | undefined,
|
||||
): AndroidHelperXmlSummaryState {
|
||||
return {
|
||||
nodeCount: 0,
|
||||
windowRootCount: 0,
|
||||
applicationWindowRootCount: 0,
|
||||
meaningfulNodeCount: 0,
|
||||
applicationMeaningfulNodeCount: 0,
|
||||
nonSystemMeaningfulNodeCount: 0,
|
||||
...(foregroundAppPackage !== undefined
|
||||
? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 }
|
||||
: {}),
|
||||
windowTypes: new Set<number>(),
|
||||
};
|
||||
}
|
||||
|
||||
function recordAndroidHelperSummaryNode(
|
||||
summary: AndroidHelperXmlSummaryState,
|
||||
node: AndroidUiNodeMetadata,
|
||||
): void {
|
||||
summary.nodeCount += 1;
|
||||
recordAndroidHelperWindowNode(summary, node);
|
||||
recordAndroidHelperMeaningfulNode(summary, node);
|
||||
}
|
||||
|
||||
function recordAndroidHelperWindowNode(
|
||||
summary: AndroidHelperXmlSummaryState,
|
||||
node: AndroidUiNodeMetadata,
|
||||
): void {
|
||||
if (node.windowType === undefined) return;
|
||||
|
||||
summary.currentWindowType = node.windowType;
|
||||
summary.windowRootCount += 1;
|
||||
summary.windowTypes.add(node.windowType);
|
||||
if (node.windowType === ANDROID_WINDOW_TYPE_APPLICATION) {
|
||||
summary.applicationWindowRootCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function recordAndroidHelperMeaningfulNode(
|
||||
summary: AndroidHelperXmlSummaryState,
|
||||
node: AndroidUiNodeMetadata,
|
||||
): void {
|
||||
if (!isMeaningfulContentNode(node)) return;
|
||||
|
||||
summary.meaningfulNodeCount += 1;
|
||||
if (summary.currentWindowType === ANDROID_WINDOW_TYPE_APPLICATION) {
|
||||
summary.applicationMeaningfulNodeCount += 1;
|
||||
}
|
||||
if (!isAndroidSystemPackage(node.packageName)) {
|
||||
summary.nonSystemMeaningfulNodeCount += 1;
|
||||
}
|
||||
if (
|
||||
summary.foregroundAppPackage !== undefined &&
|
||||
node.packageName === summary.foregroundAppPackage
|
||||
) {
|
||||
summary.foregroundAppMeaningfulNodeCount = (summary.foregroundAppMeaningfulNodeCount ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
function finalizeAndroidHelperXmlSummary(
|
||||
summary: AndroidHelperXmlSummaryState,
|
||||
): AndroidHelperXmlSummary {
|
||||
return {
|
||||
nodeCount: summary.nodeCount,
|
||||
windowRootCount: summary.windowRootCount,
|
||||
applicationWindowRootCount: summary.applicationWindowRootCount,
|
||||
meaningfulNodeCount: summary.meaningfulNodeCount,
|
||||
applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount,
|
||||
nonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount,
|
||||
...(summary.foregroundAppPackage !== undefined
|
||||
? {
|
||||
foregroundAppPackage: summary.foregroundAppPackage,
|
||||
foregroundAppMeaningfulNodeCount: summary.foregroundAppMeaningfulNodeCount ?? 0,
|
||||
}
|
||||
: {}),
|
||||
windowTypes: [...summary.windowTypes].sort((a, b) => a - b).slice(0, MAX_REPORTED_WINDOW_TYPES),
|
||||
};
|
||||
}
|
||||
|
||||
function isMeaningfulContentNode(node: AndroidUiNodeMetadata): boolean {
|
||||
if (node.visibleToUser === false) return false;
|
||||
return hasText(node.text) || hasText(node.desc) || hasText(node.resourceId);
|
||||
}
|
||||
|
||||
function hasText(value: string | null): boolean {
|
||||
return value !== null && value.trim() !== '';
|
||||
}
|
||||
|
||||
function isAndroidSystemPackage(packageName: string | null): boolean {
|
||||
return packageName === null || ANDROID_SYSTEM_PACKAGES.has(packageName);
|
||||
}
|
||||
|
||||
function buildRecoveryDiagnostics(
|
||||
summary: AndroidHelperXmlSummary,
|
||||
metadata: AndroidSnapshotBackendMetadata,
|
||||
): AndroidHelperContentRecoveryDecision['diagnostics'] {
|
||||
return {
|
||||
helperNodeCount: summary.nodeCount,
|
||||
helperWindowRootCount: summary.windowRootCount,
|
||||
helperApplicationWindowRootCount: summary.applicationWindowRootCount,
|
||||
helperMeaningfulNodeCount: summary.meaningfulNodeCount,
|
||||
helperApplicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount,
|
||||
helperNonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount,
|
||||
...(summary.foregroundAppPackage !== undefined
|
||||
? {
|
||||
helperForegroundAppPackage: summary.foregroundAppPackage,
|
||||
helperForegroundAppMeaningfulNodeCount: summary.foregroundAppMeaningfulNodeCount,
|
||||
helperForegroundAppMeaningfulNodeThreshold: MIN_FOREGROUND_APP_MEANINGFUL_NODES,
|
||||
}
|
||||
: {}),
|
||||
helperWindowTypes: summary.windowTypes,
|
||||
...(metadata.captureMode ? { helperCaptureMode: metadata.captureMode } : {}),
|
||||
};
|
||||
}
|
||||
@@ -44,6 +44,10 @@ import {
|
||||
type AndroidSnapshotHelperOutput,
|
||||
} from './snapshot-helper.ts';
|
||||
import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts';
|
||||
import {
|
||||
classifyAndroidHelperContentRecovery,
|
||||
type AndroidHelperContentRecoveryDecision,
|
||||
} from './snapshot-content-recovery.ts';
|
||||
|
||||
const UI_HIERARCHY_DUMP_TIMEOUT_MS = 8_000;
|
||||
const HELPER_INSTALL_TIMEOUT_MS = 30_000;
|
||||
@@ -62,6 +66,7 @@ const RETRYABLE_ADB_STDERR_PATTERNS = [
|
||||
] as const;
|
||||
|
||||
export type AndroidSnapshotOptions = SnapshotOptions & {
|
||||
appBundleId?: string;
|
||||
helperArtifact?: AndroidSnapshotHelperArtifact;
|
||||
helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy;
|
||||
helperSessionScope?: 'command' | 'daemon-session';
|
||||
@@ -230,7 +235,20 @@ async function captureAndroidUiHierarchyWithHelper(
|
||||
artifact,
|
||||
helperDeviceKey,
|
||||
});
|
||||
return formatAndroidHelperCaptureResult(capture, artifact, install.reason);
|
||||
const helperCapture = formatAndroidHelperCaptureResult(capture, artifact, install.reason);
|
||||
const contentRecovery = classifyAndroidHelperContentRecovery(
|
||||
helperCapture.xml,
|
||||
helperCapture.metadata,
|
||||
{ foregroundAppPackage: options.appBundleId },
|
||||
);
|
||||
if (!contentRecovery) return helperCapture;
|
||||
return await recoverAndroidHelperContentUnavailable({
|
||||
contentRecovery,
|
||||
helperDeviceKey,
|
||||
artifact,
|
||||
device,
|
||||
adb,
|
||||
});
|
||||
} catch (error) {
|
||||
return await recoverAndroidHelperCaptureFailure({
|
||||
error,
|
||||
@@ -364,6 +382,30 @@ function formatAndroidHelperCaptureResult(
|
||||
};
|
||||
}
|
||||
|
||||
async function recoverAndroidHelperContentUnavailable(params: {
|
||||
contentRecovery: AndroidHelperContentRecoveryDecision;
|
||||
helperDeviceKey: string;
|
||||
artifact: AndroidSnapshotHelperArtifact;
|
||||
device: DeviceInfo;
|
||||
adb: AndroidAdbExecutor;
|
||||
}): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> {
|
||||
emitDiagnostic({
|
||||
level: 'warn',
|
||||
phase: 'android_snapshot_helper_content_fallback',
|
||||
data: {
|
||||
reason: params.contentRecovery.reason,
|
||||
fallbackReason: params.contentRecovery.fallbackReason,
|
||||
...params.contentRecovery.diagnostics,
|
||||
},
|
||||
});
|
||||
await resetAndroidSnapshotHelperRuntime(params.adb, params.artifact.manifest.packageName);
|
||||
return await captureStockUiHierarchy(
|
||||
params.device,
|
||||
params.contentRecovery.fallbackReason,
|
||||
params.adb,
|
||||
);
|
||||
}
|
||||
|
||||
async function recoverAndroidHelperCaptureFailure(params: {
|
||||
error: unknown;
|
||||
helperDeviceKey: string;
|
||||
|
||||
@@ -460,7 +460,7 @@ type AndroidNodeInclusionInfo = {
|
||||
};
|
||||
|
||||
type AndroidTreePruneState = {
|
||||
agentVisibleContentMemo: WeakMap<AndroidNode, boolean>;
|
||||
actionableContentMemo: WeakMap<AndroidNode, boolean>;
|
||||
};
|
||||
|
||||
type AndroidCoveringCandidate = AndroidNode & {
|
||||
@@ -528,7 +528,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy {
|
||||
discardInactiveAndroidApplicationWindows(root);
|
||||
// UiAutomation can expose covered React Native navigation surfaces in the same accessibility
|
||||
// window. If a higher drawing-order sibling covers them, agents should see the foreground surface.
|
||||
pruneAndroidCoveredSubtrees(root, { agentVisibleContentMemo: new WeakMap() });
|
||||
pruneAndroidCoveredSubtrees(root, { actionableContentMemo: new WeakMap() });
|
||||
applyAndroidScrollActionHints(root);
|
||||
return root;
|
||||
}
|
||||
@@ -588,27 +588,31 @@ function canCoverSibling(
|
||||
node.visibleToUser !== false &&
|
||||
node.drawingOrder !== undefined &&
|
||||
hasPositiveRect(node) &&
|
||||
hasAgentVisibleContent(node, state)
|
||||
(hasOwnAgentVisibleContent(node) || hasActionableDescendant(node, state))
|
||||
);
|
||||
}
|
||||
|
||||
function hasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
|
||||
const cached = state.agentVisibleContentMemo.get(node);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
const result = computeHasAgentVisibleContent(node, state);
|
||||
state.agentVisibleContentMemo.set(node, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
function computeHasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean {
|
||||
function hasOwnAgentVisibleContent(node: AndroidNode): boolean {
|
||||
if (node.visibleToUser === false) return false;
|
||||
if (node.hittable) return true;
|
||||
const label = node.label?.trim() ?? '';
|
||||
if (label && !isGenericAndroidId(label)) return true;
|
||||
const identifier = node.identifier?.trim() ?? '';
|
||||
if (identifier && !isGenericAndroidId(identifier)) return true;
|
||||
return node.children.some((child) => hasAgentVisibleContent(child, state));
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasActionableDescendant(node: AndroidNode, state: AndroidTreePruneState): boolean {
|
||||
const cached = state.actionableContentMemo.get(node);
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
const result = node.children.some(
|
||||
(child) =>
|
||||
child.visibleToUser !== false &&
|
||||
(Boolean(child.hittable) || hasActionableDescendant(child, state)),
|
||||
);
|
||||
state.actionableContentMemo.set(node, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } {
|
||||
|
||||
Reference in New Issue
Block a user