From f71371ebb8976af0915fac9ae0934c2bae65714c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 20 May 2026 14:51:17 +0200 Subject: [PATCH] fix: handle platform alerts (#562) * fix: handle Android platform alerts * fix: handle iOS alerts in runner * chore: type alert metadata * refactor: colocate Android alert handling --- .../RunnerTests+Alert.swift | 155 +++++++++ .../RunnerTests+CommandExecution.swift | 28 +- .../RunnerTests+SystemModal.swift | 6 +- src/alert-contract.ts | 18 ++ src/backend.ts | 9 +- src/client-types.ts | 16 +- src/core/__tests__/capabilities.test.ts | 11 +- src/core/capabilities.ts | 4 +- src/daemon/handlers/parse-utils.ts | 7 +- src/daemon/handlers/snapshot-alert.ts | 203 ++++++------ src/index.ts | 4 + .../android/__tests__/alert-detection.test.ts | 133 ++++++++ .../android/__tests__/snapshot.test.ts | 55 ++++ src/platforms/android/alert-detection.ts | 271 ++++++++++++++++ src/platforms/android/alert.ts | 161 ++++++++++ src/platforms/android/snapshot.ts | 33 +- src/platforms/android/ui-hierarchy.ts | 4 + src/utils/__tests__/args.test.ts | 6 +- src/utils/command-schema.ts | 13 +- .../android-lifecycle.test.ts | 294 ++++++++++++++++++ .../provider-scenarios/android-world.ts | 66 +++- .../suites/agent-device-smoke-suite.ts | 11 +- website/docs/docs/commands.md | 7 +- website/docs/docs/debugging-profiling.md | 2 +- 24 files changed, 1330 insertions(+), 187 deletions(-) create mode 100644 ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift create mode 100644 src/alert-contract.ts create mode 100644 src/platforms/android/__tests__/alert-detection.test.ts create mode 100644 src/platforms/android/alert-detection.ts create mode 100644 src/platforms/android/alert.ts diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift new file mode 100644 index 000000000..5681aba81 --- /dev/null +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -0,0 +1,155 @@ +import XCTest + +extension RunnerTests { + struct RunnerAlert { + let root: XCUIElement + let ownerApp: XCUIApplication + let buttons: [XCUIElement] + } + + func resolveAlert(app activeApp: XCUIApplication) -> RunnerAlert? { + if let alert = firstExistingElement(in: activeApp.alerts.allElementsBoundByIndex) { + return runnerAlert(root: alert, ownerApp: activeApp) + } + if let popup = firstDismissPopupWindow(in: activeApp) { + return runnerAlert(root: popup, ownerApp: activeApp) + } +#if os(macOS) + return nil +#else + if let systemModal = firstBlockingSystemModal(in: springboard) { + return runnerAlert(root: systemModal, ownerApp: springboard) + } + return nil +#endif + } + + func handleAlert(_ alert: RunnerAlert, action: String) -> Response { + if action == "accept" || action == "dismiss" { + guard let button = chooseAlertButton(alert.buttons, action: action) else { + return Response(ok: false, error: ErrorPayload(message: "alert \(action) button not found")) + } + let outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)") + if let response = unsupportedResponse(for: outcome) { + return response + } + return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed")) + } + + return Response( + ok: true, + data: DataPayload( + message: preferredAlertTitle(alert.root, buttons: alert.buttons), + items: alert.buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines) } + ) + ) + } + + private func runnerAlert(root: XCUIElement, ownerApp: XCUIApplication) -> RunnerAlert? { + let buttons = actionableElements(in: root).filter { isEnabledElement($0) } + guard !buttons.isEmpty else { + return nil + } + return RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons) + } + + private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? { + elements.first { isVisibleElement($0) } + } + + private func firstDismissPopupWindow(in app: XCUIApplication) -> XCUIElement? { + safeElementsQuery { + app.windows.allElementsBoundByIndex + }.first { window in + if !isVisibleElement(window) { return false } + if isDismissPopupMarker(window.label) || isDismissPopupMarker(window.identifier) { + return true + } + return safeElementsQuery { + window.descendants(matching: .any).allElementsBoundByIndex + }.contains { descendant in + isDismissPopupMarker(descendant.label) || isDismissPopupMarker(descendant.identifier) + } + } + } + + private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? { + if action == "accept" { + if let accept = buttons.first(where: { isAcceptButton($0.label) }) { + return accept + } + return buttons.count == 1 && !isDismissButton(buttons[0].label) ? buttons[0] : nil + } + + return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last + } + + private func isAcceptButton(_ label: String) -> Bool { + let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return [ + "ok", + "allow", + "yes", + "continue", + "done", + "open settings" + ].contains(normalized) || normalized.hasPrefix("confirm") + } + + private func isDismissButton(_ label: String) -> Bool { + [ + "cancel", + "close", + "dismiss", + "don't allow", + "don’t allow", + "not now", + "no", + "keep browsing", + "later" + ].contains(label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()) + } + + private func preferredAlertTitle(_ element: XCUIElement, buttons: [XCUIElement]) -> String { + let buttonLabels = Set(buttons.map { $0.label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }) + let descendants = element.descendants(matching: .any).allElementsBoundByIndex + for descendant in descendants { + let text = descendant.label.trimmingCharacters(in: .whitespacesAndNewlines) + if text.isEmpty || + isGenericAlertLabel(text) || + buttonLabels.contains(text.lowercased()) || + descendant.elementType == .navigationBar || + actionableTypes.contains(descendant.elementType) + { + continue + } + return text + } + let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + return label.isEmpty || isGenericAlertLabel(label) ? "Alert" : label + } + + private func isGenericAlertLabel(_ label: String) -> Bool { + let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return isDismissPopupMarker(normalized) || + normalized.hasPrefix("vertical scroll bar") || + normalized.hasPrefix("horizontal scroll bar") || + normalized == "tab bar" + } + + private func isVisibleElement(_ element: XCUIElement) -> Bool { + element.exists && !element.frame.isNull && !element.frame.isEmpty + } + + private func isEnabledElement(_ element: XCUIElement) -> Bool { + var enabled = false + _ = RunnerObjCExceptionCatcher.catchException({ + enabled = element.exists && element.isEnabled + }) + return enabled + } + + private func isDismissPopupMarker(_ label: String) -> Bool { + label.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame + } +} diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 3a38f7017..bc92c4317 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -13,7 +13,7 @@ extension RunnerTests { return (gestureStartUptimeMs, currentUptimeMs()) } - private func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? { + func unsupportedResponse(for outcome: RunnerInteractionOutcome) -> Response? { switch outcome { case .performed: return nil @@ -731,32 +731,10 @@ extension RunnerTests { ) case .alert: let action = (command.action ?? "get").lowercased() - let alert = activeApp.alerts.firstMatch - if !alert.exists { + guard let alert = resolveAlert(app: activeApp) else { return Response(ok: false, error: ErrorPayload(message: "alert not found")) } - if action == "accept" { - guard let button = alert.buttons.allElementsBoundByIndex.first else { - return Response(ok: false, error: ErrorPayload(message: "alert accept button not found")) - } - let outcome = activateElement(app: activeApp, element: button, action: "alert accept") - if let response = unsupportedResponse(for: outcome) { - return response - } - return Response(ok: true, data: DataPayload(message: "accepted")) - } - if action == "dismiss" { - guard let button = alert.buttons.allElementsBoundByIndex.last else { - return Response(ok: false, error: ErrorPayload(message: "alert dismiss button not found")) - } - let outcome = activateElement(app: activeApp, element: button, action: "alert dismiss") - if let response = unsupportedResponse(for: outcome) { - return response - } - return Response(ok: true, data: DataPayload(message: "dismissed")) - } - let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label } - return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels)) + return handleAlert(alert, action: action) case .pinch: guard let scale = command.scale, scale > 0 else { return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0")) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift index 2a88a6bde..d78e33b1f 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift @@ -46,7 +46,7 @@ extension RunnerTests { #endif } - private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? { + func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? { let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE") let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in if disableSafeProbe { @@ -76,7 +76,7 @@ extension RunnerTests { return nil } - private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] { + func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] { var elements: [XCUIElement] = [] let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ elements = fetch() @@ -120,7 +120,7 @@ extension RunnerTests { return true } - private func actionableElements(in element: XCUIElement) -> [XCUIElement] { + func actionableElements(in element: XCUIElement) -> [XCUIElement] { var seen = Set() var actions: [XCUIElement] = [] let descendants = safeElementsQuery { diff --git a/src/alert-contract.ts b/src/alert-contract.ts new file mode 100644 index 000000000..3e30e7ca4 --- /dev/null +++ b/src/alert-contract.ts @@ -0,0 +1,18 @@ +export const ALERT_POLL_INTERVAL_MS = 300; +export const DEFAULT_ALERT_TIMEOUT_MS = 10_000; +export const ALERT_ACTION_RETRY_MS = 2_000; + +export type AlertAction = 'get' | 'accept' | 'dismiss' | 'wait'; + +export type AlertPlatform = 'android' | 'ios' | 'macos'; + +export type AlertSource = 'permission' | 'native-dialog' | 'system-dialog'; + +export type AlertInfo = { + title?: string; + message?: string; + buttons?: string[]; + platform?: AlertPlatform; + source?: AlertSource; + packageName?: string; +}; diff --git a/src/backend.ts b/src/backend.ts index 267e8434a..eae5c551f 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,4 +1,5 @@ import type { AndroidSnapshotBackendMetadata } from './platforms/android/snapshot-types.ts'; +import type { AlertAction, AlertInfo } from './alert-contract.ts'; import type { AppsFilter } from './commands/app-inventory-contract.ts'; import type { Point, @@ -110,13 +111,9 @@ export type BackendClipboardTextResult = { text: string; }; -export type BackendAlertAction = 'get' | 'accept' | 'dismiss' | 'wait'; +export type BackendAlertAction = AlertAction; -export type BackendAlertInfo = { - title?: string; - message?: string; - buttons?: string[]; -}; +export type BackendAlertInfo = AlertInfo; export type BackendAlertResult = | { diff --git a/src/client-types.ts b/src/client-types.ts index 4cc91ba52..2a96ee513 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -24,10 +24,12 @@ import type { import type { MetroBridgeScope } from './client-companion-tunnel-contract.ts'; import type { AppsFilter } from './commands/app-inventory-contract.ts'; import type { ScreenshotRequestFlags } from './commands/capture-screenshot-options.ts'; +import type { AlertInfo } from './alert-contract.ts'; export type { FindLocator } from './utils/finders.ts'; export type { CompanionTunnelScope, MetroBridgeScope } from './client-companion-tunnel-contract.ts'; export type { AppsFilter } from './commands/app-inventory-contract.ts'; +export type { AlertAction, AlertInfo, AlertPlatform, AlertSource } from './alert-contract.ts'; type DaemonTransportMode = 'auto' | 'socket' | 'http'; type DaemonServerMode = 'socket' | 'http' | 'dual'; @@ -404,7 +406,19 @@ export type WaitCommandResult = DaemonResponseData & { selector?: string; }; -export type AlertCommandResult = DaemonResponseData; +export type AlertCommandResult = DaemonResponseData & { + kind?: 'alertStatus' | 'alertHandled' | 'alertWait'; + action?: AlertCommandOptions['action']; + alert?: AlertInfo | null; + handled?: boolean; + button?: string; + waitedMs?: number; + timedOut?: boolean; + platform?: AlertInfo['platform']; + accepted?: boolean; + dismissed?: boolean; + items?: string[]; +}; type CommandActionResult = DaemonResponseData & { action?: T; diff --git a/src/core/__tests__/capabilities.test.ts b/src/core/__tests__/capabilities.test.ts index 3003f1710..bbe6bcffa 100644 --- a/src/core/__tests__/capabilities.test.ts +++ b/src/core/__tests__/capabilities.test.ts @@ -77,7 +77,7 @@ function assertCommandSupport(commands: string[], checks: SupportCheck[]): void test('device capability matrix stays consistent across shared command groups', () => { const scenarios: Array<{ commands: string[]; checks: SupportCheck[] }> = [ { - commands: ['alert', 'pinch'], + commands: ['pinch'], checks: [ { device: iosSimulator, expected: true, label: 'on iOS sim' }, { device: iosDevice, expected: false, label: 'on iOS device' }, @@ -85,6 +85,15 @@ test('device capability matrix stays consistent across shared command groups', ( { device: macOsDevice, expected: true, label: 'on macOS' }, ], }, + { + commands: ['alert'], + checks: [ + { device: iosSimulator, expected: true, label: 'on iOS sim' }, + { device: iosDevice, expected: false, label: 'on iOS device' }, + { device: androidDevice, expected: true, label: 'on Android' }, + { device: macOsDevice, expected: true, label: 'on macOS' }, + ], + }, { commands: ['settings', 'clipboard'], checks: [ diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index 3bd022661..2fd8a974b 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -35,9 +35,9 @@ const COMMAND_CAPABILITY_MATRIX: Record = { // macOS desktop targets report kind=device, so this stays enabled here and the // supports() guard excludes iOS physical devices. apple: { simulator: true, device: true }, - android: {}, + android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, - supports: isMacOsOrAppleSimulator, + supports: (device) => device.platform === 'android' || isMacOsOrAppleSimulator(device), }, pinch: { // macOS desktop targets report kind=device, so this stays enabled here and the diff --git a/src/daemon/handlers/parse-utils.ts b/src/daemon/handlers/parse-utils.ts index 824a98146..83b1e22c3 100644 --- a/src/daemon/handlers/parse-utils.ts +++ b/src/daemon/handlers/parse-utils.ts @@ -1,5 +1,8 @@ -export const POLL_INTERVAL_MS = 300; -export const DEFAULT_TIMEOUT_MS = 10_000; +export { + ALERT_ACTION_RETRY_MS, + ALERT_POLL_INTERVAL_MS as POLL_INTERVAL_MS, + DEFAULT_ALERT_TIMEOUT_MS as DEFAULT_TIMEOUT_MS, +} from '../../alert-contract.ts'; export function parseTimeout(value: string | undefined): number | null { if (!value) return null; diff --git a/src/daemon/handlers/snapshot-alert.ts b/src/daemon/handlers/snapshot-alert.ts index 943ef5f2a..e12cb1fb9 100644 --- a/src/daemon/handlers/snapshot-alert.ts +++ b/src/daemon/handlers/snapshot-alert.ts @@ -1,12 +1,19 @@ import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import type { AlertAction } from '../../alert-contract.ts'; import { sleep } from '../../utils/timeouts.ts'; import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; import { runMacOsAlertAction } from '../../platforms/ios/macos-helper.ts'; +import { handleAndroidAlert } from '../../platforms/android/alert.ts'; import { AppError } from '../../utils/errors.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { recordIfSession } from './snapshot-session.ts'; -import { DEFAULT_TIMEOUT_MS, parseTimeout, POLL_INTERVAL_MS } from './parse-utils.ts'; +import { + ALERT_ACTION_RETRY_MS, + DEFAULT_TIMEOUT_MS, + parseTimeout, + POLL_INTERVAL_MS, +} from './parse-utils.ts'; import { errorResponse } from './response.ts'; type HandleAlertCommandParams = { @@ -17,14 +24,17 @@ type HandleAlertCommandParams = { device: SessionState['device']; }; +type NativeAlertAction = Exclude; +type NativeAlertRunner = (action: NativeAlertAction) => Promise; + const ALERT_FALLBACK_HINT = 'If the permission sheet is visible in snapshot or screenshot but alert reports no alert, take a scoped snapshot around the visible button label and use press @ref.'; export async function handleAlertCommand( params: HandleAlertCommandParams, ): Promise { - const { req, logPath, sessionStore, session, device } = params; - const action = (req.positionals?.[0] ?? 'get').toLowerCase(); + const { req, logPath, session, device } = params; + const action = normalizeAlertAction(req.positionals?.[0]); const macOsAlertTarget = (() => { if (!session) return {}; if (session.surface === 'frontmost-app') { @@ -38,122 +48,106 @@ export async function handleAlertCommand( if (!isCommandSupportedOnDevice('alert', device)) { return errorResponse('UNSUPPORTED_OPERATION', 'alert is not supported on this device'); } - if (device.platform === 'macos') { - const runMacOsAlert = async () => - await runMacOsAlertAction( - action === 'wait' ? 'get' : (action as 'get' | 'accept' | 'dismiss'), - macOsAlertTarget, - ); - if (action === 'wait') { - const timeout = parseTimeout(req.positionals?.[1]) ?? DEFAULT_TIMEOUT_MS; - const start = Date.now(); - while (Date.now() - start < timeout) { - try { - const data = await runMacOsAlert(); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; - } catch { - // keep waiting - } - await sleep(POLL_INTERVAL_MS); - } - return errorResponse('COMMAND_FAILED', 'alert wait timed out'); - } - const resolvedAction = action === 'accept' || action === 'dismiss' ? action : 'get'; - if (resolvedAction === 'accept' || resolvedAction === 'dismiss') { - const ALERT_ACTION_RETRY_MS = 2_000; - const start = Date.now(); - let lastError: unknown; - while (Date.now() - start < ALERT_ACTION_RETRY_MS) { - try { - const data = await runMacOsAlertAction(resolvedAction, macOsAlertTarget); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; - } catch (err) { - lastError = err; - const msg = String((err as { message?: unknown })?.message ?? '').toLowerCase(); - if (!msg.includes('alert not found') && !msg.includes('no alert')) break; - } - await sleep(POLL_INTERVAL_MS); - } - throw withAlertFallbackHint(lastError); - } - const data = await runMacOsAlertAction('get', macOsAlertTarget); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; + if (device.platform === 'android') { + const timeoutMs = parseTimeout(req.positionals?.[1]) ?? DEFAULT_TIMEOUT_MS; + return recordAlertResponse( + params, + await handleAndroidAlert(device, action, { + timeoutMs, + }), + ); } - if (action === 'wait') { - const timeout = parseTimeout(req.positionals?.[1]) ?? DEFAULT_TIMEOUT_MS; - const start = Date.now(); - while (Date.now() - start < timeout) { - try { - const data = await runIosRunnerCommand( - device, - { command: 'alert', action: 'get', appBundleId: session?.appBundleId }, - { - verbose: req.flags?.verbose, - logPath, - traceLogPath: session?.trace?.outPath, - requestId: req.meta?.requestId, - }, - ); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; - } catch { - // keep waiting - } - await sleep(POLL_INTERVAL_MS); - } - return errorResponse('COMMAND_FAILED', 'alert wait timed out'); + if (device.platform === 'macos') { + const runAlert: NativeAlertRunner = async (alertAction) => + await runMacOsAlertAction(alertAction, macOsAlertTarget); + return await handleNativeAlertCommand(params, action, runAlert); } - const resolvedAction = - action === 'accept' || action === 'dismiss' ? (action as 'accept' | 'dismiss') : 'get'; const runnerOptions = { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath, requestId: req.meta?.requestId, }; - if (resolvedAction === 'accept' || resolvedAction === 'dismiss') { - const ALERT_ACTION_RETRY_MS = 2_000; - const start = Date.now(); - let lastError: unknown; - while (Date.now() - start < ALERT_ACTION_RETRY_MS) { - try { - const data = await runIosRunnerCommand( - device, - { command: 'alert', action: resolvedAction, appBundleId: session?.appBundleId }, - runnerOptions, - ); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; - } catch (err) { - lastError = err; - const msg = String((err as { message?: unknown })?.message ?? '').toLowerCase(); - if (!msg.includes('alert not found') && !msg.includes('no alert')) break; - } - await sleep(POLL_INTERVAL_MS); - } - // lastError is always set because ALERT_ACTION_RETRY_MS > 0 - throw withAlertFallbackHint(lastError); + const runAlert: NativeAlertRunner = async (alertAction) => + await runIosRunnerCommand( + device, + { command: 'alert', action: alertAction, appBundleId: session?.appBundleId }, + runnerOptions, + ); + return await handleNativeAlertCommand(params, action, runAlert); +} + +async function handleNativeAlertCommand( + params: HandleAlertCommandParams, + action: AlertAction, + runAlert: NativeAlertRunner, +): Promise { + if (action === 'wait') { + return await waitForNativeAlert(params, runAlert); } - const data = await runIosRunnerCommand( - device, - { command: 'alert', action: resolvedAction, appBundleId: session?.appBundleId }, - runnerOptions, - ); - recordIfSession(sessionStore, session, req, data as Record); - return { ok: true, data }; + const resolvedAction = action === 'accept' || action === 'dismiss' ? action : 'get'; + if (resolvedAction === 'accept' || resolvedAction === 'dismiss') { + return await handleNativeAlertAction(params, resolvedAction, runAlert); + } + + return recordAlertResponse(params, await runAlert('get')); +} + +function normalizeAlertAction(action: string | undefined): AlertAction { + if (action === 'accept' || action === 'dismiss' || action === 'wait') return action; + return 'get'; +} + +async function waitForNativeAlert( + params: HandleAlertCommandParams, + runAlert: NativeAlertRunner, +): Promise { + const timeout = parseTimeout(params.req.positionals?.[1]) ?? DEFAULT_TIMEOUT_MS; + const start = Date.now(); + while (Date.now() - start < timeout) { + try { + return recordAlertResponse(params, await runAlert('get')); + } catch { + // keep waiting + } + await sleep(POLL_INTERVAL_MS); + } + return errorResponse('COMMAND_FAILED', 'alert wait timed out'); +} + +async function handleNativeAlertAction( + params: HandleAlertCommandParams, + action: 'accept' | 'dismiss', + runAlert: NativeAlertRunner, +): Promise { + const start = Date.now(); + let lastError: unknown; + while (Date.now() - start < ALERT_ACTION_RETRY_MS) { + try { + return recordAlertResponse(params, await runAlert(action)); + } catch (err) { + lastError = err; + const msg = String((err as { message?: unknown })?.message ?? '').toLowerCase(); + if (!msg.includes('alert not found') && !msg.includes('no alert')) break; + } + await sleep(POLL_INTERVAL_MS); + } + throw withAlertFallbackHint(lastError); +} + +function recordAlertResponse(params: HandleAlertCommandParams, data: unknown): DaemonResponse { + const responseData = data as Record; + recordIfSession(params.sessionStore, params.session, params.req, responseData); + return { ok: true, data: responseData }; } function withAlertFallbackHint(error: unknown): unknown { if (!(error instanceof AppError)) { return error; } - const message = String(error.message ?? '').toLowerCase(); - if (!message.includes('alert not found') && !message.includes('no alert')) { + if (!isAlertNotFoundError(error)) { return error; } return new AppError(error.code, error.message, { @@ -161,3 +155,8 @@ function withAlertFallbackHint(error: unknown): unknown { hint: ALERT_FALLBACK_HINT, }); } + +function isAlertNotFoundError(error: unknown): boolean { + const message = String((error as { message?: unknown })?.message ?? '').toLowerCase(); + return message.includes('alert not found') || message.includes('no alert'); +} diff --git a/src/index.ts b/src/index.ts index 065f7a5a5..dfa7cae7a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,8 +31,12 @@ export type { AgentDeviceSelectionOptions, AgentDeviceSession, AgentDeviceSessionDevice, + AlertAction, AlertCommandOptions, + AlertInfo, + AlertPlatform, AlertCommandResult, + AlertSource, AppCloseOptions, AppCloseResult, AppDeployOptions, diff --git a/src/platforms/android/__tests__/alert-detection.test.ts b/src/platforms/android/__tests__/alert-detection.test.ts new file mode 100644 index 000000000..c28421791 --- /dev/null +++ b/src/platforms/android/__tests__/alert-detection.test.ts @@ -0,0 +1,133 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { chooseAndroidAlertButton, findAndroidAlertCandidate } from '../alert-detection.ts'; +import type { RawSnapshotNode } from '../../../utils/snapshot.ts'; + +test('chooseAndroidAlertButton prefers platform ids over ambiguous labels', () => { + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog'), + node(1, 'android.widget.TextView', { + label: 'Ambiguous', + identifier: 'android:id/alertTitle', + }), + button(2, 'Not now', 'android:id/button1', { x: 210, y: 612 }), + button(3, 'Allow', 'android:id/button2', { x: 52, y: 612 }), + button(4, 'Later', 'android:id/button3', { x: 130, y: 680 }), + ]); + + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, 'Not now'); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, 'Allow'); +}); + +test('chooseAndroidAlertButton classifies permission ids before labels', () => { + const candidate = findAndroidAlertCandidate([ + node(0, 'android.widget.FrameLayout', { + bundleId: 'com.google.android.permissioncontroller', + }), + node(1, 'android.widget.TextView', { + label: 'Camera access', + identifier: 'com.android.permissioncontroller:id/permission_message', + bundleId: 'com.google.android.permissioncontroller', + }), + button(2, 'No thanks', 'permission_allow_foreground_only_button', { x: 210, y: 612 }, true), + button(3, 'OK', 'permission_deny_button', { x: 52, y: 612 }, true), + ]); + + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, 'No thanks'); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, 'OK'); +}); + +test('findAndroidAlertCandidate collects descendants independent of node order', () => { + const candidate = findAndroidAlertCandidate([ + text(3, 'Leave without saving?', 'android:id/message', 2), + button(4, 'Discard', 'android:id/button1', { x: 210, y: 612 }, false, 2), + node(1, 'android.app.AlertDialog'), + text(2, 'Unsaved changes', 'android:id/alertTitle', 1), + node(0, 'android.widget.FrameLayout'), + ]); + + assert.deepEqual(candidate?.alert, { + title: 'Unsaved changes', + message: 'Leave without saving?', + buttons: ['Discard'], + platform: 'android', + source: 'native-dialog', + packageName: 'com.example.app', + }); +}); + +test('findAndroidAlertCandidate ignores normal app message ids', () => { + const candidate = findAndroidAlertCandidate([ + text(1, 'Latest chat message', 'com.example:id/message'), + button(2, 'Reply', 'com.example:id/reply', { x: 210, y: 612 }), + button(3, 'Archive', 'com.example:id/archive', { x: 52, y: 612 }), + node(0, 'android.widget.FrameLayout'), + ]); + + assert.equal(candidate, null); +}); + +test('findAndroidAlertCandidate keeps buttonless native dialogs for Back fallback', () => { + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog', { identifier: 'android:id/parentPanel' }), + text(1, 'Unsaved changes', 'android:id/alertTitle'), + text(2, 'Leave without saving?', 'android:id/message'), + ]); + + assert.deepEqual(candidate?.alert, { + title: 'Unsaved changes', + message: 'Leave without saving?', + buttons: [], + platform: 'android', + source: 'native-dialog', + packageName: 'com.example.app', + }); +}); + +test('chooseAndroidAlertButton accepts a single neutral button', () => { + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog'), + text(1, 'Storage full', 'android:id/alertTitle'), + button(2, 'Later', 'android:id/button3', { x: 130, y: 680 }), + ]); + + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, 'Later'); +}); + +function node( + index: number, + type: string, + overrides: Partial = {}, +): RawSnapshotNode { + return { + index, + parentIndex: index === 0 ? undefined : 0, + type, + bundleId: 'com.example.app', + ...overrides, + }; +} + +function text(index: number, label: string, identifier: string, parentIndex = 0): RawSnapshotNode { + return node(index, 'android.widget.TextView', { label, identifier, parentIndex }); +} + +function button( + index: number, + label: string, + identifier: string, + origin: { x: number; y: number }, + permission = false, + parentIndex = 0, +): RawSnapshotNode { + const packageName = permission ? 'com.google.android.permissioncontroller' : 'com.example.app'; + const id = permission ? `com.android.permissioncontroller:id/${identifier}` : identifier; + return node(index, 'android.widget.Button', { + label, + identifier: id, + bundleId: packageName, + parentIndex, + rect: { ...origin, width: 128, height: 52 }, + hittable: true, + }); +} diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 372f95467..6c42d9bee 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -333,6 +333,33 @@ test('snapshotAndroid uses injected helper artifact before stock uiautomator', a assert.equal(mockRunCmd.mock.calls.length, 0); }); +test('snapshotAndroid forwards alert-style helper idle timeout override', async () => { + let instrumentArgs: string[] | undefined; + const helperAdb: AndroidAdbExecutor = async (args) => { + if (args.includes('--show-versioncode')) { + return installedHelperProbe; + } + if (args.includes('instrument')) { + instrumentArgs = args; + return { + exitCode: 0, + stdout: helperOutput(''), + stderr: '', + }; + } + throw new Error(`unexpected helper adb args: ${args.join(' ')}`); + }; + + await snapshotAndroid(device, { + helperAdb, + helperArtifact, + helperWaitForIdleTimeoutMs: 0, + }); + + assert.ok(instrumentArgs); + assert.equal(instrumentArgs[instrumentArgs.indexOf('waitForIdleTimeoutMs') + 1], '0'); +}); + test('snapshotAndroid emits helper phase diagnostics', async () => { const helperAdb: AndroidAdbExecutor = async (args) => { if (args.includes('--show-versioncode')) { @@ -891,6 +918,34 @@ test('snapshotAndroid skips activity dump when snapshot has no scrollable nodes' assert.equal(result.nodes[0]?.label, 'Continue'); }); +test('snapshotAndroid skips hidden content hints when disabled', async () => { + const xml = ` + + + + + + +`; + + mockRunCmd.mockImplementation(async (_cmd, args) => { + if (args.includes('exec-out')) { + return { exitCode: 0, stdout: xml, stderr: '' }; + } + if (args.includes('dumpsys') && args.includes('activity') && args.includes('top')) { + throw new Error('dumpsys activity top should not run when hints are disabled'); + } + throw new Error(`unexpected args: ${args.join(' ')}`); + }); + + const result = await snapshotAndroid(device, { includeHiddenContentHints: false }); + + assert.equal( + result.nodes.some((node) => node.type === 'android.widget.ScrollView'), + true, + ); +}); + test('snapshotAndroid derives hidden content hints for interactive snapshots from shared visibility semantics', async () => { const xml = ` diff --git a/src/platforms/android/alert-detection.ts b/src/platforms/android/alert-detection.ts new file mode 100644 index 000000000..566acc0cf --- /dev/null +++ b/src/platforms/android/alert-detection.ts @@ -0,0 +1,271 @@ +import { centerOfRect, type RawSnapshotNode } from '../../utils/snapshot.ts'; +import type { AlertInfo, AlertSource } from '../../alert-contract.ts'; + +type AndroidAlertButtonRole = 'accept' | 'dismiss' | 'neutral'; + +export type AndroidAlertButton = { + label: string; + x: number; + y: number; + role: AndroidAlertButtonRole; +}; + +type AndroidAlertSource = Extract; + +export type AndroidAlertInfo = AlertInfo & { + buttons: string[]; + platform: 'android'; + source: AndroidAlertSource; + packageName?: string; +}; + +export type AndroidAlertCandidate = { + alert: AndroidAlertInfo; + buttons: AndroidAlertButton[]; +}; + +const ANDROID_PERMISSION_PACKAGES = new Set([ + 'com.android.permissioncontroller', + 'com.google.android.permissioncontroller', + 'com.google.android.packageinstaller', + 'com.android.packageinstaller', +]); +const ANDROID_SYSTEM_DIALOG_PACKAGES = new Set(['android', 'com.android.systemui']); +const ANDROID_ALERT_ID_PATTERN = + /^android:id\/(?:alertTitle|message|button[123]|parentPanel|buttonPanel|contentPanel)$/i; +const ANDROID_ALERT_BUTTON_ID_PATTERN = /^android:id\/button[123]$/i; +const ANDROID_PERMISSION_ID_PATTERN = /(?:^|:)id\/permission_/i; +const ANDROID_BLOCKING_DIALOG_PATTERN = + /\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i; +const ACCEPT_LABEL_PATTERN = + /^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i; +const DISMISS_LABEL_PATTERN = + /^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i; + +export function findAndroidAlertCandidate(nodes: RawSnapshotNode[]): AndroidAlertCandidate | null { + const candidate = findAndroidAlertNodes(nodes); + const candidateNodes = candidate.nodes; + if (candidateNodes.length === 0) return null; + + const buttons = findAlertButtons(candidateNodes); + const textNodes = candidateNodes.filter((node) => readNodeText(node) && !isButtonLike(node)); + const title = chooseAlertTitle(textNodes); + const message = chooseAlertMessage(textNodes, title); + const packageName = choosePackageName(candidateNodes); + return { + alert: { + ...(title ? { title } : {}), + ...(message ? { message } : {}), + buttons: buttons.map((button) => button.label), + platform: 'android', + source: candidate.source, + ...(packageName ? { packageName } : {}), + }, + buttons, + }; +} + +export function chooseAndroidAlertButton( + buttons: AndroidAlertButton[], + action: 'accept' | 'dismiss', +): AndroidAlertButton | null { + const role = action === 'accept' ? 'accept' : 'dismiss'; + const exact = buttons.find((button) => button.role === role); + if (exact) return exact; + if (action === 'dismiss') { + return buttons.find((button) => button.role === 'neutral') ?? null; + } + // Single-button Android dialogs commonly expose the only affirmative path as OK. + return buttons.length === 1 ? (buttons[0] ?? null) : null; +} + +function findAndroidAlertNodes(nodes: RawSnapshotNode[]): { + nodes: RawSnapshotNode[]; + source: AndroidAlertInfo['source']; +} { + const permissionNodes = nodes.filter((node) => isAndroidPermissionNode(node)); + if (permissionNodes.length) return { nodes: permissionNodes, source: 'permission' }; + + const systemDialogNodes = findSystemDialogNodes(nodes); + if (systemDialogNodes.length) return { nodes: systemDialogNodes, source: 'system-dialog' }; + + return { nodes: findNativeDialogNodes(nodes), source: 'native-dialog' }; +} + +function findNativeDialogNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + const dialogNodes = nodes.filter((node) => isAndroidDialogType(node.type ?? '')); + const alertIdNodes = nodes.filter((node) => ANDROID_ALERT_ID_PATTERN.test(node.identifier ?? '')); + const signalNodes = dialogNodes.length + ? [...dialogNodes, ...alertIdNodes] + : correlatedAndroidAlertIdNodes(alertIdNodes); + if (signalNodes.length === 0) return []; + const rootIndex = findCommonAncestorIndex(nodes, signalNodes); + if (rootIndex === undefined) return signalNodes; + return collectDescendants(nodes, rootIndex); +} + +function isAndroidDialogType(type: string): boolean { + return /(?:^|[.$])[^.]*Dialog$/i.test(type); +} + +function correlatedAndroidAlertIdNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + const hasButton = nodes.some((node) => + ANDROID_ALERT_BUTTON_ID_PATTERN.test(node.identifier ?? ''), + ); + const hasContent = nodes.some( + (node) => !ANDROID_ALERT_BUTTON_ID_PATTERN.test(node.identifier ?? ''), + ); + return hasButton && hasContent ? nodes : []; +} + +function findSystemDialogNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { + const signalNodes = nodes.filter((node) => isAndroidSystemDialogNode(node)); + if (signalNodes.length === 0) return []; + const rootIndex = findCommonAncestorIndex(nodes, signalNodes); + if (rootIndex === undefined) return signalNodes; + return collectDescendants(nodes, rootIndex).filter( + (node) => node.bundleId && ANDROID_SYSTEM_DIALOG_PACKAGES.has(node.bundleId), + ); +} + +function findAlertButtons(nodes: RawSnapshotNode[]): AndroidAlertButton[] { + const seen = new Set(); + const buttons: AndroidAlertButton[] = []; + for (const node of nodes) { + const label = readNodeText(node); + if (!label || !node.rect || !isButtonLike(node)) continue; + const normalized = label.trim().toLowerCase(); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + const point = centerOfRect(node.rect); + buttons.push({ + label, + x: point.x, + y: point.y, + role: classifyAndroidAlertButton(node, label), + }); + } + return buttons; +} + +function classifyAndroidAlertButton(node: RawSnapshotNode, label: string): AndroidAlertButtonRole { + const identifier = node.identifier ?? ''; + const roleFromId = classifyAndroidAlertButtonById(identifier); + if (roleFromId) return roleFromId; + if (ACCEPT_LABEL_PATTERN.test(label.trim())) return 'accept'; + if (DISMISS_LABEL_PATTERN.test(label.trim())) return 'dismiss'; + return 'neutral'; +} + +function classifyAndroidAlertButtonById(identifier: string): AndroidAlertButtonRole | null { + if (/(?:^|:)id\/button1$/i.test(identifier)) return 'accept'; + if (/(?:^|:)id\/button2$/i.test(identifier)) return 'dismiss'; + if (/(?:^|:)id\/button3$/i.test(identifier)) return 'neutral'; + if (/(?:^|:)id\/permission_allow/i.test(identifier)) return 'accept'; + if (/(?:^|:)id\/permission_deny/i.test(identifier)) return 'dismiss'; + return null; +} + +function chooseAlertTitle(nodes: RawSnapshotNode[]): string | undefined { + const explicit = nodes.find((node) => + /(?:^|:)id\/(?:alertTitle|permission_message)$/i.test(node.identifier ?? ''), + ); + return readNodeText(explicit) || readNodeText(nodes[0]); +} + +function chooseAlertMessage( + nodes: RawSnapshotNode[], + title: string | undefined, +): string | undefined { + const parts = nodes.map((node) => readNodeText(node)).filter((text) => text && text !== title); + return parts.length ? [...new Set(parts)].join('\n') : undefined; +} + +function findCommonAncestorIndex( + nodes: RawSnapshotNode[], + signalNodes: RawSnapshotNode[], +): number | undefined { + const first = signalNodes[0]; + if (!first) return undefined; + const common = ancestorIndexes(nodes, first.index); + for (const signal of signalNodes.slice(1)) { + const ancestors = new Set(ancestorIndexes(nodes, signal.index)); + for (let index = common.length - 1; index >= 0; index -= 1) { + if (!ancestors.has(common[index]!)) { + common.splice(index, 1); + } + } + } + return common[common.length - 1]; +} + +function ancestorIndexes(nodes: RawSnapshotNode[], index: number): number[] { + const byIndex = new Map(nodes.map((node) => [node.index, node])); + const indexes: number[] = []; + let current = byIndex.get(index); + while (current) { + indexes.push(current.index); + current = current.parentIndex === undefined ? undefined : byIndex.get(current.parentIndex); + } + return indexes.reverse(); +} + +function collectDescendants(nodes: RawSnapshotNode[], rootIndex: number): RawSnapshotNode[] { + const childrenByParent = new Map(); + for (const node of nodes) { + if (node.parentIndex === undefined) continue; + const siblings = childrenByParent.get(node.parentIndex) ?? []; + siblings.push(node); + childrenByParent.set(node.parentIndex, siblings); + } + + const descendants = new Set([rootIndex]); + const pending = [rootIndex]; + for (const index of pending) { + for (const child of childrenByParent.get(index) ?? []) { + if (descendants.has(child.index)) continue; + descendants.add(child.index); + pending.push(child.index); + } + } + return nodes.filter((node) => descendants.has(node.index)); +} + +function isAndroidPermissionNode(node: RawSnapshotNode): boolean { + const packageName = node.bundleId ?? ''; + return ( + ANDROID_PERMISSION_PACKAGES.has(packageName) || + ANDROID_PERMISSION_ID_PATTERN.test(node.identifier ?? '') + ); +} + +function isAndroidSystemDialogNode(node: RawSnapshotNode): boolean { + const packageName = node.bundleId ?? ''; + return ( + ANDROID_SYSTEM_DIALOG_PACKAGES.has(packageName) && + ANDROID_BLOCKING_DIALOG_PATTERN.test(readNodeText(node)) + ); +} + +function isButtonLike(node: RawSnapshotNode): boolean { + const type = node.type ?? ''; + const identifier = node.identifier ?? ''; + return Boolean( + node.hittable || + /\bbutton\b/i.test(type) || + ANDROID_ALERT_BUTTON_ID_PATTERN.test(identifier) || + /(?:^|:)id\/permission_(?:allow|deny)/i.test(identifier), + ); +} + +function readNodeText(node: RawSnapshotNode | undefined): string { + if (!node) return ''; + const parts = [node.label, node.value].filter( + (part): part is string => typeof part === 'string' && part.trim().length > 0, + ); + return parts[0]?.trim() ?? ''; +} + +function choosePackageName(nodes: RawSnapshotNode[]): string | undefined { + return nodes.find((node) => node.bundleId)?.bundleId; +} diff --git a/src/platforms/android/alert.ts b/src/platforms/android/alert.ts new file mode 100644 index 000000000..984c410c7 --- /dev/null +++ b/src/platforms/android/alert.ts @@ -0,0 +1,161 @@ +import type { AlertAction } from '../../alert-contract.ts'; +import { + ALERT_ACTION_RETRY_MS, + ALERT_POLL_INTERVAL_MS, + DEFAULT_ALERT_TIMEOUT_MS, +} from '../../alert-contract.ts'; +import { AppError } from '../../utils/errors.ts'; +import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; +import { successText } from '../../utils/success-text.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import type { DeviceInfo } from '../../utils/device.ts'; +import { + chooseAndroidAlertButton, + findAndroidAlertCandidate, + type AndroidAlertCandidate, + type AndroidAlertInfo, +} from './alert-detection.ts'; +import { backAndroid, pressAndroid } from './input-actions.ts'; +import { snapshotAndroid } from './snapshot.ts'; + +export type AndroidAlertResult = + | { + kind: 'alertStatus'; + platform: 'android'; + action: 'get'; + alert: AndroidAlertInfo | null; + message?: string; + } + | { + kind: 'alertWait'; + platform: 'android'; + action: 'wait'; + alert: AndroidAlertInfo; + waitedMs: number; + message?: string; + } + | { + kind: 'alertHandled'; + platform: 'android'; + action: 'accept' | 'dismiss'; + handled: true; + alert: AndroidAlertInfo; + button: string; + message?: string; + }; + +export async function handleAndroidAlert( + device: DeviceInfo, + action: AlertAction, + options: { timeoutMs?: number } = {}, +): Promise { + if (action === 'wait') { + return await waitForAndroidAlert(device, options.timeoutMs ?? DEFAULT_ALERT_TIMEOUT_MS); + } + if (action === 'get') { + const candidate = await readAndroidAlertCandidate(device); + return buildAndroidAlertStatusResponse(candidate?.alert ?? null); + } + return await handleAndroidAlertAction(device, action); +} + +async function waitForAndroidAlert( + device: DeviceInfo, + timeoutMs: number, +): Promise { + const start = Date.now(); + const candidate = await pollAndroidAlertCandidate(device, timeoutMs); + if (!candidate) { + throw new AppError('COMMAND_FAILED', 'alert wait timed out'); + } + return { + kind: 'alertWait', + platform: 'android', + action: 'wait', + alert: candidate.alert, + waitedMs: Date.now() - start, + ...successText('Alert visible'), + }; +} + +async function handleAndroidAlertAction( + device: DeviceInfo, + action: 'accept' | 'dismiss', +): Promise { + const candidate = await pollAndroidAlertCandidate(device, ALERT_ACTION_RETRY_MS); + if (!candidate) { + throw new AppError('COMMAND_FAILED', 'alert not found', { + hint: 'If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.', + }); + } + + const button = chooseAndroidAlertButton(candidate.buttons, action); + if (button) { + await pressAndroid(device, button.x, button.y); + return buildAndroidAlertHandledResponse(action, candidate.alert, button.label); + } + + if (action === 'dismiss') { + await backAndroid(device); + return buildAndroidAlertHandledResponse(action, candidate.alert, 'Back'); + } + + throw new AppError('COMMAND_FAILED', 'alert accept found an alert but no accept button', { + alert: candidate.alert, + hint: 'Inspect alert get --json for visible buttons, then use press by visible label/ref if needed.', + }); +} + +async function pollAndroidAlertCandidate( + device: DeviceInfo, + timeoutMs: number, +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const candidate = await readAndroidAlertCandidate(device); + if (candidate) return candidate; + await sleep(ALERT_POLL_INTERVAL_MS); + } + return null; +} + +async function readAndroidAlertCandidate( + device: DeviceInfo, +): Promise { + const result = await withDiagnosticTimer( + 'snapshot_capture', + async () => + await snapshotAndroid(device, { + helperWaitForIdleTimeoutMs: 0, + includeHiddenContentHints: false, + }), + { backend: 'android', purpose: 'alert' }, + ); + return findAndroidAlertCandidate(result.nodes); +} + +function buildAndroidAlertStatusResponse(alert: AndroidAlertInfo | null): AndroidAlertResult { + return { + kind: 'alertStatus', + platform: 'android', + action: 'get', + alert, + ...(alert ? successText('Alert visible') : successText('No alert visible')), + }; +} + +function buildAndroidAlertHandledResponse( + action: 'accept' | 'dismiss', + alert: AndroidAlertInfo, + button: string, +): AndroidAlertResult { + return { + kind: 'alertHandled', + platform: 'android', + action, + handled: true, + alert, + button, + ...successText(`Alert ${action}ed`), + }; +} diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index 4193ede2f..d344a71d4 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -47,6 +47,8 @@ type AndroidSnapshotOptions = SnapshotOptions & { helperArtifact?: AndroidSnapshotHelperArtifact; helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy; helperAdb?: AndroidAdbExecutor; + helperWaitForIdleTimeoutMs?: number; + includeHiddenContentHints?: boolean; }; export async function snapshotAndroid( @@ -61,10 +63,13 @@ export async function snapshotAndroid( const adb = resolveAndroidAdbExecutor(device, options.helperAdb); const capture = await captureAndroidUiHierarchy(device, options, adb); const xml = capture.xml; + const includeHiddenContentHints = options.includeHiddenContentHints !== false; if (!options.interactiveOnly) { const parsed = parseUiHierarchy(xml, ANDROID_SNAPSHOT_MAX_NODES, options); - const nativeHints = await deriveScrollableContentHintsIfNeeded(device, parsed.nodes, adb); - applyHiddenContentHintsToNodes(nativeHints, parsed.nodes); + if (includeHiddenContentHints) { + const nativeHints = await deriveScrollableContentHintsIfNeeded(device, parsed.nodes, adb); + applyHiddenContentHintsToNodes(nativeHints, parsed.nodes); + } return { ...parsed, androidSnapshot: capture.metadata }; } @@ -74,13 +79,19 @@ export async function snapshotAndroid( interactiveOnly: false, }); const interactiveSnapshot = buildUiHierarchySnapshot(tree, ANDROID_SNAPSHOT_MAX_NODES, options); - const nativeHints = await deriveScrollableContentHintsIfNeeded(device, fullSnapshot.nodes, adb); - applyHiddenContentHintsToInteractiveNodes(nativeHints, fullSnapshot, interactiveSnapshot); - if (nativeHints.size === 0) { - const presentationHints = deriveMobileSnapshotHiddenContentHints( - attachRefs(fullSnapshot.nodes), - ); - applyHiddenContentHintsToInteractiveNodes(presentationHints, fullSnapshot, interactiveSnapshot); + if (includeHiddenContentHints) { + const nativeHints = await deriveScrollableContentHintsIfNeeded(device, fullSnapshot.nodes, adb); + applyHiddenContentHintsToInteractiveNodes(nativeHints, fullSnapshot, interactiveSnapshot); + if (nativeHints.size === 0) { + const presentationHints = deriveMobileSnapshotHiddenContentHints( + attachRefs(fullSnapshot.nodes), + ); + applyHiddenContentHintsToInteractiveNodes( + presentationHints, + fullSnapshot, + interactiveSnapshot, + ); + } } const { sourceNodes: _sourceNodes, ...snapshot } = interactiveSnapshot; return { ...snapshot, androidSnapshot: capture.metadata }; @@ -133,7 +144,9 @@ async function captureAndroidUiHierarchy( adb, packageName: helper.artifact!.manifest.packageName, instrumentationRunner: helper.artifact!.manifest.instrumentationRunner, - waitForIdleTimeoutMs: ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + waitForIdleTimeoutMs: + options.helperWaitForIdleTimeoutMs ?? + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, }), diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index d98dd6d06..62c694b5d 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -157,6 +157,7 @@ function appendAndroidSnapshotNode( label: node.label ?? undefined, value: node.value ?? undefined, identifier: node.identifier ?? undefined, + bundleId: node.packageName ?? undefined, rect: node.rect, enabled: node.enabled, hittable: node.hittable, @@ -345,6 +346,7 @@ export type AndroidUiHierarchy = { label: string | null; value: string | null; identifier: string | null; + packageName: string | null; rect?: Rect; enabled?: boolean; hittable?: boolean; @@ -371,6 +373,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { label: null, value: null, identifier: null, + packageName: null, depth: -1, children: [], }; @@ -391,6 +394,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { label: attrs.text || attrs.desc, value: attrs.text, identifier: attrs.resourceId, + packageName: attrs.packageName, rect: attrs.rect, enabled: attrs.enabled, hittable: attrs.clickable ?? attrs.focusable, diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index bfcc85f44..77482f096 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -953,8 +953,8 @@ test('usageForCommand resolves debugging help topic', () => { if (help === null) throw new Error('Expected debugging help text'); assert.match(help, /agent-device help debugging/); assert.match(help, /agent-device alert wait 3000/); - assert.match(help, /Do not use alert wait for them/); - assert.match(help, /do not use settings permission to answer a dialog already on screen/); + assert.match(help, /iOS support is runner-derived/); + assert.match(help, /Do not use settings permission to answer a dialog already on screen/); }); test('usageForCommand resolves remote help topic', () => { @@ -1043,7 +1043,7 @@ test('usageForCommand resolves react-native help topic', () => { assert.match(help, /logs clear --restart/); assert.match(help, /network dump --include headers/); assert.match(help, /React Native warning\/error overlays belong to the app run/); - assert.match(help, /Android runtime permission dialogs are visible UI/); + assert.match(help, /Android runtime permission dialogs and native alerts are handled by alert/); assert.match(help, /snapshot times out because the UI never becomes idle/); assert.match(help, /Report React render offenders separately/); }); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index e3d80da0d..d7eea11af 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -383,14 +383,15 @@ Network: network log is a supported alias, but network dump --include headers is the clearest plan form. Do not write network log headers. Alerts: - Native alerts: + Native and platform dialogs: agent-device alert wait 3000 agent-device alert accept agent-device alert dismiss - If alert accept says no alert but a permission sheet is visibly on screen, treat it as normal UI: + Android support is snapshot-derived for runtime permission prompts and native app dialogs. iOS support is runner-derived for XCTest alerts, app-owned modal popups with native blocking markers, and blocking system dialogs. Use cheap alert get for an immediate check; use alert wait only when a prompt may appear after async work. + If alert says no alert but a sheet is visibly on screen, treat it as app-owned UI: agent-device snapshot -i agent-device press 'label="Allow"' - Android runtime permission prompts are visible UI. Do not use alert wait for them, and do not use settings permission to answer a dialog already on screen. Reserve settings permission for setup/resetting permission state before a flow. + Do not use settings permission to answer a dialog already on screen. Reserve settings permission for setup/resetting permission state before a flow. Diagnostics and traces: Use --debug for CLI/daemon diagnostic ids and log paths. @@ -502,7 +503,7 @@ Overlays and busy RN UIs: React Native warning/error overlays belong to the app run. Treat them as blockers before normal app work: press visible Dismiss/Close immediately when unrelated, or press the collapsed warning banner first if only a warning chip is visible; then re-snapshot and report the overlay in the final summary. Use screenshot --overlay-refs only if visual evidence is required. Full-screen RedBox stack traces are app errors, not failed navigation. If Minimize is visible, prefer Minimize over Dismiss; Dismiss can re-trigger infinite-loop render errors such as getSnapshot/useOnyx stack paths. If snapshot times out because the UI never becomes idle, Android accessibility may be blocked by busy or continuously changing app UI. After that timeout, use screenshot as visual truth instead of repeatedly retrying snapshots. - Android runtime permission dialogs are visible UI: inspect snapshot -i and press Allow/Deny by visible label/ref. Use alert wait/accept/dismiss only where the platform supports native alerts. + Android runtime permission dialogs and native alerts are handled by alert wait/accept/dismiss. If alert reports no alert, treat the visible surface as app-owned UI and use snapshot -i plus press by label/ref. React DevTools routing: Keep the agent-device react-devtools prefix on every React DevTools command. @@ -1626,8 +1627,8 @@ const COMMAND_SCHEMAS: Record = { ...SELECTOR_COMMAND_SCHEMAS, alert: { usageOverride: 'alert [get|accept|dismiss|wait] [timeout]', - helpDescription: 'Inspect or handle alert (iOS simulator and macOS desktop)', - summary: 'Inspect or handle iOS/macOS alerts', + helpDescription: 'Inspect or handle platform alerts/dialogs', + summary: 'Inspect or handle platform alerts', positionalArgs: ['action?', 'timeout?'], allowedFlags: [], }, diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index a0f24c9bd..fd627e2c7 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -72,6 +72,155 @@ test('Provider-backed integration Android text provider handles Unicode without ); }); +test('Provider-backed integration Android alert handles runtime permission dialog', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: androidRuntimePermissionXml }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertGet = await client.command.alert({ action: 'get', ...world.selection }); + assert.equal(alertGet.kind, 'alertStatus'); + assert.deepEqual(alertGet.alert, { + title: 'Allow Demo to send you notifications?', + buttons: ['Don’t allow', 'Allow'], + platform: 'android', + source: 'permission', + packageName: 'com.google.android.permissioncontroller', + }); + + const alertAccept = await client.command.alert({ action: 'accept', ...world.selection }); + assert.equal(alertAccept.kind, 'alertHandled'); + assert.equal(alertAccept.button, 'Allow'); + assert.deepEqual( + world.adbCalls.filter((call) => call.join(' ') === 'shell input tap 274 638'), + [['shell', 'input', 'tap', '274', '638']], + ); + + const alertDismiss = await client.command.alert({ action: 'dismiss', ...world.selection }); + assert.equal(alertDismiss.kind, 'alertHandled'); + assert.equal(alertDismiss.button, 'Don’t allow'); + assert.deepEqual( + world.adbCalls.filter((call) => call.join(' ') === 'shell input tap 116 638'), + [['shell', 'input', 'tap', '116', '638']], + ); + }, + ); +}); + +test('Provider-backed integration Android alert handles native AlertDialog actions', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: androidNativeAlertXml }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertGet = await client.command.alert({ action: 'get', ...world.selection }); + assert.deepEqual(alertGet.alert, { + title: 'Unsaved changes', + message: 'Leave without saving?', + buttons: ['Cancel', 'Discard'], + platform: 'android', + source: 'native-dialog', + packageName: 'com.example.demo', + }); + + const alertAccept = await client.command.alert({ action: 'accept', ...world.selection }); + assert.equal(alertAccept.button, 'Discard'); + const alertDismiss = await client.command.alert({ action: 'dismiss', ...world.selection }); + assert.equal(alertDismiss.button, 'Cancel'); + assert.deepEqual( + world.adbCalls.filter((call) => + ['shell input tap 274 638', 'shell input tap 116 638'].includes(call.join(' ')), + ), + [ + ['shell', 'input', 'tap', '274', '638'], + ['shell', 'input', 'tap', '116', '638'], + ], + ); + }, + ); +}); + +test('Provider-backed integration Android alert handles system dialogs', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: androidSystemDialogXml }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertGet = await client.command.alert({ action: 'get', ...world.selection }); + assert.deepEqual(alertGet.alert, { + title: "Demo isn't responding", + message: 'Do you want to close it?', + buttons: ['Close app', 'Wait'], + platform: 'android', + source: 'system-dialog', + packageName: 'com.android.systemui', + }); + + const alertDismiss = await client.command.alert({ action: 'dismiss', ...world.selection }); + assert.equal(alertDismiss.button, 'Close app'); + assertCommandCall(world.adbCalls, ['shell', 'input', 'tap', '116', '638']); + }, + ); +}); + +test('Provider-backed integration Android alert dismiss falls back to Back without a dismiss button', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: androidButtonlessAlertXml }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertDismiss = await client.command.alert({ action: 'dismiss', ...world.selection }); + assert.equal(alertDismiss.kind, 'alertHandled'); + assert.equal(alertDismiss.button, 'Back'); + assertCommandCall(world.adbCalls, ['shell', 'input', 'keyevent', '4']); + }, + ); +}); + +test('Provider-backed integration Android alert wait polls until a dialog appears', async () => { + let snapshotCount = 0; + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: () => { + snapshotCount += 1; + return snapshotCount === 1 ? androidAppOwnedSheetXml() : androidRuntimePermissionXml(); + }, + }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertWait = await client.command.alert({ + action: 'wait', + timeoutMs: 1000, + ...world.selection, + }); + assert.equal(alertWait.kind, 'alertWait'); + assert.equal(alertWait.alert?.source, 'permission'); + assert.ok(snapshotCount >= 2); + }, + ); +}); + +test('Provider-backed integration Android alert ignores app-owned sheets', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: androidAppOwnedSheetXml }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'com.example.demo', ...world.selection }); + + const alertGet = await client.command.alert({ action: 'get', ...world.selection }); + assert.equal(alertGet.kind, 'alertStatus'); + assert.equal(alertGet.alert, null); + }, + ); +}); + async function runAndroidSetupAndInstallWorkflow( world: AndroidSettingsWorld, client: AgentDeviceClient, @@ -246,6 +395,151 @@ async function runAndroidSetupAndInstallWorkflow( assert.equal(keyboard.visible, false); } +function androidRuntimePermissionXml(): string { + const packageName = 'com.google.android.permissioncontroller'; + return androidXml([ + rootNode(packageName), + textNode( + 1, + 'Allow Demo to send you notifications?', + 'com.android.permissioncontroller:id/permission_message', + packageName, + '[24,300][366,352]', + ), + buttonNode( + 2, + 'Don’t allow', + 'com.android.permissioncontroller:id/permission_deny_button', + '[52,612][180,664]', + packageName, + ), + buttonNode( + 3, + 'Allow', + 'com.android.permissioncontroller:id/permission_allow_button', + '[210,612][338,664]', + packageName, + ), + ' ', + ]); +} + +function androidNativeAlertXml(): string { + return androidDialogXml([ + textNode(2, 'Unsaved changes', 'android:id/alertTitle'), + textNode(3, 'Leave without saving?', 'android:id/message'), + buttonNode(4, 'Cancel', 'android:id/button2', '[52,612][180,664]'), + buttonNode(5, 'Discard', 'android:id/button1', '[210,612][338,664]'), + ]); +} + +function androidSystemDialogXml(): string { + const packageName = 'com.android.systemui'; + return androidXml([ + rootNode(packageName), + textNode(1, 'Demo isn't responding', 'android:id/alertTitle', packageName), + textNode(2, 'Do you want to close it?', 'android:id/message', packageName), + buttonNode(3, 'Close app', 'android:id/button2', '[52,612][180,664]', packageName), + buttonNode(4, 'Wait', 'android:id/button1', '[210,612][338,664]', packageName), + ' ', + ]); +} + +function androidButtonlessAlertXml(): string { + return androidDialogXml([ + textNode(2, 'Unsaved changes', 'android:id/alertTitle'), + textNode(3, 'Leave without saving?', 'android:id/message'), + ]); +} + +function androidAppOwnedSheetXml(): string { + return androidXml([ + rootNode('com.example.demo', 'com.example.demo:id/root'), + textNode(1, 'Choose an option', 'com.example.demo:id/title'), + buttonNode(2, 'Allow', 'com.example.demo:id/allow_button', '[210,612][338,664]'), + ' ', + ]); +} + +function androidDialogXml(children: string[]): string { + return androidXml([ + rootNode(), + androidNode({ + index: 1, + id: 'android:id/parentPanel', + type: 'android.app.AlertDialog', + bounds: '[24,240][366,680]', + selfClosing: false, + }), + ...children, + ' ', + ' ', + ]); +} + +function androidXml(body: string[]): string { + return [ + '', + '', + ...body, + '', + ].join('\n'); +} + +function rootNode(packageName = 'com.example.demo', id = 'android:id/content'): string { + return androidNode({ index: 0, id, type: 'FrameLayout', packageName, selfClosing: false }); +} + +function textNode( + index: number, + text: string, + id: string, + packageName = 'com.example.demo', + bounds?: string, +): string { + return androidNode({ index, text, id, packageName, ...(bounds ? { bounds } : {}) }); +} + +function buttonNode( + index: number, + text: string, + id: string, + bounds: string, + packageName = 'com.example.demo', +): string { + return androidNode({ index, text, id, type: 'Button', packageName, bounds, clickable: true }); +} + +function androidNode(options: { + index: number; + id: string; + text?: string; + type?: string; + packageName?: string; + bounds?: string; + clickable?: boolean; + selfClosing?: boolean; +}): string { + const type = options.type ?? 'TextView'; + const className = type.includes('.') ? type : `android.widget.${type}`; + const tagEnd = options.selfClosing === false ? '>' : ' />'; + return [ + ` string; }): Promise { const hostAdbGuard = installFakeHostAdbGuard(); const adbCalls: string[][] = []; @@ -75,7 +76,7 @@ export async function createAndroidSettingsWorld(options?: { if (args.join(' ') === 'shell cmd clipboard set text android otp') { clipboardText = 'android otp'; } - return androidAdbResult(args, searchText, clipboardText); + return androidAdbResult(args, searchText, clipboardText, options?.snapshotXml); }, install: async (apk, options) => { apkInstallCalls.push({ apkPath: apk, replace: options?.replace }); @@ -175,20 +176,45 @@ function androidAdbResult( args: string[], searchText: string, clipboardText: string, + snapshotXml?: () => string, ): { stdout: string; stderr: string; exitCode: number; stdoutBuffer?: Buffer } { - if (args.join(' ') === 'shell getprop sys.boot_completed') { + const key = args.join(' '); + return ( + androidDeviceStateAdbResult(key, clipboardText) ?? + androidMetricsAdbResult(key) ?? + androidPackageAdbResult(key, args) ?? + androidCaptureAdbResult(key, searchText, snapshotXml) ?? { stdout: '', stderr: '', exitCode: 0 } + ); +} + +type AndroidAdbResult = { + stdout: string; + stderr: string; + exitCode: number; + stdoutBuffer?: Buffer; +}; + +function androidDeviceStateAdbResult( + key: string, + clipboardText: string, +): AndroidAdbResult | undefined { + if (key === 'shell getprop sys.boot_completed') { return { stdout: '1\n', stderr: '', exitCode: 0 }; } - if (args.join(' ') === 'shell cmd clipboard get text') { + if (key === 'shell cmd clipboard get text') { return { stdout: `clipboard text: ${clipboardText}\n`, stderr: '', exitCode: 0 }; } - if (args.join(' ') === 'shell dumpsys input_method') { + if (key === 'shell dumpsys input_method') { return { stdout: 'mInputShown=false inputType=0x1\n', stderr: '', exitCode: 0 }; } - if (args.join(' ') === 'shell pidof com.example.demo') { + if (key === 'shell pidof com.example.demo') { return { stdout: '4242\n', stderr: '', exitCode: 0 }; } - if (args.join(' ') === 'shell dumpsys cpuinfo') { + return undefined; +} + +function androidMetricsAdbResult(key: string): AndroidAdbResult | undefined { + if (key === 'shell dumpsys cpuinfo') { return { stdout: [ 'Load: 1.0 / 0.5 / 0.25', @@ -200,7 +226,7 @@ function androidAdbResult( exitCode: 0, }; } - if (args.join(' ') === 'shell dumpsys meminfo com.example.demo') { + if (key === 'shell dumpsys meminfo com.example.demo') { return { stdout: [ '** MEMINFO in pid 18227 [com.example.demo] **', @@ -215,7 +241,7 @@ function androidAdbResult( exitCode: 0, }; } - if (args.join(' ') === 'shell dumpsys gfxinfo com.example.demo framestats') { + if (key === 'shell dumpsys gfxinfo com.example.demo framestats') { return { stdout: [ 'Uptime: 10000', @@ -227,6 +253,10 @@ function androidAdbResult( exitCode: 0, }; } + return undefined; +} + +function androidPackageAdbResult(key: string, args: string[]): AndroidAdbResult | undefined { if ( args.slice(0, 7).join(' ') === 'shell cmd package query-activities --brief -a android.intent.action.MAIN' @@ -237,31 +267,39 @@ function androidAdbResult( exitCode: 0, }; } - if (args.join(' ') === 'shell pm list packages -3') { + if (key === 'shell pm list packages -3') { return { stdout: 'package:com.example.demo\npackage:com.example.serviceonly\n', stderr: '', exitCode: 0, }; } - if (args.join(' ') === 'shell dumpsys window windows') { + if (key === 'shell dumpsys window windows') { return { stdout: 'mCurrentFocus=Window{42 u0 com.android.settings/.Settings}\n', stderr: '', exitCode: 0, }; } - if (args.join(' ') === 'exec-out uiautomator dump /dev/tty') { + return undefined; +} + +function androidCaptureAdbResult( + key: string, + searchText: string, + snapshotXml?: () => string, +): AndroidAdbResult | undefined { + if (key === 'exec-out uiautomator dump /dev/tty') { return { - stdout: androidSettingsXml(searchText), + stdout: snapshotXml?.() ?? androidSettingsXml(searchText), stderr: '', exitCode: 0, }; } - if (args.join(' ') === 'exec-out screencap -p') { + if (key === 'exec-out screencap -p') { return { stdout: '', stderr: '', exitCode: 0, stdoutBuffer: validPng() }; } - return { stdout: '', stderr: '', exitCode: 0 }; + return undefined; } export function androidSettingsXml( diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 80f29511c..9873e12ff 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1118,21 +1118,16 @@ const SKILL_GUIDANCE_CASES: Case[] = [ 'Action sheet trigger selector: id="composer-actions"', 'Scan option text: Scan document', 'Expected result text after upload: Document uploaded', - 'Android camera permission may appear as visible UI', + 'Android camera permission may appear as a runtime permission dialog', ], task: 'Plan commands to open the composer action sheet, choose Scan document, handle any visible permission prompt, and wait for the upload result.', outputs: [ /composer-actions/i, /Scan document/i, - /Allow|snapshot -i/i, + /alert (?:wait|accept)|Allow|snapshot -i/i, /wait\b[^\n]*Document uploaded/i, ], - forbiddenOutputs: [ - RAW_COORDINATE_TARGET, - PSEUDO_ASSERTION_COMMAND, - /alert wait/i, - /settings permission/i, - ], + forbiddenOutputs: [RAW_COORDINATE_TARGET, PSEUDO_ASSERTION_COMMAND, /settings permission/i], }), makeCase({ id: 'evidence-screenshot-overlay-refs', diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 9d62e078e..20ad24acb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -232,9 +232,10 @@ agent-device alert dismiss - `wait @ref` resolves the ref to its label/text from that stored snapshot, then polls for that text; it does not track the original node identity. - Because `wait @ref` is text-based after resolution, duplicate labels can match a different element than the original ref target. - `wait` shares the selector/snapshot resolution flow used by `click`, `fill`, `get`, and `is`. -- `alert` inspects or handles system alerts on iOS simulator and macOS desktop targets. +- `alert` inspects or handles system alerts on iOS simulator, macOS desktop, and Android native/runtime permission dialogs. - `alert` without an action is equivalent to `alert get`. -- `alert wait [timeout]` waits for an alert to appear before returning it. +- Use `alert get` for an immediate cheap check. Use `alert wait ` only when a prompt may appear after async work. +- Android support is snapshot-derived. If `alert` reports no alert but a sheet is visible, treat it as app-owned UI and use `snapshot -i` plus `press` by visible label/ref. - If an iOS permission sheet is visible in `snapshot` or `screenshot` but `alert accept` reports no alert, fall back to a scoped `snapshot -i -s ""` plus `press @ref`; not every simulator permission surface is exposed as a native XCTest alert. ## Interactions @@ -588,7 +589,7 @@ agent-device react-devtools profile report @c5 - Do not repeatedly raise broad `profile slow` limits such as `--limit 50`, `--limit 200`, or `--limit 500` unless you have a specific target that needs more rows. - Keep using `snapshot`, `press`, `fill`, `logs`, `network`, and `perf` for device/app runtime evidence. Use `react-devtools` for React internals. - For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to React DevTools or debugging evidence, start with `agent-device help react-native`. -- On Android, permission prompts are visible UI; use `snapshot -i` and press visible `Allow`/`Deny` controls instead of `alert wait`. Do not use `settings permission` to answer a dialog already on screen; reserve it for setup or resetting permission state before a flow. +- On Android, use `alert get`, `alert wait `, `alert accept`, and `alert dismiss` for runtime permission prompts and native alerts. On iOS, use the same alert commands for XCTest alerts, app-owned modal popups with native blocking markers, and blocking system dialogs. Do not use `settings permission` to answer a dialog already on screen; reserve it for setup or resetting permission state before a flow. - React Native development builds can connect to the DevTools daemon on port 8097. For Android emulators or physical devices, run `adb reverse tcp:8097 tcp:8097` if the app cannot reach the host. If Metro is local, also run `adb reverse tcp:8081 tcp:8081`. - For Android and iOS sessions connected through a remote bridge profile, `react-devtools` registers a lease-scoped companion tunnel to the sandbox-local DevTools daemon at `127.0.0.1:8097`. Android bridge profiles use the bridge-owned remote `adb reverse` mapping; iOS bridge profiles use the bridge-owned wildcard Metro host tunnel. The CLI keeps the companion alive until `agent-device react-devtools stop` or `agent-device disconnect`. - For remote iOS bridge sessions, open the app once to create the bridge session, run `agent-device react-devtools start`, then relaunch the same bundle id with `agent-device open --platform ios --relaunch` before `wait --connected`. React Native attempts the legacy DevTools websocket during JavaScript startup, so starting DevTools after the first launch can miss that connection attempt. diff --git a/website/docs/docs/debugging-profiling.md b/website/docs/docs/debugging-profiling.md index 6d8c84238..8a0d1cd7e 100644 --- a/website/docs/docs/debugging-profiling.md +++ b/website/docs/docs/debugging-profiling.md @@ -36,7 +36,7 @@ For React Native apps, overlays, Metro/Fast Refresh blockers, and routing to Rea React Native warning/error overlays belong to the app run. Treat them as findings or blockers: capture them, check `react-devtools errors` when connected, dismiss visible `Dismiss`/`Close` controls only when unrelated, then re-snapshot and report the overlay. -On Android, runtime permission dialogs are visible UI for `snapshot -i` plus visible `Allow`/`Deny` refs or labels. Do not use `alert wait` for Android permission prompts, and do not use `settings permission` to answer a dialog already on screen. Reserve `settings permission` for setup or resetting permission state before a flow. +Use `alert wait`, `alert accept`, and `alert dismiss` for Android runtime permission prompts, Android native alerts, and iOS platform/app-owned modal dialogs. Do not use `settings permission` to answer a dialog already on screen. Reserve `settings permission` for setup or resetting permission state before a flow. ## Fast path