refactor(type): single-pass daemon routing, shared binder source, owner cell tests

Review follow-ups on #1935.

- The type handler now calls the bound executor directly: the interaction-
  runtime hop validated and formatted what executeBoundTypeText validates and
  formats again, so it is gone — no boundTypeText backend member, no second
  result rebuild. The ADR 0014 frame expiry moves to the handler.
- New contracts/interactor-operation-binding.ts: one local resolver and one
  fail-closed provider resolver shared by the screenshot, focus, and type
  binders — three private copies retired, provider error text preserved.
- provider-limrun/interaction-operations.ts: the interactor-backed interaction
  cells move out of the app-log owner (586 -> 563 lines, below its pre-unit
  size); text interaction is composed by that owner, not defined in it.
- Every owner runtime test now pins the focusPoint/typeText fact cells and
  bound-operation presence for its exact kinds: apple, android (incl. the
  synthetic-simulator refusal), harmonyos, linux, web, vega (refusal + hint),
  webdriver (reachability-gated, incl. inactive session), limrun (live +
  recovery). The webdriver unsupported-capability row documents that
  interaction gates on interactor reachability, not capture declarations.
- The Linux replay assertion is now change-sensitive: type "555" then wait for
  a 555 node — no calculator button carries that label, so the wait passes only
  if the keystrokes landed in the display; deleting the type step turns it red.
This commit is contained in:
Michał Pierzchała
2026-08-21 14:02:22 +02:00
parent 82fb8c2dca
commit 3d44b6dea3
17 changed files with 226 additions and 127 deletions
+10 -19
View File
@@ -1,6 +1,11 @@
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import type { Point } from '@agent-device/kernel/snapshot';
import {
localInteractorSource,
providerInteractorSource,
type LocalInteractorOperationResolver,
type ProviderInteractorOperationResolver,
} from './interactor-operation-binding.ts';
import type { Interactor, RunnerContext } from './interactor-types.ts';
import type { RuntimeOperationFact } from './platform-runtime.ts';
import type { SessionSurface } from './session-surface.ts';
@@ -58,10 +63,7 @@ function bindFocusPoint(
});
}
export type LocalFocusInteractorResolver = (
device: DeviceInfo,
runner: RunnerContext,
) => Promise<Interactor>;
export type LocalFocusInteractorResolver = LocalInteractorOperationResolver;
export function bindLocalFocusInteractor(
params: Readonly<{
@@ -70,13 +72,10 @@ export function bindLocalFocusInteractor(
resolveInteractor: LocalFocusInteractorResolver;
}>,
): FocusRuntimeOperations {
return bindFocusPoint(
params.signal,
async (runner) => await params.resolveInteractor(params.device, runner),
);
return bindFocusPoint(params.signal, localInteractorSource(params));
}
export type ProviderFocusInteractorResolver = (runner: RunnerContext) => Interactor | undefined;
export type ProviderFocusInteractorResolver = ProviderInteractorOperationResolver;
/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */
export function bindProviderFocusInteractor(
@@ -86,13 +85,5 @@ export function bindProviderFocusInteractor(
resolveInteractor: ProviderFocusInteractorResolver;
}>,
): FocusRuntimeOperations {
return bindFocusPoint(params.signal, async (runner) => {
const interactor = params.resolveInteractor(runner);
if (interactor) return interactor;
throw new AppError(
'UNSUPPORTED_OPERATION',
'Provider-owned focus operation has no bound provider interactor.',
{ reason: 'provider-runtime-interactor-missing', deviceId: params.device.id },
);
});
return bindFocusPoint(params.signal, providerInteractorSource({ ...params, operation: 'focus' }));
}
@@ -0,0 +1,47 @@
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import type { Interactor, RunnerContext } from './interactor-types.ts';
/**
* The two ways an interactor-backed operation reaches its mechanics, shared by every binder that
* rides the `Interactor` seam (screenshot, focus, type, element-text). Each operation module owns
* its input/result contract; what they must NOT each own is a private copy of interactor
* resolution — that is duplication of mechanism, and the provider fail-closed rule below must be
* one rule, not one per operation.
*/
export type LocalInteractorOperationResolver = (
device: DeviceInfo,
runner: RunnerContext,
) => Promise<Interactor>;
export type ProviderInteractorOperationResolver = (runner: RunnerContext) => Interactor | undefined;
/** Resolves the already-selected local owner's interactor for one bound operation. */
export function localInteractorSource(
params: Readonly<{ device: DeviceInfo; resolveInteractor: LocalInteractorOperationResolver }>,
): (runner: RunnerContext) => Promise<Interactor> {
return async (runner) => await params.resolveInteractor(params.device, runner);
}
/**
* Resolves a provider's own interactor for one bound operation, failing closed when the exact
* owner no longer exposes it: facts advertised the operation, so a missing interactor is an
* ownership bug to surface, never a refusal to degrade around.
*/
export function providerInteractorSource(
params: Readonly<{
device: DeviceInfo;
operation: string;
resolveInteractor: ProviderInteractorOperationResolver;
}>,
): (runner: RunnerContext) => Promise<Interactor> {
return async (runner) => {
const interactor = params.resolveInteractor(runner);
if (interactor) return interactor;
throw new AppError(
'UNSUPPORTED_OPERATION',
`Provider-owned ${params.operation} operation has no bound provider interactor.`,
{ reason: 'provider-runtime-interactor-missing', deviceId: params.device.id },
);
};
}
+13 -16
View File
@@ -1,5 +1,10 @@
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import {
localInteractorSource,
providerInteractorSource,
type LocalInteractorOperationResolver,
type ProviderInteractorOperationResolver,
} from './interactor-operation-binding.ts';
import type { Interactor, RunnerContext, ScreenshotOptions } from './interactor-types.ts';
import type { RuntimeOperationFact } from './platform-runtime.ts';
@@ -59,13 +64,10 @@ export function bindLocalScreenshotInteractor(
params: Readonly<{
device: DeviceInfo;
signal: AbortSignal;
resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise<Interactor>;
resolveInteractor: LocalInteractorOperationResolver;
}>,
): ScreenshotRuntimeOperations {
return bindScreenshotCapture(
params.signal,
async (runner) => await params.resolveInteractor(params.device, runner),
);
return bindScreenshotCapture(params.signal, localInteractorSource(params));
}
/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */
@@ -73,16 +75,11 @@ export function bindProviderScreenshotInteractor(
params: Readonly<{
device: DeviceInfo;
signal: AbortSignal;
resolveInteractor: (runner: RunnerContext) => Interactor | undefined;
resolveInteractor: ProviderInteractorOperationResolver;
}>,
): ScreenshotRuntimeOperations {
return bindScreenshotCapture(params.signal, async (runner) => {
const interactor = params.resolveInteractor(runner);
if (interactor) return interactor;
throw new AppError(
'UNSUPPORTED_OPERATION',
'Provider-owned screenshot operation has no bound provider interactor.',
{ reason: 'provider-runtime-interactor-missing', deviceId: params.device.id },
);
});
return bindScreenshotCapture(
params.signal,
providerInteractorSource({ ...params, operation: 'screenshot' }),
);
}
+10 -19
View File
@@ -1,5 +1,10 @@
import type { DeviceInfo } from '@agent-device/kernel/device';
import { AppError } from '@agent-device/kernel/errors';
import {
localInteractorSource,
providerInteractorSource,
type LocalInteractorOperationResolver,
type ProviderInteractorOperationResolver,
} from './interactor-operation-binding.ts';
import type { Interactor, RunnerContext, TypeTextBackendResult } from './interactor-types.ts';
import type { RuntimeOperationFact } from './platform-runtime.ts';
import type { SessionSurface } from './session-surface.ts';
@@ -60,10 +65,7 @@ function bindTypeText(
});
}
export type LocalTypeTextInteractorResolver = (
device: DeviceInfo,
runner: RunnerContext,
) => Promise<Interactor>;
export type LocalTypeTextInteractorResolver = LocalInteractorOperationResolver;
export function bindLocalTypeTextInteractor(
params: Readonly<{
@@ -72,13 +74,10 @@ export function bindLocalTypeTextInteractor(
resolveInteractor: LocalTypeTextInteractorResolver;
}>,
): TypeTextRuntimeOperations {
return bindTypeText(
params.signal,
async (runner) => await params.resolveInteractor(params.device, runner),
);
return bindTypeText(params.signal, localInteractorSource(params));
}
export type ProviderTypeTextInteractorResolver = (runner: RunnerContext) => Interactor | undefined;
export type ProviderTypeTextInteractorResolver = ProviderInteractorOperationResolver;
/** Provider bindings fail closed when their exact owner no longer exposes its interactor. */
export function bindProviderTypeTextInteractor(
@@ -88,13 +87,5 @@ export function bindProviderTypeTextInteractor(
resolveInteractor: ProviderTypeTextInteractorResolver;
}>,
): TypeTextRuntimeOperations {
return bindTypeText(params.signal, async (runner) => {
const interactor = params.resolveInteractor(runner);
if (interactor) return interactor;
throw new AppError(
'UNSUPPORTED_OPERATION',
'Provider-owned type operation has no bound provider interactor.',
{ reason: 'provider-runtime-interactor-missing', deviceId: params.device.id },
);
});
return bindTypeText(params.signal, providerInteractorSource({ ...params, operation: 'type' }));
}
@@ -105,6 +105,12 @@ test.each([
expect(facts.operations.setViewport).toMatchObject({ available: false });
expect(binding.operations.setViewport).toBeUndefined();
expect(facts.operations.captureScreenshot).toEqual({ available: true });
// Interaction cells (R40/R41): every real Android kind drives touch and text through adb;
// only the synthetic `simulator` row (covered below) lacks a device behind it.
expect(facts.operations.focusPoint).toEqual({ available: true });
expect(facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
expect(binding.operations.captureScreenshot).toBeTypeOf('function');
expect(binding.operations.captureSnapshot).toBeTypeOf('function');
expect(binding.operations.readTextAtPoint).toBeTypeOf('function');
@@ -276,6 +282,15 @@ test.each([
expect(facts.operations.bootTargetHeadless.available).toBe(runtimeDevice.kind === 'emulator');
expect(facts.operations.captureSnapshot.available).toBe(runtimeDevice.kind !== 'simulator');
expect(facts.operations.readTextAtPoint.available).toBe(runtimeDevice.kind !== 'simulator');
// R40/R41: the synthetic simulator row is the one Android cell with no adb touch or text.
expect(facts.operations.focusPoint.available).toBe(runtimeDevice.kind !== 'simulator');
expect(facts.operations.typeText.available).toBe(runtimeDevice.kind !== 'simulator');
expect(binding.operations.focusPoint).toBeTypeOf(
runtimeDevice.kind === 'simulator' ? 'undefined' : 'function',
);
expect(binding.operations.typeText).toBeTypeOf(
runtimeDevice.kind === 'simulator' ? 'undefined' : 'function',
);
expect(binding.operations.captureSnapshot).toBeTypeOf(
runtimeDevice.kind === 'simulator' ? 'undefined' : 'function',
);
@@ -72,6 +72,13 @@ test.each([
expect(facts.operations.listApps.available).toBe(
device.appleOs !== 'watchos' && device.iosPhysicalDeviceBackend !== 'xctest',
);
// R40/R41: touch and text ride the Apple interactor, which exists for the simulator and
// physical device kinds — every leaf in this table is one of those two, so both cells are
// available across it (parity with the retired buckets).
expect(facts.operations.focusPoint).toEqual({ available: true });
expect(facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) {
const fact = facts.operations[operation];
expect(fact.available).toBe(available);
@@ -76,6 +76,11 @@ test.each([
// legacy dispatch already did once its Apple-runner fall-through failed.
expect(facts.operations.readTextAtPoint).toMatchObject({ available: false });
expect(binding.operations.readTextAtPoint).toBeUndefined();
// R40/R41: hdc drives touch and text on both real kinds this table enumerates.
expect(facts.operations.focusPoint).toEqual({ available: true });
expect(facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
await expect(binding.operations.ensureReady?.({})).resolves.toMatchObject({ booted: true });
await expect(
binding.operations.listApps?.({ device: runtimeDevice, filter: 'all' }),
@@ -110,6 +110,15 @@ test.each([
);
expect(binding.facts.operations.setViewport).toMatchObject({ available: false });
expect(binding.operations.setViewport).toBeUndefined();
// R40/R41: the desktop is the only Linux cell with a pointer and keyboard to drive.
expect(binding.facts.operations.focusPoint.available).toBe(device.kind === 'device');
expect(binding.facts.operations.typeText.available).toBe(device.kind === 'device');
expect(binding.operations.focusPoint).toBeTypeOf(
device.kind === 'device' ? 'function' : 'undefined',
);
expect(binding.operations.typeText).toBeTypeOf(
device.kind === 'device' ? 'function' : 'undefined',
);
expect(binding.facts.operations.captureScreenshot.available).toBe(device.kind === 'device');
expect(binding.operations.captureScreenshot).toBeTypeOf(
device.kind === 'device' ? 'function' : 'undefined',
@@ -135,6 +135,19 @@ test.each([
reason: 'unsupported-platform-leaf',
});
expect(binding.operations.captureSnapshot).toBeUndefined();
// R40/R41: Vega exposes remote navigation only; touch and text refuse with the owner hint.
expect(binding.facts.operations.focusPoint).toMatchObject({
available: false,
reason: 'unsupported-platform-leaf',
hint: expect.stringContaining('remote navigation only'),
});
expect(binding.facts.operations.typeText).toMatchObject({
available: false,
reason: 'unsupported-platform-leaf',
hint: expect.stringContaining('remote navigation only'),
});
expect(binding.operations.focusPoint).toBeUndefined();
expect(binding.operations.typeText).toBeUndefined();
expect(binding.facts.operations.readTextAtPoint.available).toBe(false);
expect(binding.operations.readTextAtPoint).toBeUndefined();
expect(binding.facts.operations.setViewport).toMatchObject({ available: false });
@@ -58,6 +58,11 @@ test('preserves a narrow web provider dump including empty successful entries',
// legacy `read` dispatch had no web arm at all and threw on every call before falling back.
expect(binding.facts.operations.readTextAtPoint.available).toBe(false);
expect(binding.operations.readTextAtPoint).toBeUndefined();
// R40/R41: the browser device drives touch and text through the one web interactor.
expect(binding.facts.operations.focusPoint).toEqual({ available: true });
expect(binding.facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
expect(binding.operations.captureSnapshot).toBeTypeOf('function');
expectLifecycleFacts(binding);
});
@@ -145,6 +150,10 @@ test.each([
expect(binding.operations.setViewport).toBeUndefined();
expect(binding.facts.operations.captureScreenshot.available).toBe(false);
expect(binding.operations.captureScreenshot).toBeUndefined();
expect(binding.facts.operations.focusPoint.available).toBe(false);
expect(binding.operations.focusPoint).toBeUndefined();
expect(binding.facts.operations.typeText.available).toBe(false);
expect(binding.operations.typeText).toBeUndefined();
expect(binding.operations.captureSnapshot).toBeUndefined();
},
);
@@ -283,6 +283,11 @@ test.each([
expect(binding.operations.setViewport).toBeUndefined();
expect(binding.facts.operations.captureScreenshot).toEqual({ available: true });
expect(binding.operations.captureScreenshot).toBeTypeOf('function');
// R40/R41: interaction cells ride the same provider interactor the captures do.
expect(binding.facts.operations.focusPoint).toEqual({ available: true });
expect(binding.facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
expect(binding.operations.captureSnapshot).toBeTypeOf('function');
// Limrun owns the device remotely and exposes no local point-read tool, so the live read is
// unavailable and `get` answers from the captured tree — never by borrowing the local runtime.
@@ -349,6 +354,8 @@ test('fails closed for a stale Android identity before exposing facts or binding
expect(facts.operations.captureSnapshotWithoutActiveApp).toMatchObject({ available: false });
expect(facts.operations.setViewport).toMatchObject({ available: false });
expect(facts.operations.captureScreenshot).toMatchObject({ available: false });
expect(facts.operations.focusPoint).toMatchObject({ available: false });
expect(facts.operations.typeText).toMatchObject({ available: false });
await expect(
owner.bind({ device: staleDevice, intent: { kind: 'ordinary' }, scope }),
).rejects.toMatchObject({
@@ -1,6 +1,10 @@
import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device';
import type { AppsFilter, ProviderPortReverseOptions } from '@agent-device/contracts/device';
import type { Interactor, RunnerContext } from '@agent-device/contracts/interaction';
import {
bindLimrunInteractionOperations,
limrunInteractionOperationFacts,
} from './interaction-operations.ts';
import { AppError } from '@agent-device/kernel/errors';
import { parseLimrunDeviceId } from './device.ts';
import type {
@@ -21,17 +25,11 @@ import {
import {
applicationLifecycleOperationFacts,
availableApplicationLifecycleOperations,
bindProviderFocusInteractor,
bindProviderScreenshotInteractor,
bindProviderTypeTextInteractor,
bindProviderSnapshotInteractor,
createUnavailablePlatformRuntimeFacts,
providerRuntimeOwner,
sameRuntimeOwner,
screenshotRuntimeOperationFacts,
elementTextRuntimeOperationFacts,
focusRuntimeOperationFacts,
typeTextRuntimeOperationFacts,
selectorObservationRuntimeOperationFacts,
snapshotRuntimeOperationFacts,
viewportRuntimeOperationFacts,
@@ -370,26 +368,7 @@ function bindLimrunAppLogs(
}),
runtimeFacts.operations,
),
...bindProviderSnapshotInteractor({
device,
signal,
resolveInteractor: (runner) => options.getInteractor(device, runner),
}),
...bindProviderFocusInteractor({
device,
signal,
resolveInteractor: (runner) => options.getInteractor(device, runner),
}),
...bindProviderTypeTextInteractor({
device,
signal,
resolveInteractor: (runner) => options.getInteractor(device, runner),
}),
...bindProviderScreenshotInteractor({
device,
signal,
resolveInteractor: (runner) => options.getInteractor(device, runner),
}),
...bindLimrunInteractionOperations({ device, signal, getInteractor: options.getInteractor }),
...createLimrunAppDeploymentOperations(
deploymentOptions(options),
device,
@@ -479,8 +458,7 @@ function facts(
...viewportRuntimeOperationFacts({ setViewport: viewportUnavailable }),
// Focus rides the same provider interactor the captures do, and a live-session Limrun
// device always has one, so it is available wherever a capture is.
...focusRuntimeOperationFacts({ focus: available }),
...typeTextRuntimeOperationFacts({ type: available }),
...limrunInteractionOperationFacts(),
...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }),
ensureReady: available,
bootTarget: available,
@@ -526,8 +504,7 @@ function recoveryFacts(
findSelector: liveSessionUnavailable,
}),
...viewportRuntimeOperationFacts({ setViewport: liveSessionUnavailable }),
...focusRuntimeOperationFacts({ focus: liveSessionUnavailable }),
...typeTextRuntimeOperationFacts({ type: liveSessionUnavailable }),
...limrunInteractionOperationFacts(liveSessionUnavailable),
ensureReady: liveSessionUnavailable,
bootTarget: liveSessionUnavailable,
bootTargetHeadless: liveSessionUnavailable,
@@ -0,0 +1,47 @@
import {
bindProviderFocusInteractor,
bindProviderScreenshotInteractor,
bindProviderSnapshotInteractor,
bindProviderTypeTextInteractor,
focusRuntimeOperationFacts,
typeTextRuntimeOperationFacts,
} from '@agent-device/contracts/platform';
import type { Interactor, RunnerContext } from '@agent-device/contracts/interaction';
import type { RuntimeOperationUnavailability } from '@agent-device/contracts/platform';
import type { DeviceInfo } from '@agent-device/kernel/device';
const available = Object.freeze({ available: true } as const);
/**
* The interactor-backed interaction cells a live Limrun session serves: everything here rides
* one provider interactor, and a live session always has one, so the cells are available
* together. Extracted from the app-log owner because text/point interaction is not app-log
* behavior — the owner module composes this, it does not define it.
*/
export function limrunInteractionOperationFacts(
liveSessionUnavailable?: RuntimeOperationUnavailability,
) {
const cell = liveSessionUnavailable ?? available;
return Object.freeze({
...focusRuntimeOperationFacts({ focus: cell }),
...typeTextRuntimeOperationFacts({ type: cell }),
});
}
/** Binds the interactor-backed operations (snapshot, screenshot, focus, type) for one session. */
export function bindLimrunInteractionOperations(
params: Readonly<{
device: DeviceInfo;
signal: AbortSignal;
getInteractor(device: DeviceInfo, runner?: RunnerContext): Interactor | undefined;
}>,
) {
const { device, signal } = params;
const resolveInteractor = (runner: RunnerContext) => params.getInteractor(device, runner);
return Object.freeze({
...bindProviderSnapshotInteractor({ device, signal, resolveInteractor }),
...bindProviderFocusInteractor({ device, signal, resolveInteractor }),
...bindProviderTypeTextInteractor({ device, signal, resolveInteractor }),
...bindProviderScreenshotInteractor({ device, signal, resolveInteractor }),
});
}
@@ -239,6 +239,11 @@ test('captures through only the active exact WebDriver interactor', async () =>
expect(binding.operations.setViewport).toBeUndefined();
expect(binding.facts.operations.captureScreenshot).toEqual({ available: true });
expect(binding.operations.captureScreenshot).toBeTypeOf('function');
// R40/R41: interaction cells share the captures' reachability gate and interactor.
expect(binding.facts.operations.focusPoint).toEqual({ available: true });
expect(binding.facts.operations.typeText).toEqual({ available: true });
expect(binding.operations.focusPoint).toBeTypeOf('function');
expect(binding.operations.typeText).toBeTypeOf('function');
// Provider ownership is authoritative and fails closed: a WebDriver owner's transport carries
// no local point-read tool, so it advertises none and never borrows the local family read.
expect(binding.facts.operations.readTextAtPoint).toMatchObject({
@@ -280,6 +285,11 @@ test.each([
expect(facts.operations.captureSnapshotWithoutActiveApp.available).toBe(false);
expect(facts.operations.setViewport.available).toBe(false);
expect(facts.operations.captureScreenshot.available).toBe(false);
// R40/R41: interaction cells gate on interactor reachability, not on the capture
// capability declarations — a provider that can drive its interactor can touch and type
// even when it declares no snapshot/screenshot. Only a dead session closes them.
expect(facts.operations.focusPoint.available).toBe(state.isSessionActive());
expect(facts.operations.typeText.available).toBe(state.isSessionActive());
expect(facts.operations.readTextAtPoint.available).toBe(false);
if (state.isSessionActive()) {
const binding = await owner.bind({
@@ -30,12 +30,6 @@ import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts';
type InteractionRuntimeParams = InteractionHandlerParams & {
captureSnapshotForSession: CaptureSnapshotForSession;
pairedGestureViewport?: Rect;
/**
* R41: `type` executes only through its admitted request-bound runtime. The member exists
* exactly when the `type` handler admitted and bound one — no caller can fall back to legacy
* dispatch, and every other interaction command simply has no text-entry backend.
*/
boundTypeText?: (text: string) => Promise<Record<string, unknown> | void>;
};
export function createInteractionRuntime(params: InteractionRuntimeParams) {
@@ -204,12 +198,6 @@ function createInteractionBackend(
),
);
},
typeText: params.boundTypeText
? async (_context, text): Promise<BackendActionResult> => {
expireRefFrame(session);
return toBackendActionResult(await params.boundTypeText?.(text));
}
: undefined,
};
}
+12 -26
View File
@@ -4,12 +4,11 @@ import { handleTouchInteractionCommands } from './interaction-touch.ts';
import { captureSnapshotForSession } from './interaction-snapshot.ts';
import { refSnapshotFlagGuardResponse } from './interaction-flags.ts';
import { dispatchGetViaRuntime, dispatchIsViaRuntime } from '../selector-runtime.ts';
import { createInteractionRuntime } from './interaction-runtime.ts';
import { finalizeTouchInteraction } from './interaction-common.ts';
import { expireRefFrame } from '../ref-frame.ts';
import { errorResponse, noActiveSessionError } from './response.ts';
import { PUBLIC_COMMANDS } from '../../command-catalog.ts';
import { normalizeError } from '@agent-device/kernel/errors';
import { successText } from '../../utils/success-text.ts';
import {
ensureAndroidBlockingSystemDialogReady,
recoverAndroidBlockingSystemDialog,
@@ -102,23 +101,12 @@ async function recoverAndroidRecordingDialogForType(
}
async function runTypeTextViaRuntime(
params: InteractionHandlerParams & {
captureSnapshotForSession: typeof captureSnapshotForSession;
},
params: InteractionHandlerParams,
session: SessionState,
boundTypeText: BoundTypeTextExecutor,
recordingRecoveryWarning?: string,
): Promise<DaemonResponse> {
const { req, sessionName, sessionStore } = params;
const text = (req.positionals ?? []).join(' ');
const runtime = createInteractionRuntime({
...params,
boundTypeText: async (typedText) =>
await boundTypeText(
[typedText],
params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
),
});
const { req, sessionStore } = params;
const actionStartedAt = Date.now();
try {
const readiness = await ensureAndroidBlockingSystemDialogReady({
@@ -126,23 +114,21 @@ async function runTypeTextViaRuntime(
command: req.command,
phase: 'before-command',
});
const result = await runtime.interactions.typeText(text, {
session: sessionName,
requestId: req.meta?.requestId,
delayMs: req.flags?.delayMs,
});
// ADR 0014 side-effect seam: the entry mutates the focused field; expire the frame before
// executing so a later step cannot reuse it. R41: the bound executor already validates and
// composes the retired leaf's exact result, so nothing here re-validates or re-formats it.
expireRefFrame(session);
const result = await boundTypeText(
req.positionals ?? [],
params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
);
await ensureAndroidBlockingSystemDialogReady({
session,
command: req.command,
phase: 'after-command',
});
const actionFinishedAt = Date.now();
const responseData: Record<string, unknown> = {
...(result.backendResult ?? {}),
text: result.text,
delayMs: result.delayMs,
...successText(result.message ?? `Typed ${Array.from(result.text).length} chars`),
};
const responseData: Record<string, unknown> = { ...result };
appendTypeReadinessWarnings(responseData, recordingRecoveryWarning, readiness);
return finalizeTouchInteraction({
session,
@@ -17,8 +17,8 @@ focus 100 100
# The session survives the focus: a crashed desktop would fail here, not silently pass above.
is exists "appname=gnome-calculator || windowtitle=Calculator || label=Calculator || label=0 || label=1 || label=5"
# R41 (#1739): `type` executes through the bound `typeText` operation rather than the retired
# interactor leaf. gnome-calculator accepts keyboard digits, so this types through the same
# ydotool primitive the contract test pins — and the session-survival assertion below fails
# loudly if the desktop crashed instead of silently passing.
type "5"
is exists "appname=gnome-calculator || windowtitle=Calculator || label=Calculator || label=0 || label=1 || label=5"
# interactor leaf. The digits are chosen so the assertion cannot be satisfied by pre-existing
# UI: no calculator button is labelled 555, so "555" exists in the tree only if the typed
# keystrokes actually landed in the display. Deleting the type step turns the wait red.
type "555"
wait "label=555 || text=555 || value=555" 10000