test(android-e2e): record rotation state and logcat rotation decisions on a failed step (#2350)

* test(android-e2e): record rotation state and logcat rotation decisions on a failed step

The Android smoke has failed on the post-alert canary since 2026-09-03,
and the failed-step screenshot from run 34021894996 shows why the reads
miss: the device is in landscape at that point, with the canary below
the fold, although `orientation portrait` had taken effect (the fixture
confirmed it and every tap before the alert landed at x=540). Nothing we
keep says what rotated it. A failed step now also writes
failed-step-N-device.txt with the two rotation settings, the display's
rotation lines, and WindowManager's rotation decisions from logcat, read
through adb so they stand even when the CLI path failed.

* test(android-e2e): keep the rotation evidence to WindowManager decisions and display rotation fields

* test(e2e): own failed-step evidence in one collector, bound the device probes, test it

Review follow-up on the rotation evidence. The collectors move out of
the harness closure into failed-step-evidence.ts (fallow complexity),
where the platform hook runs alongside the screenshot and snapshot and
is bounded as a group (15s) so it can never delay them; a hook that
throws, answers nothing, or never answers records nothing for the device
file and leaves the CLI evidence in place. The Android probes get a 5s
per-command bound, and logcat lines are capped in count and length.
Deterministic tests cover the file contents, the hook failure and
timeout cases, and the harness naming every evidence file, device file
included, in failed-step.txt.

test/integration/android-emulator-e2e/live-harness.ts

@@ -1,5 +1,7 @@
+import { execFile } from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';
+import { promisify } from 'node:util';
 
 import {
   createLiveDeviceContext,
@@ -37,9 +39,73 @@ export function createContext(): LiveContext {
   };
 }
 
+const execFileAsync = promisify(execFile);
+
+const ROTATION_PROBE_TIMEOUT_MS = 5_000;
+const ROTATION_LOG_LINES = 60;
+const ROTATION_LOG_LINE_LENGTH = 240;
+
+/**
+ * What the OS says about rotation when a step fails: the two settings `orientation` writes, the
+ * display's current rotation, and the WindowManager rotation decisions logcat still holds (with
+ * the reason it gives). Read through adb, not agent-device, so it stands even when the CLI path
+ * is what failed; the shared collector bounds the whole read so it never delays the screenshot.
+ */
+async function readAndroidRotationEvidence(context: LiveContext): Promise<string> {
+  const probes: readonly [string, string[]][] = [
+    ['accelerometer_rotation', ['shell', 'settings', 'get', 'system', 'accelerometer_rotation']],
+    ['user_rotation', ['shell', 'settings', 'get', 'system', 'user_rotation']],
+    ['display rotation', ['shell', 'dumpsys', 'display']],
+    ['logcat rotation decisions', ['logcat', '-d', '-v', 'time']],
+  ];
+  const sections: string[] = [];
+  for (const [title, args] of probes) {
+    try {
+      const { stdout } = await execFileAsync('adb', ['-s', context.serial, ...args], {
+        maxBuffer: 64 * 1024 * 1024,
+        timeout: ROTATION_PROBE_TIMEOUT_MS,
+      });
+      sections.push(`## ${title}\n${selectRotationLines(title, stdout)}`);
+    } catch (error) {
+      sections.push(
+        `## ${title}\n(failed: ${error instanceof Error ? error.message : String(error)})`,
+      );
+    }
+  }
+  return `${sections.join('\n\n')}\n`;
+}
+
+function selectRotationLines(title: string, output: string): string {
+  if (title === 'display rotation') {
+    return output
+      .split('\n')
+      .filter((line) =>
+        /mCurrentOrientation|mRotation=|installOrientation|\brotation \d/.test(line),
+      )
+      .map((line) => line.trim().slice(0, ROTATION_LOG_LINE_LENGTH))
+      .slice(0, 8)
+      .join('\n');
+  }
+  if (title === 'logcat rotation decisions') {
+    return output
+      .split('\n')
+      .filter(
+        (line) =>
+          /(WindowManager|DisplayRotation|WindowOrientationListener|RotationResolver|DisplayContent|SensorService)/.test(
+            line,
+          ) && /rotat|orient/i.test(line),
+      )
+      .slice(-ROTATION_LOG_LINES)
+      .map((line) => line.slice(0, ROTATION_LOG_LINE_LENGTH))
+      .join('\n');
+  }
+  return output.trim();
+}
+
 const harness = createLiveDeviceHarness<LiveContext, AndroidEmulatorBehaviorId>({
   behaviorsForScenario: liveBehaviorsForScenario,
   commandsForScenario: liveCommandsForScenario,
+  deviceEvidence: readAndroidRotationEvidence,
   commonFlags: (context, args) => [
     ...args,
     '--platform',

test/integration/live-device-e2e-failed-step-evidence.test.ts

@@ -0,0 +1,133 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+
+import type { CliJsonResult } from './cli-json.ts';
+import { collectFailedStepEvidence } from './live-device-e2e/failed-step-evidence.ts';
+import { createLiveDeviceContext, createLiveDeviceHarness } from './live-device-e2e/runtime.ts';
+
+function fakeCli(artifacts: { screenshot?: boolean; snapshot?: boolean } = {}) {
+  const calls: string[][] = [];
+  const runCli = async (args: string[]): Promise<CliJsonResult> => {
+    calls.push(args);
+    if (args[0] === 'screenshot') {
+      if (artifacts.screenshot === false) return { status: 1, stdout: '', stderr: 'no screen' };
+      fs.writeFileSync(args[1]!, 'png');
+      return { status: 0, stdout: '', stderr: '', json: { success: true } };
+    }
+    if (args[0] === 'snapshot') {
+      if (artifacts.snapshot === false) return { status: 1, stdout: '', stderr: 'no tree' };
+      return { status: 0, stdout: '', stderr: '', json: { success: true, data: { nodes: [] } } };
+    }
+    return { status: 1, stdout: '', stderr: 'step failed', json: { success: false } };
+  };
+  return { calls, runCli };
+}
+
+function tempStem(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'failed-step-evidence-'));
+  return path.join(dir, 'failed-step-3');
+}
+
+test('device facts land in their own file next to the screenshot and snapshot', async () => {
+  const stem = tempStem();
+  const cli = fakeCli();
+
+  const evidence = await collectFailedStepEvidence({
+    stem,
+    runCli: cli.runCli,
+    deviceEvidence: async () => '## user_rotation\n1\n',
+  });
+
+  assert.deepEqual(evidence, {
+    screenshotPath: `${stem}.png`,
+    snapshotPath: `${stem}-snapshot.json`,
+    devicePath: `${stem}-device.txt`,
+  });
+  assert.equal(fs.readFileSync(`${stem}-device.txt`, 'utf8'), '## user_rotation\n1\n');
+  assert.deepEqual(JSON.parse(fs.readFileSync(`${stem}-snapshot.json`, 'utf8')), {
+    success: true,
+    data: { nodes: [] },
+  });
+  assert.deepEqual(
+    cli.calls.map((args) => args[0]),
+    ['screenshot', 'snapshot'],
+  );
+});
+
+test('a device hook that throws or returns nothing still leaves the CLI evidence in place', async () => {
+  for (const deviceEvidence of [
+    async () => {
+      throw new Error('adb unavailable');
+    },
+    async () => undefined,
+  ]) {
+    const stem = tempStem();
+    const evidence = await collectFailedStepEvidence({
+      stem,
+      runCli: fakeCli().runCli,
+      deviceEvidence,
+    });
+
+    assert.deepEqual(evidence, {
+      screenshotPath: `${stem}.png`,
+      snapshotPath: `${stem}-snapshot.json`,
+    });
+    assert.equal(fs.existsSync(`${stem}-device.txt`), false);
+  }
+});
+
+test('a device hook that never answers is bounded and never delays the CLI evidence', async () => {
+  const stem = tempStem();
+  const startedAt = Date.now();
+
+  const evidence = await collectFailedStepEvidence({
+    stem,
+    runCli: fakeCli().runCli,
+    deviceEvidence: () => new Promise<string>(() => undefined),
+    deviceEvidenceTimeoutMs: 50,
+  });
+
+  assert.deepEqual(evidence, {
+    screenshotPath: `${stem}.png`,
+    snapshotPath: `${stem}-snapshot.json`,
+  });
+  assert.ok(Date.now() - startedAt < 1_000);
+});
+
+test('a failed step names its evidence files, including the device file, in failed-step.txt', async () => {
+  const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'failed-step-harness-'));
+  const cli = fakeCli();
+  const harness = createLiveDeviceHarness<
+    ReturnType<typeof createLiveDeviceContext<string>>,
+    string
+  >({
+    behaviorsForScenario: () => [],
+    commandsForScenario: () => [],
+    commonFlags: (_context, args) => [...args, '--json'],
+    runCli: cli.runCli,
+    deviceEvidence: async () => 'accelerometer_rotation=1\n',
+    writeCoverageReport: () => undefined,
+  });
+  const context = createLiveDeviceContext<string>({ artifactRoot, session: 'evidence' });
+
+  await assert.rejects(
+    harness.runStep(context, 'read the canary', ['get', 'text', 'id="canary"']),
+    (error: Error) => {
+      assert.match(error.message, /step: read the canary/);
+      assert.match(error.message, /device: .*failed-step-1-device\.txt/);
+      assert.match(error.message, /screenshot: .*failed-step-1\.png/);
+      assert.match(error.message, /snapshot: .*failed-step-1-snapshot\.json/);
+      return true;
+    },
+  );
+
+  const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8');
+  assert.match(report, /device: .*failed-step-1-device\.txt/);
+  assert.equal(
+    fs.readFileSync(path.join(context.artifactDir, 'failed-step-1-device.txt'), 'utf8'),
+    'accelerometer_rotation=1\n',
+  );
+});

test/integration/live-device-e2e/failed-step-evidence.ts

@@ -0,0 +1,102 @@
+import fs from 'node:fs';
+
+import type { CliJsonResult } from '../cli-json.ts';
+
+export type FailedStepEvidence = {
+  screenshotPath?: string;
+  snapshotPath?: string;
+  devicePath?: string;
+};
+
+export type FailedStepEvidenceInput = {
+  /** Artifact path prefix, e.g. `<artifactDir>/failed-step-7`. */
+  stem: string;
+  /** The CLI bound to the failed step's device and session. */
+  runCli: (args: string[]) => Promise<CliJsonResult>;
+  /** Platform-owned device facts, read outside the CLI. */
+  deviceEvidence?: () => Promise<string | undefined>;
+  /** Upper bound for the device facts as a group; the CLI evidence never waits on them. */
+  deviceEvidenceTimeoutMs?: number;
+};
+
+const DEVICE_EVIDENCE_TIMEOUT_MS = 15_000;
+
+/**
+ * What the device showed when a step failed: the pixels and the accessibility tree the next
+ * capture would have read, plus whatever the platform can say about the device outside
+ * agent-device. Every collector is best-effort and independent: a throw, a non-zero exit, or a
+ * timed-out hook records nothing for that item and nothing else.
+ */
+export async function collectFailedStepEvidence(
+  input: FailedStepEvidenceInput,
+): Promise<FailedStepEvidence> {
+  const [cli, devicePath] = await Promise.all([
+    collectCliEvidence(input),
+    collectDeviceEvidence(input),
+  ]);
+  return { ...cli, ...(devicePath ? { devicePath } : {}) };
+}
+
+async function collectCliEvidence(
+  input: FailedStepEvidenceInput,
+): Promise<Pick<FailedStepEvidence, 'screenshotPath' | 'snapshotPath'>> {
+  const screenshotPath = await captureScreenshot(input);
+  const snapshotPath = await captureSnapshot(input);
+  return {
+    ...(screenshotPath ? { screenshotPath } : {}),
+    ...(snapshotPath ? { snapshotPath } : {}),
+  };
+}
+
+async function captureScreenshot(input: FailedStepEvidenceInput): Promise<string | undefined> {
+  const screenshotPath = `${input.stem}.png`;
+  try {
+    const result = await input.runCli(['screenshot', screenshotPath]);
+    return result.status === 0 ? screenshotPath : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+async function captureSnapshot(input: FailedStepEvidenceInput): Promise<string | undefined> {
+  const snapshotPath = `${input.stem}-snapshot.json`;
+  try {
+    const result = await input.runCli(['snapshot']);
+    if (result.status !== 0 || result.json === undefined) return undefined;
+    fs.writeFileSync(snapshotPath, JSON.stringify(result.json, null, 2));
+    return snapshotPath;
+  } catch {
+    return undefined;
+  }
+}
+
+async function collectDeviceEvidence(input: FailedStepEvidenceInput): Promise<string | undefined> {
+  if (!input.deviceEvidence) return undefined;
+  const devicePath = `${input.stem}-device.txt`;
+  try {
+    const facts = await withinTimeout(
+      input.deviceEvidence(),
+      input.deviceEvidenceTimeoutMs ?? DEVICE_EVIDENCE_TIMEOUT_MS,
+    );
+    if (facts === undefined) return undefined;
+    fs.writeFileSync(devicePath, facts);
+    return devicePath;
+  } catch {
+    return undefined;
+  }
+}
+
+async function withinTimeout<T>(pending: Promise<T>, timeoutMs: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const expired = new Promise<never>((_, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`device evidence exceeded ${timeoutMs}ms`)),
+      timeoutMs,
+    );
+  });
+  try {
+    return await Promise.race([pending, expired]);
+  } finally {
+    clearTimeout(timer);
+  }
+}

test/integration/live-device-e2e/runtime.ts

@@ -3,6 +3,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 
 import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts';
+import { collectFailedStepEvidence, type FailedStepEvidence } from './failed-step-evidence.ts';
 
 export type StepRecord = {
   accepted: boolean;
@@ -49,6 +50,12 @@ type HarnessOptions<Context, BehaviorId extends string> = {
     env: NodeJS.ProcessEnv,
     options?: { timeoutMs?: number },
   ) => Promise<CliJsonResult>;
+  /**
+   * Platform-owned device facts for a failed step (rotation state, system logs), read outside
+   * agent-device so they describe the device even when the CLI path is what failed. Best-effort:
+   * a throw or undefined records nothing.
+   */
+  deviceEvidence?: (context: Context) => Promise<string | undefined>;
   writeCoverageReport: (context: Context) => void;
 };
 
@@ -176,6 +183,7 @@ export function createLiveDeviceHarness<
         `artifacts: ${context.artifactDir}`,
         `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`,
         `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`,
+        `device: ${evidence.devicePath ?? '(not collected)'}`,
       ].join('\n');
       fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message);
       assert.fail(message);
@@ -185,37 +193,14 @@ export function createLiveDeviceHarness<
     }
   }
 
-  /**
-   * What the device showed when a step failed: the pixels and the accessibility tree the
-   * next capture would have read. Best-effort, never throws; a failed capture yields undefined.
-   */
-  async function captureFailedStepEvidence(
-    context: Context,
-  ): Promise<{ screenshotPath?: string; snapshotPath?: string }> {
-    const stem = path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`);
-    const screenshotPath = `${stem}.png`;
-    const snapshotPath = `${stem}-snapshot.json`;
+  function captureFailedStepEvidence(context: Context): Promise<FailedStepEvidence> {
     const runCli = options.runCli ?? runBuiltCliJson;
-    const evidence: { screenshotPath?: string; snapshotPath?: string } = {};
-    try {
-      const screenshot = await runCli(
-        options.commonFlags(context, ['screenshot', screenshotPath]),
-        context.env,
-      );
-      if (screenshot.status === 0) evidence.screenshotPath = screenshotPath;
-    } catch {
-      // evidence only
-    }
-    try {
-      const snapshot = await runCli(options.commonFlags(context, ['snapshot']), context.env);
-      if (snapshot.status === 0 && snapshot.json !== undefined) {
-        fs.writeFileSync(snapshotPath, JSON.stringify(snapshot.json, null, 2));
-        evidence.snapshotPath = snapshotPath;
-      }
-    } catch {
-      // evidence only
-    }
-    return evidence;
+    const deviceEvidence = options.deviceEvidence;
+    return collectFailedStepEvidence({
+      stem: path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`),
+      runCli: (args) => runCli(options.commonFlags(context, args), context.env),
+      ...(deviceEvidence ? { deviceEvidence: () => deviceEvidence(context) } : {}),
+    });
   }
 
   function updateSessionState(context: Context, command: string | undefined, status: number): void {