Files
callstack__agent-device/src/utils/exec.ts
T

813 lines
24 KiB
TypeScript
Raw Normal View History

import { AsyncLocalStorage } from 'node:async_hooks';
import { constants } from 'node:fs';
import { access, stat } from 'node:fs/promises';
import path from 'node:path';
import { spawn, spawnSync, type ChildProcess, type StdioOptions } from 'node:child_process';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
refactor: move errors/redaction/device into src/kernel — Phase 5 slice 3 (#940) * refactor: move errors/redaction/device into src/kernel — Phase 5 slice 3 Relocates the foundational primitive trio from src/utils/ into the kernel/ layer (joining snapshot.ts from slice 2), per the target folder DAG in plans/perfect-shape.md §5.5. A pure path codemod, no behavior change. They form a closed cluster — device -> errors -> redaction, with redaction a leaf — so kernel/ takes no upward dependency, and every importer becomes a clean downward import toward kernel. errors.ts is the most-imported module in the tree; device.ts the §5.5-named headliner. Moving all three atomically avoids a half-state where one would import another across the utils/kernel boundary. Imports rewritten by a resolve-based codemod (compares each specifier's resolved path to the moved files, so the unrelated commands/management/device.ts and other same-named files are untouched): 483 sites across 402 files. The two platform-descriptor doc comments and the fallow health baseline key for device.ts are updated to the new path; the contracts-schema-public guard that asserts the error helpers pull no diagnostics/node: deps now reads kernel/. Verified: tsc --noEmit, oxfmt + oxlint --deny-warnings, rslib build, full vitest suite (2877 pass), fallow audit clean (411 changed files), Layering Guard empty; kernel/ files import only within kernel. * docs: update guidance references to kernel/{device,errors} after the move AGENTS.md (Apple-family sync rule + normalizeError), ADR-0009, and plans/apple-platform-consolidation.md still named the old src/utils/ paths. Point them at src/kernel/. plans/perfect-shape.md's utils/device.ts mention is left as-is — it describes the pre-move diagnosis.
2026-06-30 07:25:02 +02:00
import { AppError } from '../kernel/errors.ts';
import { emitDiagnostic, getDiagnosticsMeta, updateDiagnosticsScope } from './diagnostics.ts';
import { parseBooleanLiteral } from './source-value.ts';
2026-01-30 19:41:00 +01:00
export type ExecResult = {
stdout: string;
stderr: string;
exitCode: number;
stdoutBuffer?: Buffer;
};
export type ExecOptions = {
2026-01-30 19:41:00 +01:00
cwd?: string;
env?: NodeJS.ProcessEnv;
allowFailure?: boolean;
binaryStdout?: boolean;
2026-02-02 15:12:23 +01:00
stdin?: string | Buffer;
timeoutMs?: number;
detached?: boolean;
signal?: AbortSignal;
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
/** Max stdout/stderr bytes for synchronous runs (default Node ~1MB). */
maxBuffer?: number;
2026-01-30 19:41:00 +01:00
};
type ExecStreamOptions = ExecOptions & {
2026-01-30 19:41:00 +01:00
onStdoutChunk?: (chunk: string) => void;
onStderrChunk?: (chunk: string) => void;
onSpawn?: (child: ReturnType<typeof spawn>) => void;
2026-01-30 19:41:00 +01:00
};
2026-02-02 21:33:45 +01:00
export type ExecBackgroundResult = {
child: ReturnType<typeof spawn>;
wait: Promise<ExecResult>;
};
type ExecDetachedOptions = ExecOptions & {
stdio?: StdioOptions;
};
export type ExecDetachedExit = {
pid: number;
exitCode?: number;
signal?: NodeJS.Signals;
error?: string;
};
export type ExecDetachedProcess = {
pid: number;
exited: Promise<ExecDetachedExit>;
};
export type ExecBackgroundOptions = ExecOptions & {
/**
* Capture stdout/stderr into the wait result when the child has piped stdio.
* Set false when the caller owns, ignores, or forwards the streams.
*/
captureOutput?: boolean;
stdio?: StdioOptions;
};
const BARE_COMMAND_RE = /^[A-Za-z0-9][A-Za-z0-9._+-]*$/;
const WINDOWS_PATH_EXTENSIONS = ['.com', '.exe', '.bat', '.cmd'];
const EXEC_DIAGNOSTIC_ARG_LIMIT = 6;
export type CommandExecutorOverride = (
cmd: string,
args: string[],
options: ExecOptions,
) => Promise<ExecResult> | undefined;
const commandExecutorOverrideScope = new AsyncLocalStorage<CommandExecutorOverride | undefined>();
export async function withCommandExecutorOverride<T>(
override: CommandExecutorOverride | undefined,
fn: () => Promise<T>,
): Promise<T> {
if (!override) return await fn();
return await commandExecutorOverrideScope.run(override, fn);
}
// Used by local executors to bypass the active override for intentional host command execution.
export async function withoutCommandExecutorOverride<T>(fn: () => Promise<T>): Promise<T> {
return await commandExecutorOverrideScope.run(undefined, fn);
}
2026-01-30 19:41:00 +01:00
export async function runCmd(
cmd: string,
args: string[],
options: ExecOptions = {},
): Promise<ExecResult> {
const overrideResult = commandExecutorOverrideScope.getStore()?.(cmd, args, options);
refactor: harden exec failure wrapping end to end (#1074) * refactor: harden exec failure wrapping end to end Follow-up to #1072, closing the structural gaps that let the missing-processExitError bug class exist: - requireExecSuccess(result, message, extra?) in utils/exec.ts: guards an allowFailure result and throws the curated COMMAND_FAILED (flag set via execFailureDetails) itself. Result-side by design — tool providers and executor overrides return results without throwing, so an ExecOptions knob interpreted at spawn time would silently not fire on those paths. ~30 pure guard-and-throw sites across apple/android/web converted; sites with tolerance branches, cleanup, or exit-0 reachability keep their explicit shape. - Source-scan guard (src/__tests__/exec-wrap-guard.test.ts): fails when an AppError('COMMAND_FAILED', ...) details literal rebuilds the stdout/stderr/exitCode trio inline without the helper; intentional holdouts opt out with a documented exec-guard-allow comment. The guard immediately found three wrap sites the July audit missed (runtime-hints run-as probe, device-ready not_ready branch, perf export table) — now converted. - coerceExecResult at the provider boundaries (executor overrides, apple tool provider scope, android adb provider scope): SDK-supplied callbacks cross an unchecked boundary; coercing once there replaces the per-site String(result.stdout ?? '') defensiveness, which is removed. - normalizeError stderr excerpts now strip noise prefixes (adb:/xcrun:/ simctl:/error:) before rendering; skip/strip lists stay in the kernel deliberately (platforms cannot hook normalizeError) with a comment marking the registry escape hatch if they grow. - AppErrorDetails exported and documented in kernel/errors.ts: the magic keys (hint, processExitError, retriable, reason, diagnosticId, logPath, stdout/stderr/exitCode) now carry types and doc comments. - runIosDevicectl gains tolerateOutput; the devicectl uninstall path in app-install.ts reuses it instead of duplicating the wrap + hint logic (its failure hint now falls back to the devicectl default hint). * fix: reconcile exec hardening with the adb failure classifier Main landed the central adb failure classifier (androidAdbResultError + withAdbFailureHintProvider) while this branch was in flight. Resolution: - Android call sites keep main's androidAdbResultError form — it composes execFailureDetails with the classified adb hint, which is strictly richer than a plain requireExecSuccess conversion for adb invocations. requireExecSuccess remains the shape for non-adb tools and apple/web. - Provider result coercion folds into withAdbFailureHintProvider's enrichment pass (exec/pull/install), so the existing WeakSet memo also prevents coercer stacking. - The reverse-remove wrap in adb-executor now uses androidAdbResultError instead of hand-rolling the trio (flagged by the exec-wrap guard). - Guard-test scan split into helpers (fallow cyclomatic threshold) and the perf artifact-tail clone carries fallow-ignore markers on both platforms, matching the pre-existing marker on the apple side. - Two expectations updated for the stderr noise-prefix strip: classifier compose test and perf sampling reason ('device offline', no 'error:').
2026-07-04 12:56:50 +02:00
if (overrideResult) return coerceExecResult(await overrideResult);
return await runSpawnedCommand(cmd, args, options);
}
export async function runCmdStreaming(
cmd: string,
args: string[],
options: ExecStreamOptions = {},
): Promise<ExecResult> {
const overrideResult = commandExecutorOverrideScope.getStore()?.(cmd, args, options);
refactor: harden exec failure wrapping end to end (#1074) * refactor: harden exec failure wrapping end to end Follow-up to #1072, closing the structural gaps that let the missing-processExitError bug class exist: - requireExecSuccess(result, message, extra?) in utils/exec.ts: guards an allowFailure result and throws the curated COMMAND_FAILED (flag set via execFailureDetails) itself. Result-side by design — tool providers and executor overrides return results without throwing, so an ExecOptions knob interpreted at spawn time would silently not fire on those paths. ~30 pure guard-and-throw sites across apple/android/web converted; sites with tolerance branches, cleanup, or exit-0 reachability keep their explicit shape. - Source-scan guard (src/__tests__/exec-wrap-guard.test.ts): fails when an AppError('COMMAND_FAILED', ...) details literal rebuilds the stdout/stderr/exitCode trio inline without the helper; intentional holdouts opt out with a documented exec-guard-allow comment. The guard immediately found three wrap sites the July audit missed (runtime-hints run-as probe, device-ready not_ready branch, perf export table) — now converted. - coerceExecResult at the provider boundaries (executor overrides, apple tool provider scope, android adb provider scope): SDK-supplied callbacks cross an unchecked boundary; coercing once there replaces the per-site String(result.stdout ?? '') defensiveness, which is removed. - normalizeError stderr excerpts now strip noise prefixes (adb:/xcrun:/ simctl:/error:) before rendering; skip/strip lists stay in the kernel deliberately (platforms cannot hook normalizeError) with a comment marking the registry escape hatch if they grow. - AppErrorDetails exported and documented in kernel/errors.ts: the magic keys (hint, processExitError, retriable, reason, diagnosticId, logPath, stdout/stderr/exitCode) now carry types and doc comments. - runIosDevicectl gains tolerateOutput; the devicectl uninstall path in app-install.ts reuses it instead of duplicating the wrap + hint logic (its failure hint now falls back to the devicectl default hint). * fix: reconcile exec hardening with the adb failure classifier Main landed the central adb failure classifier (androidAdbResultError + withAdbFailureHintProvider) while this branch was in flight. Resolution: - Android call sites keep main's androidAdbResultError form — it composes execFailureDetails with the classified adb hint, which is strictly richer than a plain requireExecSuccess conversion for adb invocations. requireExecSuccess remains the shape for non-adb tools and apple/web. - Provider result coercion folds into withAdbFailureHintProvider's enrichment pass (exec/pull/install), so the existing WeakSet memo also prevents coercer stacking. - The reverse-remove wrap in adb-executor now uses androidAdbResultError instead of hand-rolling the trio (flagged by the exec-wrap guard). - Guard-test scan split into helpers (fallow cyclomatic threshold) and the perf artifact-tail clone carries fallow-ignore markers on both platforms, matching the pre-existing marker on the apple side. - Two expectations updated for the stderr noise-prefix strip: classifier compose test and perf sampling reason ('device offline', no 'error:').
2026-07-04 12:56:50 +02:00
if (overrideResult) return coerceExecResult(await overrideResult);
return await runSpawnedCommand(cmd, args, options);
}
refactor: harden exec failure wrapping end to end (#1074) * refactor: harden exec failure wrapping end to end Follow-up to #1072, closing the structural gaps that let the missing-processExitError bug class exist: - requireExecSuccess(result, message, extra?) in utils/exec.ts: guards an allowFailure result and throws the curated COMMAND_FAILED (flag set via execFailureDetails) itself. Result-side by design — tool providers and executor overrides return results without throwing, so an ExecOptions knob interpreted at spawn time would silently not fire on those paths. ~30 pure guard-and-throw sites across apple/android/web converted; sites with tolerance branches, cleanup, or exit-0 reachability keep their explicit shape. - Source-scan guard (src/__tests__/exec-wrap-guard.test.ts): fails when an AppError('COMMAND_FAILED', ...) details literal rebuilds the stdout/stderr/exitCode trio inline without the helper; intentional holdouts opt out with a documented exec-guard-allow comment. The guard immediately found three wrap sites the July audit missed (runtime-hints run-as probe, device-ready not_ready branch, perf export table) — now converted. - coerceExecResult at the provider boundaries (executor overrides, apple tool provider scope, android adb provider scope): SDK-supplied callbacks cross an unchecked boundary; coercing once there replaces the per-site String(result.stdout ?? '') defensiveness, which is removed. - normalizeError stderr excerpts now strip noise prefixes (adb:/xcrun:/ simctl:/error:) before rendering; skip/strip lists stay in the kernel deliberately (platforms cannot hook normalizeError) with a comment marking the registry escape hatch if they grow. - AppErrorDetails exported and documented in kernel/errors.ts: the magic keys (hint, processExitError, retriable, reason, diagnosticId, logPath, stdout/stderr/exitCode) now carry types and doc comments. - runIosDevicectl gains tolerateOutput; the devicectl uninstall path in app-install.ts reuses it instead of duplicating the wrap + hint logic (its failure hint now falls back to the devicectl default hint). * fix: reconcile exec hardening with the adb failure classifier Main landed the central adb failure classifier (androidAdbResultError + withAdbFailureHintProvider) while this branch was in flight. Resolution: - Android call sites keep main's androidAdbResultError form — it composes execFailureDetails with the classified adb hint, which is strictly richer than a plain requireExecSuccess conversion for adb invocations. requireExecSuccess remains the shape for non-adb tools and apple/web. - Provider result coercion folds into withAdbFailureHintProvider's enrichment pass (exec/pull/install), so the existing WeakSet memo also prevents coercer stacking. - The reverse-remove wrap in adb-executor now uses androidAdbResultError instead of hand-rolling the trio (flagged by the exec-wrap guard). - Guard-test scan split into helpers (fallow cyclomatic threshold) and the perf artifact-tail clone carries fallow-ignore markers on both platforms, matching the pre-existing marker on the apple side. - Two expectations updated for the stderr noise-prefix strip: classifier compose test and perf sampling reason ('device offline', no 'error:').
2026-07-04 12:56:50 +02:00
/**
* Normalize an exec result produced outside this module. Tool providers,
* executor overrides, and SDK-supplied adb executors are plain callbacks whose
* results cross an unchecked boundary; coercing once here lets downstream code
* trust the ExecResult types instead of re-wrapping fields defensively at every
* use. A non-number exitCode coerces to 1 — the same failure branch such a
* result already landed in at every `exitCode !== 0` guard.
*/
export function coerceExecResult<T extends Pick<ExecResult, 'stdout' | 'stderr' | 'exitCode'>>(
result: T,
): T {
const stdout = typeof result.stdout === 'string' ? result.stdout : String(result.stdout ?? '');
const stderr = typeof result.stderr === 'string' ? result.stderr : String(result.stderr ?? '');
const exitCode = typeof result.exitCode === 'number' ? result.exitCode : 1;
if (stdout === result.stdout && stderr === result.stderr && exitCode === result.exitCode) {
return result;
}
return { ...result, stdout, stderr, exitCode };
}
function runSpawnedCommand(
cmd: string,
args: string[],
options: ExecStreamOptions = {},
2026-01-30 19:41:00 +01:00
): Promise<ExecResult> {
const executable = normalizeExecutableCommand(cmd);
const execTrace = createExecTraceContext();
2026-01-30 19:41:00 +01:00
return new Promise((resolve, reject) => {
const child = spawn(executable, args, {
2026-01-30 19:41:00 +01:00
cwd: options.cwd,
env: options.env,
2026-02-02 15:12:23 +01:00
stdio: ['pipe', 'pipe', 'pipe'],
detached: options.detached,
windowsHide: true,
shell: false,
2026-01-30 19:41:00 +01:00
});
options.onSpawn?.(child);
2026-01-30 19:41:00 +01:00
let stdout = '';
const stdoutChunks: Buffer[] | undefined = options.binaryStdout ? [] : undefined;
2026-01-30 19:41:00 +01:00
let stderr = '';
let didTimeout = false;
const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
const timeoutHandle = timeoutMs
? setTimeout(() => {
2026-03-17 18:18:53 +01:00
didTimeout = true;
killProcessTree(child, options.detached);
2026-03-17 18:18:53 +01:00
}, timeoutMs)
: null;
const abort = watchCommandAbort(child, options);
2026-01-30 19:41:00 +01:00
if (!options.binaryStdout) child.stdout.setEncoding('utf8');
child.stderr.setEncoding('utf8');
void writeChildStdin(child, options.stdin).catch((err: unknown) => {
if (abort.didAbort || didTimeout) return;
if (isEpipeError(err)) return;
reject(createStdinError(executable, cmd, args, err));
killProcessTree(child, options.detached);
});
2026-02-02 15:12:23 +01:00
2026-01-30 19:41:00 +01:00
child.stdout.on('data', (chunk) => {
if (options.binaryStdout) {
stdoutChunks?.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
return;
2026-01-30 19:41:00 +01:00
}
const text = String(chunk);
stdout += text;
options.onStdoutChunk?.(text);
2026-01-30 19:41:00 +01:00
});
child.stderr.on('data', (chunk) => {
const text = String(chunk);
stderr += text;
options.onStderrChunk?.(text);
2026-01-30 19:41:00 +01:00
});
child.on('error', (err) => {
if (timeoutHandle) clearTimeout(timeoutHandle);
abort.dispose();
execTrace.emitForegroundCompletion(cmd, args);
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
reject(spawnRejectionError(abort, executable, cmd, args, err));
2026-01-30 19:41:00 +01:00
});
child.on('close', (code) => {
if (timeoutHandle) clearTimeout(timeoutHandle);
abort.dispose();
execTrace.emitForegroundCompletion(cmd, args);
2026-01-30 19:41:00 +01:00
const exitCode = code ?? 1;
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
if (!abort.didAbort && didTimeout && timeoutMs) {
reject(createTimeoutError(executable, cmd, args, timeoutMs, exitCode, stdout, stderr));
return;
}
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
const failure = commandCloseFailure(
abort,
executable,
cmd,
args,
exitCode,
options.allowFailure,
stdout,
stderr,
);
if (failure) {
reject(failure);
2026-01-30 19:41:00 +01:00
return;
}
resolve({
stdout,
stderr,
exitCode,
stdoutBuffer: stdoutChunks ? Buffer.concat(stdoutChunks) : undefined,
});
2026-01-30 19:41:00 +01:00
});
});
}
export async function whichCmd(cmd: string): Promise<boolean> {
const candidate = normalizeExecutableLookup(cmd);
if (!candidate) return false;
if (path.isAbsolute(candidate)) {
return isExecutablePath(candidate);
}
const pathValue = process.env.PATH;
if (!pathValue) return false;
const pathExtensions = resolvePathExtensions();
for (const directory of pathValue.split(path.delimiter)) {
const trimmedDirectory = directory.trim();
if (!trimmedDirectory) continue;
for (const entry of resolveExecutableCandidates(candidate, pathExtensions)) {
if (await isExecutablePath(path.join(trimmedDirectory, entry))) {
return true;
}
}
2026-01-30 19:41:00 +01:00
}
return false;
2026-01-30 19:41:00 +01:00
}
export async function resolveExecutableOverridePath(
rawPath: string | undefined,
envName: string,
): Promise<string | undefined> {
const candidate = normalizeOverridePath(rawPath, envName, 'executable');
if (!candidate) return undefined;
if (!(await isExecutablePath(candidate))) {
throw new AppError(
'TOOL_MISSING',
`${envName} points to a missing or non-executable file: ${candidate}`,
{ envName, path: candidate },
);
}
return candidate;
}
export async function resolveFileOverridePath(
rawPath: string | undefined,
envName: string,
): Promise<string | undefined> {
const candidate = normalizeOverridePath(rawPath, envName, 'file');
if (!candidate) return undefined;
if (!(await isFilePath(candidate))) {
throw new AppError(
'TOOL_MISSING',
`${envName} points to a missing or non-file path: ${candidate}`,
{ envName, path: candidate },
);
}
return candidate;
}
2026-01-30 19:41:00 +01:00
export function runCmdSync(cmd: string, args: string[], options: ExecOptions = {}): ExecResult {
const executable = normalizeExecutableCommand(cmd);
const result = spawnSync(executable, args, {
2026-01-30 19:41:00 +01:00
cwd: options.cwd,
env: options.env,
2026-02-02 15:12:23 +01:00
stdio: ['pipe', 'pipe', 'pipe'],
2026-01-30 19:41:00 +01:00
encoding: options.binaryStdout ? undefined : 'utf8',
2026-02-02 15:12:23 +01:00
input: options.stdin,
timeout: normalizeTimeoutMs(options.timeoutMs),
windowsHide: true,
shell: false,
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
...(options.maxBuffer !== undefined ? { maxBuffer: options.maxBuffer } : {}),
2026-01-30 19:41:00 +01:00
});
if (result.error) {
const code = (result.error as NodeJS.ErrnoException).code;
if (code === 'ETIMEDOUT') {
2026-03-17 18:18:53 +01:00
throw new AppError(
'COMMAND_FAILED',
`${executable} timed out after ${normalizeTimeoutMs(options.timeoutMs)}ms`,
2026-03-17 18:18:53 +01:00
{
cmd,
args,
timeoutMs: normalizeTimeoutMs(options.timeoutMs),
},
result.error,
);
}
2026-01-30 19:41:00 +01:00
if (code === 'ENOENT') {
throw createMissingToolError(executable, cmd, result.error);
2026-01-30 19:41:00 +01:00
}
throw createCommandFailedError(executable, cmd, args, result.error);
2026-01-30 19:41:00 +01:00
}
const stdoutBuffer = options.binaryStdout
? Buffer.isBuffer(result.stdout)
? result.stdout
: Buffer.from(result.stdout ?? '')
: undefined;
const stdout = options.binaryStdout
? ''
: typeof result.stdout === 'string'
? result.stdout
: (result.stdout ?? '').toString();
const stderr =
typeof result.stderr === 'string' ? result.stderr : (result.stderr ?? '').toString();
const exitCode = result.status ?? 1;
if (exitCode !== 0 && !options.allowFailure) {
throw createExitError(executable, cmd, args, exitCode, stdout, stderr);
2026-01-30 19:41:00 +01:00
}
return { stdout, stderr, exitCode, stdoutBuffer };
}
export function runCmdDetached(
cmd: string,
args: string[],
options: ExecDetachedOptions = {},
): number {
return runCmdDetachedMonitored(cmd, args, options).pid;
}
export function runCmdDetachedMonitored(
cmd: string,
args: string[],
options: ExecDetachedOptions = {},
): ExecDetachedProcess {
const executable = normalizeExecutableCommand(cmd);
const child = spawn(executable, args, {
2026-01-30 19:41:00 +01:00
cwd: options.cwd,
env: options.env,
stdio: options.stdio ?? 'ignore',
2026-01-30 19:41:00 +01:00
detached: true,
windowsHide: true,
shell: false,
2026-01-30 19:41:00 +01:00
});
const pid = child.pid ?? 0;
const exited = new Promise<ExecDetachedExit>((resolve) => {
child.once('error', (err) => {
resolve({ pid, error: err.message });
});
child.once('exit', (code, signal) => {
resolve({
pid,
...(typeof code === 'number' ? { exitCode: code } : {}),
...(signal ? { signal } : {}),
});
});
});
2026-01-30 19:41:00 +01:00
child.unref();
return { pid, exited };
2026-01-30 19:41:00 +01:00
}
2026-02-02 21:33:45 +01:00
export function runCmdBackground(
cmd: string,
args: string[],
options: ExecBackgroundOptions = {},
2026-02-02 21:33:45 +01:00
): ExecBackgroundResult {
const executable = normalizeExecutableCommand(cmd);
const execTrace = createExecTraceContext();
const child = spawn(executable, args, {
2026-02-02 21:33:45 +01:00
cwd: options.cwd,
env: options.env,
stdio: options.stdio ?? ['ignore', 'pipe', 'pipe'],
detached: options.detached,
windowsHide: true,
shell: false,
2026-02-02 21:33:45 +01:00
});
execTrace.emitBackgroundSpawn(cmd, args);
2026-02-02 21:33:45 +01:00
let stdout = '';
let stderr = '';
const captureOutput = options.captureOutput ?? true;
const abort = watchCommandAbort(child, options);
2026-02-02 21:33:45 +01:00
if (captureOutput) {
child.stdout?.setEncoding('utf8');
child.stderr?.setEncoding('utf8');
2026-02-02 21:33:45 +01:00
child.stdout?.on('data', (chunk) => {
stdout += chunk;
});
child.stderr?.on('data', (chunk) => {
stderr += chunk;
});
}
2026-02-02 21:33:45 +01:00
const wait = new Promise<ExecResult>((resolve, reject) => {
child.on('error', (err) => {
abort.dispose();
execTrace.emitBackgroundCompletion(cmd, args, 'error');
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
reject(spawnRejectionError(abort, executable, cmd, args, err));
2026-02-02 21:33:45 +01:00
});
child.on('close', (code) => {
abort.dispose();
execTrace.emitBackgroundCompletion(cmd, args, 'exit');
2026-02-02 21:33:45 +01:00
const exitCode = code ?? 1;
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
const failure = commandCloseFailure(
abort,
executable,
cmd,
args,
exitCode,
options.allowFailure,
stdout,
stderr,
);
if (failure) {
reject(failure);
2026-02-02 21:33:45 +01:00
return;
}
resolve({ stdout, stderr, exitCode });
});
});
return { child, wait };
}
type ExecTraceContext = {
emitBackgroundCompletion: (cmd: string, args: string[], event: 'error' | 'exit') => void;
emitBackgroundSpawn: (cmd: string, args: string[]) => void;
emitForegroundCompletion: (cmd: string, args: string[]) => void;
};
function createExecTraceContext(): ExecTraceContext {
const diagnosticsMeta = getDiagnosticsMeta();
const diagnosticsDebugEnabled = diagnosticsMeta.debug === true;
const envTraceEnabled = parseBooleanLiteral(process.env.AGENT_DEVICE_EXEC_TRACE ?? '') === true;
if (!diagnosticsDebugEnabled && !envTraceEnabled) {
return createDisabledExecTraceContext();
}
if (envTraceEnabled && diagnosticsMeta.flushOnSuccess !== true) {
updateDiagnosticsScope({ flushOnSuccess: true });
}
const startedAtMs = Date.now();
let completionEmitted = false;
return {
emitForegroundCompletion: (cmd, args) => {
if (completionEmitted) return;
completionEmitted = true;
emitExecCommandDiagnostic({
cmd,
args,
startedAtMs,
});
},
emitBackgroundSpawn: (cmd, args) => {
emitExecCommandDiagnostic({
cmd,
args,
data: { event: 'spawn' },
});
},
emitBackgroundCompletion: (cmd, args, event) => {
if (completionEmitted) return;
completionEmitted = true;
emitExecCommandDiagnostic({
cmd,
args,
startedAtMs,
data: { event },
});
},
};
}
function createDisabledExecTraceContext(): ExecTraceContext {
return {
emitForegroundCompletion: () => {},
emitBackgroundSpawn: () => {},
emitBackgroundCompletion: () => {},
};
}
function emitExecCommandDiagnostic(params: {
cmd: string;
args: string[];
startedAtMs?: number;
data?: Record<string, unknown>;
}): void {
const argsPrefix = params.args.slice(0, EXEC_DIAGNOSTIC_ARG_LIMIT);
emitDiagnostic({
level: 'debug',
phase: 'exec_command',
durationMs:
params.startedAtMs === undefined ? undefined : Math.max(0, Date.now() - params.startedAtMs),
data: {
command: params.cmd,
argsPrefix,
...(params.args.length > argsPrefix.length
? { omittedArgCount: params.args.length - argsPrefix.length }
: {}),
...(params.data ?? {}),
},
});
}
function normalizeExecutableCommand(cmd: string): string {
const candidate = normalizeExecutableLookup(cmd);
if (!candidate) {
throw new AppError('INVALID_ARGS', `Invalid executable command: ${JSON.stringify(cmd)}`, {
cmd,
hint: 'Use a bare command name from PATH or an absolute executable path.',
});
}
return candidate;
}
function createSpawnError(executable: string, cmd: string, args: string[], err: Error): AppError {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
return createMissingToolError(executable, cmd, err);
}
return createCommandFailedError(executable, cmd, args, err);
}
function createMissingToolError(executable: string, cmd: string, cause: Error): AppError {
return new AppError('TOOL_MISSING', `${executable} not found in PATH`, { cmd }, cause);
}
function createCommandFailedError(
executable: string,
cmd: string,
args: string[],
cause: Error,
): AppError {
return new AppError('COMMAND_FAILED', `Failed to run ${executable}`, { cmd, args }, cause);
}
function createStdinError(
executable: string,
cmd: string,
args: string[],
cause: unknown,
): AppError {
return new AppError(
'COMMAND_FAILED',
`Failed to write stdin for ${executable}`,
{ cmd, args },
cause instanceof Error ? cause : undefined,
);
}
function createCommandCanceledError(executable: string, cmd: string, args: string[]): AppError {
return new AppError('COMMAND_FAILED', 'request canceled', {
cmd,
args,
executable,
reason: 'request_canceled',
});
}
function createTimeoutError(
executable: string,
cmd: string,
args: string[],
timeoutMs: number,
exitCode: number,
stdout: string,
stderr: string,
): AppError {
refactor: harden exec failure wrapping end to end (#1074) * refactor: harden exec failure wrapping end to end Follow-up to #1072, closing the structural gaps that let the missing-processExitError bug class exist: - requireExecSuccess(result, message, extra?) in utils/exec.ts: guards an allowFailure result and throws the curated COMMAND_FAILED (flag set via execFailureDetails) itself. Result-side by design — tool providers and executor overrides return results without throwing, so an ExecOptions knob interpreted at spawn time would silently not fire on those paths. ~30 pure guard-and-throw sites across apple/android/web converted; sites with tolerance branches, cleanup, or exit-0 reachability keep their explicit shape. - Source-scan guard (src/__tests__/exec-wrap-guard.test.ts): fails when an AppError('COMMAND_FAILED', ...) details literal rebuilds the stdout/stderr/exitCode trio inline without the helper; intentional holdouts opt out with a documented exec-guard-allow comment. The guard immediately found three wrap sites the July audit missed (runtime-hints run-as probe, device-ready not_ready branch, perf export table) — now converted. - coerceExecResult at the provider boundaries (executor overrides, apple tool provider scope, android adb provider scope): SDK-supplied callbacks cross an unchecked boundary; coercing once there replaces the per-site String(result.stdout ?? '') defensiveness, which is removed. - normalizeError stderr excerpts now strip noise prefixes (adb:/xcrun:/ simctl:/error:) before rendering; skip/strip lists stay in the kernel deliberately (platforms cannot hook normalizeError) with a comment marking the registry escape hatch if they grow. - AppErrorDetails exported and documented in kernel/errors.ts: the magic keys (hint, processExitError, retriable, reason, diagnosticId, logPath, stdout/stderr/exitCode) now carry types and doc comments. - runIosDevicectl gains tolerateOutput; the devicectl uninstall path in app-install.ts reuses it instead of duplicating the wrap + hint logic (its failure hint now falls back to the devicectl default hint). * fix: reconcile exec hardening with the adb failure classifier Main landed the central adb failure classifier (androidAdbResultError + withAdbFailureHintProvider) while this branch was in flight. Resolution: - Android call sites keep main's androidAdbResultError form — it composes execFailureDetails with the classified adb hint, which is strictly richer than a plain requireExecSuccess conversion for adb invocations. requireExecSuccess remains the shape for non-adb tools and apple/web. - Provider result coercion folds into withAdbFailureHintProvider's enrichment pass (exec/pull/install), so the existing WeakSet memo also prevents coercer stacking. - The reverse-remove wrap in adb-executor now uses androidAdbResultError instead of hand-rolling the trio (flagged by the exec-wrap guard). - Guard-test scan split into helpers (fallow cyclomatic threshold) and the perf artifact-tail clone carries fallow-ignore markers on both platforms, matching the pre-existing marker on the apple side. - Two expectations updated for the stderr noise-prefix strip: classifier compose test and perf sampling reason ('device offline', no 'error:').
2026-07-04 12:56:50 +02:00
// exec-guard-allow: deliberately no processExitError — "timed out after Nms"
// beats whatever partial stderr the killed process left behind.
return new AppError('COMMAND_FAILED', `${executable} timed out after ${timeoutMs}ms`, {
cmd,
args,
stdout,
stderr,
exitCode,
timeoutMs,
});
}
function createExitError(
executable: string,
cmd: string,
args: string[],
exitCode: number,
stdout: string,
stderr: string,
): AppError {
fix: surface stderr excerpts from hand-rolled exec failure wraps (#1072) * fix: surface stderr excerpts from hand-rolled exec failure wraps normalizeError only replaces a generic COMMAND_FAILED message with the first meaningful stderr line when details.processExitError is true, but only createExitError in utils/exec.ts set that flag. Hand-rolled wrap sites (throw new AppError('COMMAND_FAILED', msg, {stdout, stderr, exitCode}) after an allowFailure run) missed it, so users saw messages like "xcrun exited with code 22" with a generic hint instead of the actual stderr excerpt. Add execFailureDetails(result, extra?) to utils/exec.ts returning the stdout/stderr/exitCode spread plus processExitError: true (extras spread after the base keys so call sites can keep truncation or String() normalization), and convert ~42 exit-guarded wrap sites across the apple platform (apps launch/terminate paths, devicectl, simulator boot, perf, perf-xctrace, runner modules, dsym/symbolication, macOS host provider) and android/web (app-helpers, logcat, devices, perf, settings, snapshot helpers, app-lifecycle, device-input-state, multitouch-helper, agent-browser provider). Sites reachable at exit 0 (parse/stdout checks), failure aggregations, timeout/forced-kill errors, and sites that already build richer messages (macos helper JSON, runner-contract early-exit, maestro run-script) are deliberately left unflagged so stderr enrichment never replaces a more specific message. * fix: append stderr excerpts to curated wrap messages instead of replacing them Spreading processExitError to curated call sites made normalizeError replace specific messages like 'uiautomator dump did not return XML' with the bare stderr line, losing the operation context (caught by the provider-failures integration test). Keep wholesale replacement only for the information-free generic '<tool> exited with code N' wraps; curated messages now keep their specific reason and gain the stderr excerpt as a ': <excerpt>' suffix (skipped when already present). Separate commit rather than amend so the semantics change to maybeEnrichCommandFailedMessage stays individually reviewable on the PR.
2026-07-04 10:25:03 +02:00
return new AppError(
'COMMAND_FAILED',
`${executable} exited with code ${exitCode}`,
execFailureDetails({ stdout, stderr, exitCode }, { cmd, args }),
);
refactor: harden exec failure wrapping end to end (#1074) * refactor: harden exec failure wrapping end to end Follow-up to #1072, closing the structural gaps that let the missing-processExitError bug class exist: - requireExecSuccess(result, message, extra?) in utils/exec.ts: guards an allowFailure result and throws the curated COMMAND_FAILED (flag set via execFailureDetails) itself. Result-side by design — tool providers and executor overrides return results without throwing, so an ExecOptions knob interpreted at spawn time would silently not fire on those paths. ~30 pure guard-and-throw sites across apple/android/web converted; sites with tolerance branches, cleanup, or exit-0 reachability keep their explicit shape. - Source-scan guard (src/__tests__/exec-wrap-guard.test.ts): fails when an AppError('COMMAND_FAILED', ...) details literal rebuilds the stdout/stderr/exitCode trio inline without the helper; intentional holdouts opt out with a documented exec-guard-allow comment. The guard immediately found three wrap sites the July audit missed (runtime-hints run-as probe, device-ready not_ready branch, perf export table) — now converted. - coerceExecResult at the provider boundaries (executor overrides, apple tool provider scope, android adb provider scope): SDK-supplied callbacks cross an unchecked boundary; coercing once there replaces the per-site String(result.stdout ?? '') defensiveness, which is removed. - normalizeError stderr excerpts now strip noise prefixes (adb:/xcrun:/ simctl:/error:) before rendering; skip/strip lists stay in the kernel deliberately (platforms cannot hook normalizeError) with a comment marking the registry escape hatch if they grow. - AppErrorDetails exported and documented in kernel/errors.ts: the magic keys (hint, processExitError, retriable, reason, diagnosticId, logPath, stdout/stderr/exitCode) now carry types and doc comments. - runIosDevicectl gains tolerateOutput; the devicectl uninstall path in app-install.ts reuses it instead of duplicating the wrap + hint logic (its failure hint now falls back to the devicectl default hint). * fix: reconcile exec hardening with the adb failure classifier Main landed the central adb failure classifier (androidAdbResultError + withAdbFailureHintProvider) while this branch was in flight. Resolution: - Android call sites keep main's androidAdbResultError form — it composes execFailureDetails with the classified adb hint, which is strictly richer than a plain requireExecSuccess conversion for adb invocations. requireExecSuccess remains the shape for non-adb tools and apple/web. - Provider result coercion folds into withAdbFailureHintProvider's enrichment pass (exec/pull/install), so the existing WeakSet memo also prevents coercer stacking. - The reverse-remove wrap in adb-executor now uses androidAdbResultError instead of hand-rolling the trio (flagged by the exec-wrap guard). - Guard-test scan split into helpers (fallow cyclomatic threshold) and the perf artifact-tail clone carries fallow-ignore markers on both platforms, matching the pre-existing marker on the apple side. - Two expectations updated for the stderr noise-prefix strip: classifier compose test and perf sampling reason ('device offline', no 'error:').
2026-07-04 12:56:50 +02:00
}
/**
* Guard an exec result that was obtained with `allowFailure: true`: throw a
* curated COMMAND_FAILED (with `processExitError` set, so normalizeError
* appends the stderr excerpt) on non-zero exit, and pass the result through
* otherwise. This is the standard shape for "run tool, fail with a specific
* message" call sites — it works under every executor (local spawn, tool
* providers, command overrides) because it checks the returned result rather
* than relying on the spawn layer to throw. `extra` accepts a function so
* failure-only work (hint classification) is not paid on the success path.
*/
export function requireExecSuccess(
result: ExecResult,
message: string,
extra?: Record<string, unknown> | ((result: ExecResult) => Record<string, unknown>),
): ExecResult {
if (result.exitCode === 0) return result;
throw new AppError(
'COMMAND_FAILED',
message,
execFailureDetails(result, typeof extra === 'function' ? extra(result) : extra),
);
fix: surface stderr excerpts from hand-rolled exec failure wraps (#1072) * fix: surface stderr excerpts from hand-rolled exec failure wraps normalizeError only replaces a generic COMMAND_FAILED message with the first meaningful stderr line when details.processExitError is true, but only createExitError in utils/exec.ts set that flag. Hand-rolled wrap sites (throw new AppError('COMMAND_FAILED', msg, {stdout, stderr, exitCode}) after an allowFailure run) missed it, so users saw messages like "xcrun exited with code 22" with a generic hint instead of the actual stderr excerpt. Add execFailureDetails(result, extra?) to utils/exec.ts returning the stdout/stderr/exitCode spread plus processExitError: true (extras spread after the base keys so call sites can keep truncation or String() normalization), and convert ~42 exit-guarded wrap sites across the apple platform (apps launch/terminate paths, devicectl, simulator boot, perf, perf-xctrace, runner modules, dsym/symbolication, macOS host provider) and android/web (app-helpers, logcat, devices, perf, settings, snapshot helpers, app-lifecycle, device-input-state, multitouch-helper, agent-browser provider). Sites reachable at exit 0 (parse/stdout checks), failure aggregations, timeout/forced-kill errors, and sites that already build richer messages (macos helper JSON, runner-contract early-exit, maestro run-script) are deliberately left unflagged so stderr enrichment never replaces a more specific message. * fix: append stderr excerpts to curated wrap messages instead of replacing them Spreading processExitError to curated call sites made normalizeError replace specific messages like 'uiautomator dump did not return XML' with the bare stderr line, losing the operation context (caught by the provider-failures integration test). Keep wholesale replacement only for the information-free generic '<tool> exited with code N' wraps; curated messages now keep their specific reason and gain the stderr excerpt as a ': <excerpt>' suffix (skipped when already present). Separate commit rather than amend so the semantics change to maybeEnrichCommandFailedMessage stays individually reviewable on the PR.
2026-07-04 10:25:03 +02:00
}
/**
* COMMAND_FAILED details for a non-zero exec result. `processExitError: true`
* lets normalizeError surface the first meaningful stderr line as the user-facing
* message instead of the generic wrap message.
*/
export function execFailureDetails(
result: Pick<ExecResult, 'stdout' | 'stderr' | 'exitCode'>,
extra?: Record<string, unknown>,
): Record<string, unknown> {
return {
stdout: result.stdout,
stderr: result.stderr,
exitCode: result.exitCode,
processExitError: true,
fix: surface stderr excerpts from hand-rolled exec failure wraps (#1072) * fix: surface stderr excerpts from hand-rolled exec failure wraps normalizeError only replaces a generic COMMAND_FAILED message with the first meaningful stderr line when details.processExitError is true, but only createExitError in utils/exec.ts set that flag. Hand-rolled wrap sites (throw new AppError('COMMAND_FAILED', msg, {stdout, stderr, exitCode}) after an allowFailure run) missed it, so users saw messages like "xcrun exited with code 22" with a generic hint instead of the actual stderr excerpt. Add execFailureDetails(result, extra?) to utils/exec.ts returning the stdout/stderr/exitCode spread plus processExitError: true (extras spread after the base keys so call sites can keep truncation or String() normalization), and convert ~42 exit-guarded wrap sites across the apple platform (apps launch/terminate paths, devicectl, simulator boot, perf, perf-xctrace, runner modules, dsym/symbolication, macOS host provider) and android/web (app-helpers, logcat, devices, perf, settings, snapshot helpers, app-lifecycle, device-input-state, multitouch-helper, agent-browser provider). Sites reachable at exit 0 (parse/stdout checks), failure aggregations, timeout/forced-kill errors, and sites that already build richer messages (macos helper JSON, runner-contract early-exit, maestro run-script) are deliberately left unflagged so stderr enrichment never replaces a more specific message. * fix: append stderr excerpts to curated wrap messages instead of replacing them Spreading processExitError to curated call sites made normalizeError replace specific messages like 'uiautomator dump did not return XML' with the bare stderr line, losing the operation context (caught by the provider-failures integration test). Keep wholesale replacement only for the information-free generic '<tool> exited with code N' wraps; curated messages now keep their specific reason and gain the stderr excerpt as a ': <excerpt>' suffix (skipped when already present). Separate commit rather than amend so the semantics change to maybeEnrichCommandFailedMessage stays individually reviewable on the PR.
2026-07-04 10:25:03 +02:00
...extra,
};
}
feat: e2e command perf benchmark harness + nightly CI (#630) * feat: add e2e command perf benchmark harness + nightly CI Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown reports. Per-command timing comes from wrapping each batchable command in its own single-step batch (daemon durationMs) plus wall-clock around the process. Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and runs the CLI from source via --experimental-strip-types (no dist build). * refactor(perf): drive the harness CLI via runCmdSync, not spawnSync Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not node:child_process directly. Switch the perf harness's invokeCli to runCmdSync (allowFailure so non-zero exits are recorded as samples) and add a maxBuffer option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default). * perf(harness): warm the runner after open so the first measured command is clean The first interaction after open/relaunch pays the one-time iOS XCUITest runner startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was landing on the first measured command each round (snapshot -i), inflating it ~10x vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after each round's reset-open, and after every freshRoot relaunch, so no measured command absorbs runner startup. Noted in the report header. * refactor(perf): address review + fix Fallow CI - exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer change pulled exec.ts into the audit scope). - .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/ test-app) so its naturally-moderate functions don't trip the complexity gate. - config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so --n/--rounds/--warmup report the actual flag + reject non-integers clearly. - harness.ts: extract toSample(); type sampleError param as CliResult. - scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/ step.args ?? []). - comment/legend rewords (platform defaults are local-convenience/CI-overridden; elements = node count). check:fallow now green; typecheck/lint/unit pass. * perf(harness): downgrade sample ok when a batch step reports ok:false Defensive belt-and-suspenders for the Codex review note: stop-only batch already surfaces a failed step as a top-level failure (caught by invokeCli), but if an on-error=continue mode ever keeps the batch ok while a step fails, don't silently count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
type CommandAbort = { readonly didAbort: boolean };
// Error to reject a spawned child's `error` event with: canceled if we aborted, else a spawn error.
function spawnRejectionError(
abort: CommandAbort,
executable: string,
cmd: string,
args: string[],
err: Error,
): AppError {
return abort.didAbort
? createCommandCanceledError(executable, cmd, args)
: createSpawnError(executable, cmd, args, err);
}
// Failure (if any) for a spawned child's `close` event: canceled if we aborted, an exit error on
// a non-zero code unless allowed, otherwise null (the command resolves successfully).
function commandCloseFailure(
abort: CommandAbort,
executable: string,
cmd: string,
args: string[],
exitCode: number,
allowFailure: boolean | undefined,
stdout: string,
stderr: string,
): AppError | null {
if (abort.didAbort) return createCommandCanceledError(executable, cmd, args);
if (exitCode !== 0 && !allowFailure) {
return createExitError(executable, cmd, args, exitCode, stdout, stderr);
}
return null;
}
function normalizeOverridePath(
rawPath: string | undefined,
envName: string,
kind: 'executable' | 'file',
): string | undefined {
const candidate = rawPath?.trim();
if (!candidate) return undefined;
if (!path.isAbsolute(candidate) || candidate.includes('\0')) {
throw new AppError(
'INVALID_ARGS',
`${envName} must be an absolute ${kind} path, not ${JSON.stringify(rawPath)}`,
{ envName, path: rawPath },
);
}
return candidate;
}
function normalizeExecutableLookup(cmd: string): string | null {
const candidate = cmd.trim();
if (!candidate || candidate.includes('\0')) return null;
if (path.isAbsolute(candidate)) return candidate;
if (candidate.includes('/') || candidate.includes('\\')) {
return null;
}
return BARE_COMMAND_RE.test(candidate) ? candidate : null;
}
function resolvePathExtensions(): string[] {
if (process.platform !== 'win32') return [''];
const rawPathExt = process.env.PATHEXT;
if (!rawPathExt) return WINDOWS_PATH_EXTENSIONS;
const extensions = rawPathExt
.split(';')
.map((value) => value.trim().toLowerCase())
.filter((value) => value.length > 0);
return extensions.length > 0 ? extensions : WINDOWS_PATH_EXTENSIONS;
}
function resolveExecutableCandidates(cmd: string, pathExtensions: string[]): string[] {
if (process.platform !== 'win32') return [cmd];
const lowered = cmd.toLowerCase();
if (pathExtensions.some((extension) => lowered.endsWith(extension))) {
return [cmd];
}
return pathExtensions.map((extension) => `${cmd}${extension}`);
}
export async function isExecutablePath(filePath: string): Promise<boolean> {
try {
if (!(await isFilePath(filePath))) return false;
await access(filePath, process.platform === 'win32' ? constants.F_OK : constants.X_OK);
return true;
} catch {
return false;
2026-01-30 19:41:00 +01:00
}
}
async function isFilePath(filePath: string): Promise<boolean> {
try {
const fileStat = await stat(filePath);
return fileStat.isFile();
} catch {
return false;
}
}
function normalizeTimeoutMs(value: number | undefined): number | undefined {
if (!Number.isFinite(value)) return undefined;
const timeout = Math.floor(value as number);
if (timeout <= 0) return undefined;
return timeout;
}
function watchCommandAbort(
child: ChildProcess,
options: Pick<ExecOptions, 'detached' | 'signal'>,
): { readonly didAbort: boolean; dispose: () => void } {
let didAbort = false;
const onAbort = () => {
didAbort = true;
killProcessTree(child, options.detached);
};
if (options.signal?.aborted) {
onAbort();
} else {
options.signal?.addEventListener('abort', onAbort, { once: true });
}
return {
get didAbort() {
return didAbort;
},
dispose: () => {
options.signal?.removeEventListener('abort', onAbort);
},
};
}
function killProcessTree(child: ChildProcess, detached: boolean | undefined): void {
if (detached && child.pid && process.platform !== 'win32') {
try {
process.kill(-child.pid, 'SIGKILL');
return;
} catch {}
}
child.kill('SIGKILL');
}
async function writeChildStdin(
child: ChildProcess,
stdin: string | Buffer | undefined,
): Promise<void> {
if (!child.stdin) return;
if (stdin === undefined) {
child.stdin?.end();
return;
}
await pipeline(Readable.from([stdin]), child.stdin);
}
function isEpipeError(error: unknown): boolean {
return (
error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'EPIPE'
);
}