2026-04-29 18:25:21 -04:00
|
|
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
2026-04-07 20:19:26 +02:00
|
|
|
import { constants } from 'node:fs';
|
|
|
|
|
import { access, stat } from 'node:fs/promises';
|
|
|
|
|
import path from 'node:path';
|
2026-04-27 17:19:08 -04:00
|
|
|
import { spawn, spawnSync, type ChildProcess, type StdioOptions } from 'node:child_process';
|
|
|
|
|
import { Readable } from 'node:stream';
|
|
|
|
|
import { pipeline } from 'node:stream/promises';
|
refactor: move errors/redaction/device into src/kernel — Phase 5 slice 3 (#940)
* refactor: move errors/redaction/device into src/kernel — Phase 5 slice 3
Relocates the foundational primitive trio from src/utils/ into the kernel/ layer
(joining snapshot.ts from slice 2), per the target folder DAG in
plans/perfect-shape.md §5.5. A pure path codemod, no behavior change.
They form a closed cluster — device -> errors -> redaction, with redaction a
leaf — so kernel/ takes no upward dependency, and every importer becomes a clean
downward import toward kernel. errors.ts is the most-imported module in the
tree; device.ts the §5.5-named headliner. Moving all three atomically avoids a
half-state where one would import another across the utils/kernel boundary.
Imports rewritten by a resolve-based codemod (compares each specifier's resolved
path to the moved files, so the unrelated commands/management/device.ts and
other same-named files are untouched): 483 sites across 402 files. The two
platform-descriptor doc comments and the fallow health baseline key for
device.ts are updated to the new path; the contracts-schema-public guard that
asserts the error helpers pull no diagnostics/node: deps now reads kernel/.
Verified: tsc --noEmit, oxfmt + oxlint --deny-warnings, rslib build, full vitest
suite (2877 pass), fallow audit clean (411 changed files), Layering Guard empty;
kernel/ files import only within kernel.
* docs: update guidance references to kernel/{device,errors} after the move
AGENTS.md (Apple-family sync rule + normalizeError), ADR-0009, and
plans/apple-platform-consolidation.md still named the old src/utils/ paths.
Point them at src/kernel/. plans/perfect-shape.md's utils/device.ts mention is
left as-is — it describes the pre-move diagnosis.
2026-06-30 07:25:02 +02:00
|
|
|
import { AppError } from '../kernel/errors.ts';
|
2026-01-30 19:41:00 +01:00
|
|
|
|
|
|
|
|
export type ExecResult = {
|
|
|
|
|
stdout: string;
|
|
|
|
|
stderr: string;
|
|
|
|
|
exitCode: number;
|
|
|
|
|
stdoutBuffer?: Buffer;
|
|
|
|
|
};
|
|
|
|
|
|
2026-04-29 18:25:21 -04:00
|
|
|
export type ExecOptions = {
|
2026-01-30 19:41:00 +01:00
|
|
|
cwd?: string;
|
|
|
|
|
env?: NodeJS.ProcessEnv;
|
|
|
|
|
allowFailure?: boolean;
|
|
|
|
|
binaryStdout?: boolean;
|
2026-02-02 15:12:23 +01:00
|
|
|
stdin?: string | Buffer;
|
2026-02-12 17:17:56 +01:00
|
|
|
timeoutMs?: number;
|
2026-02-20 17:18:37 +01:00
|
|
|
detached?: boolean;
|
2026-04-27 17:19:08 -04:00
|
|
|
signal?: AbortSignal;
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
/** Max stdout/stderr bytes for synchronous runs (default Node ~1MB). */
|
|
|
|
|
maxBuffer?: number;
|
2026-01-30 19:41:00 +01:00
|
|
|
};
|
|
|
|
|
|
2026-02-20 19:44:58 +01:00
|
|
|
type ExecStreamOptions = ExecOptions & {
|
2026-01-30 19:41:00 +01:00
|
|
|
onStdoutChunk?: (chunk: string) => void;
|
|
|
|
|
onStderrChunk?: (chunk: string) => void;
|
2026-02-20 17:18:37 +01:00
|
|
|
onSpawn?: (child: ReturnType<typeof spawn>) => void;
|
2026-01-30 19:41:00 +01:00
|
|
|
};
|
|
|
|
|
|
2026-02-02 21:33:45 +01:00
|
|
|
export type ExecBackgroundResult = {
|
|
|
|
|
child: ReturnType<typeof spawn>;
|
|
|
|
|
wait: Promise<ExecResult>;
|
|
|
|
|
};
|
|
|
|
|
|
refactor: P3 polish — Interactor type safety, error response builder, OXC (#316)
* refactor: polish improvements from codebase review (P3)
- Export Interactor type and add `satisfies Interactor` to both return
objects in getInteractor() for compile-time completeness checking
- Create errorResponse/successResponse builder in handlers/response.ts
and adopt it in 5 representative handler files (find, record-trace,
interaction-press, session-deploy, snapshot-wait)
- Add ESLint with typescript-eslint, fix unused imports and trivial
lint issues, configure rules to warn for stylistic patterns
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* refactor: complete P3 items — consolidate runner files, document macOS exports, standardize dispatch
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address P3 review feedback — remove invalid ESLint rules, dead code, unused export
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* chore: replace eslint and prettier with oxc
* chore: tighten oxc config and upgrade typescript
* refactor: extract ios runner contract helpers
* docs: tighten agent guidance and add check scripts
* fix: resolve post-rebase validation issues
* fix: remove shell interpolation from metro launcher
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 20:43:24 +02:00
|
|
|
type ExecDetachedOptions = ExecOptions & {
|
|
|
|
|
stdio?: StdioOptions;
|
|
|
|
|
};
|
|
|
|
|
|
2026-06-12 17:32:56 +02:00
|
|
|
export type ExecDetachedExit = {
|
|
|
|
|
pid: number;
|
|
|
|
|
exitCode?: number;
|
|
|
|
|
signal?: NodeJS.Signals;
|
|
|
|
|
error?: string;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
export type ExecDetachedProcess = {
|
|
|
|
|
pid: number;
|
|
|
|
|
exited: Promise<ExecDetachedExit>;
|
|
|
|
|
};
|
|
|
|
|
|
2026-05-13 13:05:25 +02:00
|
|
|
export type ExecBackgroundOptions = ExecOptions & {
|
|
|
|
|
/**
|
|
|
|
|
* Capture stdout/stderr into the wait result when the child has piped stdio.
|
|
|
|
|
* Set false when the caller owns, ignores, or forwards the streams.
|
|
|
|
|
*/
|
|
|
|
|
captureOutput?: boolean;
|
|
|
|
|
stdio?: StdioOptions;
|
|
|
|
|
};
|
|
|
|
|
|
2026-04-07 20:19:26 +02:00
|
|
|
const BARE_COMMAND_RE = /^[A-Za-z0-9][A-Za-z0-9._+-]*$/;
|
|
|
|
|
const WINDOWS_PATH_EXTENSIONS = ['.com', '.exe', '.bat', '.cmd'];
|
2026-04-29 18:25:21 -04:00
|
|
|
export type CommandExecutorOverride = (
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecOptions,
|
|
|
|
|
) => Promise<ExecResult> | undefined;
|
|
|
|
|
|
|
|
|
|
const commandExecutorOverrideScope = new AsyncLocalStorage<CommandExecutorOverride | undefined>();
|
|
|
|
|
|
|
|
|
|
export async function withCommandExecutorOverride<T>(
|
|
|
|
|
override: CommandExecutorOverride | undefined,
|
|
|
|
|
fn: () => Promise<T>,
|
|
|
|
|
): Promise<T> {
|
|
|
|
|
if (!override) return await fn();
|
|
|
|
|
return await commandExecutorOverrideScope.run(override, fn);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Used by local executors to bypass the active override for intentional host command execution.
|
|
|
|
|
export async function withoutCommandExecutorOverride<T>(fn: () => Promise<T>): Promise<T> {
|
|
|
|
|
return await commandExecutorOverrideScope.run(undefined, fn);
|
|
|
|
|
}
|
2026-04-07 20:19:26 +02:00
|
|
|
|
2026-01-30 19:41:00 +01:00
|
|
|
export async function runCmd(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecOptions = {},
|
2026-04-27 17:19:08 -04:00
|
|
|
): Promise<ExecResult> {
|
2026-04-29 18:25:21 -04:00
|
|
|
const overrideResult = commandExecutorOverrideScope.getStore()?.(cmd, args, options);
|
|
|
|
|
if (overrideResult) return await overrideResult;
|
2026-04-27 17:19:08 -04:00
|
|
|
return await runSpawnedCommand(cmd, args, options);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function runCmdStreaming(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecStreamOptions = {},
|
|
|
|
|
): Promise<ExecResult> {
|
2026-04-29 18:25:21 -04:00
|
|
|
const overrideResult = commandExecutorOverrideScope.getStore()?.(cmd, args, options);
|
|
|
|
|
if (overrideResult) return await overrideResult;
|
2026-04-27 17:19:08 -04:00
|
|
|
return await runSpawnedCommand(cmd, args, options);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function runSpawnedCommand(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecStreamOptions = {},
|
2026-01-30 19:41:00 +01:00
|
|
|
): Promise<ExecResult> {
|
2026-04-07 20:19:26 +02:00
|
|
|
const executable = normalizeExecutableCommand(cmd);
|
2026-01-30 19:41:00 +01:00
|
|
|
return new Promise((resolve, reject) => {
|
2026-04-07 20:19:26 +02:00
|
|
|
const child = spawn(executable, args, {
|
2026-01-30 19:41:00 +01:00
|
|
|
cwd: options.cwd,
|
|
|
|
|
env: options.env,
|
2026-02-02 15:12:23 +01:00
|
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
2026-02-20 17:18:37 +01:00
|
|
|
detached: options.detached,
|
2026-05-12 19:15:11 +02:00
|
|
|
windowsHide: true,
|
2026-04-07 20:19:26 +02:00
|
|
|
shell: false,
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
2026-04-27 17:19:08 -04:00
|
|
|
options.onSpawn?.(child);
|
2026-01-30 19:41:00 +01:00
|
|
|
|
|
|
|
|
let stdout = '';
|
2026-04-27 17:19:08 -04:00
|
|
|
const stdoutChunks: Buffer[] | undefined = options.binaryStdout ? [] : undefined;
|
2026-01-30 19:41:00 +01:00
|
|
|
let stderr = '';
|
2026-02-12 17:17:56 +01:00
|
|
|
let didTimeout = false;
|
|
|
|
|
const timeoutMs = normalizeTimeoutMs(options.timeoutMs);
|
|
|
|
|
const timeoutHandle = timeoutMs
|
|
|
|
|
? setTimeout(() => {
|
2026-03-17 18:18:53 +01:00
|
|
|
didTimeout = true;
|
2026-04-27 17:19:08 -04:00
|
|
|
killProcessTree(child, options.detached);
|
2026-03-17 18:18:53 +01:00
|
|
|
}, timeoutMs)
|
2026-02-12 17:17:56 +01:00
|
|
|
: null;
|
2026-05-13 13:05:25 +02:00
|
|
|
const abort = watchCommandAbort(child, options);
|
2026-01-30 19:41:00 +01:00
|
|
|
|
|
|
|
|
if (!options.binaryStdout) child.stdout.setEncoding('utf8');
|
|
|
|
|
child.stderr.setEncoding('utf8');
|
|
|
|
|
|
2026-04-27 17:19:08 -04:00
|
|
|
void writeChildStdin(child, options.stdin).catch((err: unknown) => {
|
2026-05-13 13:05:25 +02:00
|
|
|
if (abort.didAbort || didTimeout) return;
|
2026-04-27 17:19:08 -04:00
|
|
|
if (isEpipeError(err)) return;
|
|
|
|
|
reject(createStdinError(executable, cmd, args, err));
|
|
|
|
|
killProcessTree(child, options.detached);
|
|
|
|
|
});
|
2026-02-02 15:12:23 +01:00
|
|
|
|
2026-01-30 19:41:00 +01:00
|
|
|
child.stdout.on('data', (chunk) => {
|
|
|
|
|
if (options.binaryStdout) {
|
2026-04-27 17:19:08 -04:00
|
|
|
stdoutChunks?.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
|
|
|
return;
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
2026-04-27 17:19:08 -04:00
|
|
|
const text = String(chunk);
|
|
|
|
|
stdout += text;
|
|
|
|
|
options.onStdoutChunk?.(text);
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
child.stderr.on('data', (chunk) => {
|
2026-04-27 17:19:08 -04:00
|
|
|
const text = String(chunk);
|
|
|
|
|
stderr += text;
|
|
|
|
|
options.onStderrChunk?.(text);
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
child.on('error', (err) => {
|
2026-02-12 17:17:56 +01:00
|
|
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
2026-05-13 13:05:25 +02:00
|
|
|
abort.dispose();
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
reject(spawnRejectionError(abort, executable, cmd, args, err));
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
child.on('close', (code) => {
|
2026-02-12 17:17:56 +01:00
|
|
|
if (timeoutHandle) clearTimeout(timeoutHandle);
|
2026-05-13 13:05:25 +02:00
|
|
|
abort.dispose();
|
2026-01-30 19:41:00 +01:00
|
|
|
const exitCode = code ?? 1;
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
if (!abort.didAbort && didTimeout && timeoutMs) {
|
2026-04-27 17:19:08 -04:00
|
|
|
reject(createTimeoutError(executable, cmd, args, timeoutMs, exitCode, stdout, stderr));
|
2026-02-12 17:17:56 +01:00
|
|
|
return;
|
|
|
|
|
}
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
const failure = commandCloseFailure(
|
|
|
|
|
abort,
|
|
|
|
|
executable,
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
exitCode,
|
|
|
|
|
options.allowFailure,
|
|
|
|
|
stdout,
|
|
|
|
|
stderr,
|
|
|
|
|
);
|
|
|
|
|
if (failure) {
|
|
|
|
|
reject(failure);
|
2026-01-30 19:41:00 +01:00
|
|
|
return;
|
|
|
|
|
}
|
2026-04-27 17:19:08 -04:00
|
|
|
resolve({
|
|
|
|
|
stdout,
|
|
|
|
|
stderr,
|
|
|
|
|
exitCode,
|
|
|
|
|
stdoutBuffer: stdoutChunks ? Buffer.concat(stdoutChunks) : undefined,
|
|
|
|
|
});
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function whichCmd(cmd: string): Promise<boolean> {
|
2026-04-11 21:36:54 +02:00
|
|
|
const candidate = normalizeExecutableLookup(cmd);
|
2026-04-07 20:19:26 +02:00
|
|
|
if (!candidate) return false;
|
|
|
|
|
|
|
|
|
|
if (path.isAbsolute(candidate)) {
|
2026-04-11 21:36:54 +02:00
|
|
|
return isExecutablePath(candidate);
|
2026-04-07 20:19:26 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const pathValue = process.env.PATH;
|
|
|
|
|
if (!pathValue) return false;
|
|
|
|
|
const pathExtensions = resolvePathExtensions();
|
|
|
|
|
for (const directory of pathValue.split(path.delimiter)) {
|
|
|
|
|
const trimmedDirectory = directory.trim();
|
|
|
|
|
if (!trimmedDirectory) continue;
|
|
|
|
|
for (const entry of resolveExecutableCandidates(candidate, pathExtensions)) {
|
2026-04-11 21:36:54 +02:00
|
|
|
if (await isExecutablePath(path.join(trimmedDirectory, entry))) {
|
2026-04-07 20:19:26 +02:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
2026-04-07 20:19:26 +02:00
|
|
|
|
|
|
|
|
return false;
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
|
|
|
|
|
2026-04-11 21:36:54 +02:00
|
|
|
export async function resolveExecutableOverridePath(
|
|
|
|
|
rawPath: string | undefined,
|
|
|
|
|
envName: string,
|
|
|
|
|
): Promise<string | undefined> {
|
|
|
|
|
const candidate = normalizeOverridePath(rawPath, envName, 'executable');
|
|
|
|
|
if (!candidate) return undefined;
|
|
|
|
|
if (!(await isExecutablePath(candidate))) {
|
|
|
|
|
throw new AppError(
|
|
|
|
|
'TOOL_MISSING',
|
|
|
|
|
`${envName} points to a missing or non-executable file: ${candidate}`,
|
|
|
|
|
{ envName, path: candidate },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return candidate;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function resolveFileOverridePath(
|
|
|
|
|
rawPath: string | undefined,
|
|
|
|
|
envName: string,
|
|
|
|
|
): Promise<string | undefined> {
|
|
|
|
|
const candidate = normalizeOverridePath(rawPath, envName, 'file');
|
|
|
|
|
if (!candidate) return undefined;
|
|
|
|
|
if (!(await isFilePath(candidate))) {
|
|
|
|
|
throw new AppError(
|
|
|
|
|
'TOOL_MISSING',
|
|
|
|
|
`${envName} points to a missing or non-file path: ${candidate}`,
|
|
|
|
|
{ envName, path: candidate },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return candidate;
|
|
|
|
|
}
|
|
|
|
|
|
2026-01-30 19:41:00 +01:00
|
|
|
export function runCmdSync(cmd: string, args: string[], options: ExecOptions = {}): ExecResult {
|
2026-04-07 20:19:26 +02:00
|
|
|
const executable = normalizeExecutableCommand(cmd);
|
|
|
|
|
const result = spawnSync(executable, args, {
|
2026-01-30 19:41:00 +01:00
|
|
|
cwd: options.cwd,
|
|
|
|
|
env: options.env,
|
2026-02-02 15:12:23 +01:00
|
|
|
stdio: ['pipe', 'pipe', 'pipe'],
|
2026-01-30 19:41:00 +01:00
|
|
|
encoding: options.binaryStdout ? undefined : 'utf8',
|
2026-02-02 15:12:23 +01:00
|
|
|
input: options.stdin,
|
2026-02-12 17:17:56 +01:00
|
|
|
timeout: normalizeTimeoutMs(options.timeoutMs),
|
2026-05-12 19:15:11 +02:00
|
|
|
windowsHide: true,
|
2026-04-07 20:19:26 +02:00
|
|
|
shell: false,
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
...(options.maxBuffer !== undefined ? { maxBuffer: options.maxBuffer } : {}),
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (result.error) {
|
|
|
|
|
const code = (result.error as NodeJS.ErrnoException).code;
|
2026-02-12 17:17:56 +01:00
|
|
|
if (code === 'ETIMEDOUT') {
|
2026-03-17 18:18:53 +01:00
|
|
|
throw new AppError(
|
|
|
|
|
'COMMAND_FAILED',
|
2026-04-07 20:19:26 +02:00
|
|
|
`${executable} timed out after ${normalizeTimeoutMs(options.timeoutMs)}ms`,
|
2026-03-17 18:18:53 +01:00
|
|
|
{
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
timeoutMs: normalizeTimeoutMs(options.timeoutMs),
|
|
|
|
|
},
|
|
|
|
|
result.error,
|
|
|
|
|
);
|
2026-02-12 17:17:56 +01:00
|
|
|
}
|
2026-01-30 19:41:00 +01:00
|
|
|
if (code === 'ENOENT') {
|
2026-04-27 17:19:08 -04:00
|
|
|
throw createMissingToolError(executable, cmd, result.error);
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
2026-04-27 17:19:08 -04:00
|
|
|
throw createCommandFailedError(executable, cmd, args, result.error);
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const stdoutBuffer = options.binaryStdout
|
|
|
|
|
? Buffer.isBuffer(result.stdout)
|
|
|
|
|
? result.stdout
|
|
|
|
|
: Buffer.from(result.stdout ?? '')
|
|
|
|
|
: undefined;
|
|
|
|
|
const stdout = options.binaryStdout
|
|
|
|
|
? ''
|
|
|
|
|
: typeof result.stdout === 'string'
|
|
|
|
|
? result.stdout
|
|
|
|
|
: (result.stdout ?? '').toString();
|
|
|
|
|
const stderr =
|
|
|
|
|
typeof result.stderr === 'string' ? result.stderr : (result.stderr ?? '').toString();
|
|
|
|
|
const exitCode = result.status ?? 1;
|
|
|
|
|
|
|
|
|
|
if (exitCode !== 0 && !options.allowFailure) {
|
2026-04-27 17:19:08 -04:00
|
|
|
throw createExitError(executable, cmd, args, exitCode, stdout, stderr);
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { stdout, stderr, exitCode, stdoutBuffer };
|
|
|
|
|
}
|
|
|
|
|
|
refactor: P3 polish — Interactor type safety, error response builder, OXC (#316)
* refactor: polish improvements from codebase review (P3)
- Export Interactor type and add `satisfies Interactor` to both return
objects in getInteractor() for compile-time completeness checking
- Create errorResponse/successResponse builder in handlers/response.ts
and adopt it in 5 representative handler files (find, record-trace,
interaction-press, session-deploy, snapshot-wait)
- Add ESLint with typescript-eslint, fix unused imports and trivial
lint issues, configure rules to warn for stylistic patterns
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* refactor: complete P3 items — consolidate runner files, document macOS exports, standardize dispatch
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address P3 review feedback — remove invalid ESLint rules, dead code, unused export
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* chore: replace eslint and prettier with oxc
* chore: tighten oxc config and upgrade typescript
* refactor: extract ios runner contract helpers
* docs: tighten agent guidance and add check scripts
* fix: resolve post-rebase validation issues
* fix: remove shell interpolation from metro launcher
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 20:43:24 +02:00
|
|
|
export function runCmdDetached(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecDetachedOptions = {},
|
|
|
|
|
): number {
|
2026-06-12 17:32:56 +02:00
|
|
|
return runCmdDetachedMonitored(cmd, args, options).pid;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function runCmdDetachedMonitored(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
options: ExecDetachedOptions = {},
|
|
|
|
|
): ExecDetachedProcess {
|
2026-04-07 20:19:26 +02:00
|
|
|
const executable = normalizeExecutableCommand(cmd);
|
|
|
|
|
const child = spawn(executable, args, {
|
2026-01-30 19:41:00 +01:00
|
|
|
cwd: options.cwd,
|
|
|
|
|
env: options.env,
|
refactor: P3 polish — Interactor type safety, error response builder, OXC (#316)
* refactor: polish improvements from codebase review (P3)
- Export Interactor type and add `satisfies Interactor` to both return
objects in getInteractor() for compile-time completeness checking
- Create errorResponse/successResponse builder in handlers/response.ts
and adopt it in 5 representative handler files (find, record-trace,
interaction-press, session-deploy, snapshot-wait)
- Add ESLint with typescript-eslint, fix unused imports and trivial
lint issues, configure rules to warn for stylistic patterns
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* refactor: complete P3 items — consolidate runner files, document macOS exports, standardize dispatch
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* fix: address P3 review feedback — remove invalid ESLint rules, dead code, unused export
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* chore: replace eslint and prettier with oxc
* chore: tighten oxc config and upgrade typescript
* refactor: extract ios runner contract helpers
* docs: tighten agent guidance and add check scripts
* fix: resolve post-rebase validation issues
* fix: remove shell interpolation from metro launcher
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-29 20:43:24 +02:00
|
|
|
stdio: options.stdio ?? 'ignore',
|
2026-01-30 19:41:00 +01:00
|
|
|
detached: true,
|
2026-05-12 19:15:11 +02:00
|
|
|
windowsHide: true,
|
2026-04-07 20:19:26 +02:00
|
|
|
shell: false,
|
2026-01-30 19:41:00 +01:00
|
|
|
});
|
2026-06-12 17:32:56 +02:00
|
|
|
const pid = child.pid ?? 0;
|
|
|
|
|
const exited = new Promise<ExecDetachedExit>((resolve) => {
|
|
|
|
|
child.once('error', (err) => {
|
|
|
|
|
resolve({ pid, error: err.message });
|
|
|
|
|
});
|
|
|
|
|
child.once('exit', (code, signal) => {
|
|
|
|
|
resolve({
|
|
|
|
|
pid,
|
|
|
|
|
...(typeof code === 'number' ? { exitCode: code } : {}),
|
|
|
|
|
...(signal ? { signal } : {}),
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
});
|
2026-01-30 19:41:00 +01:00
|
|
|
child.unref();
|
2026-06-12 17:32:56 +02:00
|
|
|
return { pid, exited };
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
|
|
|
|
|
2026-02-02 21:33:45 +01:00
|
|
|
export function runCmdBackground(
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
2026-05-13 13:05:25 +02:00
|
|
|
options: ExecBackgroundOptions = {},
|
2026-02-02 21:33:45 +01:00
|
|
|
): ExecBackgroundResult {
|
2026-04-07 20:19:26 +02:00
|
|
|
const executable = normalizeExecutableCommand(cmd);
|
|
|
|
|
const child = spawn(executable, args, {
|
2026-02-02 21:33:45 +01:00
|
|
|
cwd: options.cwd,
|
|
|
|
|
env: options.env,
|
2026-05-13 13:05:25 +02:00
|
|
|
stdio: options.stdio ?? ['ignore', 'pipe', 'pipe'],
|
2026-02-20 17:18:37 +01:00
|
|
|
detached: options.detached,
|
2026-05-12 19:15:11 +02:00
|
|
|
windowsHide: true,
|
2026-04-07 20:19:26 +02:00
|
|
|
shell: false,
|
2026-02-02 21:33:45 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
let stdout = '';
|
|
|
|
|
let stderr = '';
|
2026-05-13 13:05:25 +02:00
|
|
|
const captureOutput = options.captureOutput ?? true;
|
|
|
|
|
const abort = watchCommandAbort(child, options);
|
2026-02-02 21:33:45 +01:00
|
|
|
|
2026-05-13 13:05:25 +02:00
|
|
|
if (captureOutput) {
|
|
|
|
|
child.stdout?.setEncoding('utf8');
|
|
|
|
|
child.stderr?.setEncoding('utf8');
|
2026-02-02 21:33:45 +01:00
|
|
|
|
2026-05-13 13:05:25 +02:00
|
|
|
child.stdout?.on('data', (chunk) => {
|
|
|
|
|
stdout += chunk;
|
|
|
|
|
});
|
|
|
|
|
child.stderr?.on('data', (chunk) => {
|
|
|
|
|
stderr += chunk;
|
|
|
|
|
});
|
|
|
|
|
}
|
2026-02-02 21:33:45 +01:00
|
|
|
|
|
|
|
|
const wait = new Promise<ExecResult>((resolve, reject) => {
|
|
|
|
|
child.on('error', (err) => {
|
2026-05-13 13:05:25 +02:00
|
|
|
abort.dispose();
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
reject(spawnRejectionError(abort, executable, cmd, args, err));
|
2026-02-02 21:33:45 +01:00
|
|
|
});
|
|
|
|
|
child.on('close', (code) => {
|
2026-05-13 13:05:25 +02:00
|
|
|
abort.dispose();
|
2026-02-02 21:33:45 +01:00
|
|
|
const exitCode = code ?? 1;
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
const failure = commandCloseFailure(
|
|
|
|
|
abort,
|
|
|
|
|
executable,
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
exitCode,
|
|
|
|
|
options.allowFailure,
|
|
|
|
|
stdout,
|
|
|
|
|
stderr,
|
|
|
|
|
);
|
|
|
|
|
if (failure) {
|
|
|
|
|
reject(failure);
|
2026-02-02 21:33:45 +01:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
resolve({ stdout, stderr, exitCode });
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
return { child, wait };
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-07 20:19:26 +02:00
|
|
|
function normalizeExecutableCommand(cmd: string): string {
|
2026-04-11 21:36:54 +02:00
|
|
|
const candidate = normalizeExecutableLookup(cmd);
|
2026-04-07 20:19:26 +02:00
|
|
|
if (!candidate) {
|
|
|
|
|
throw new AppError('INVALID_ARGS', `Invalid executable command: ${JSON.stringify(cmd)}`, {
|
|
|
|
|
cmd,
|
2026-04-11 21:36:54 +02:00
|
|
|
hint: 'Use a bare command name from PATH or an absolute executable path.',
|
2026-04-07 20:19:26 +02:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return candidate;
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-27 17:19:08 -04:00
|
|
|
function createSpawnError(executable: string, cmd: string, args: string[], err: Error): AppError {
|
|
|
|
|
const code = (err as NodeJS.ErrnoException).code;
|
|
|
|
|
if (code === 'ENOENT') {
|
|
|
|
|
return createMissingToolError(executable, cmd, err);
|
|
|
|
|
}
|
|
|
|
|
return createCommandFailedError(executable, cmd, args, err);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createMissingToolError(executable: string, cmd: string, cause: Error): AppError {
|
|
|
|
|
return new AppError('TOOL_MISSING', `${executable} not found in PATH`, { cmd }, cause);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createCommandFailedError(
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
cause: Error,
|
|
|
|
|
): AppError {
|
|
|
|
|
return new AppError('COMMAND_FAILED', `Failed to run ${executable}`, { cmd, args }, cause);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createStdinError(
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
cause: unknown,
|
|
|
|
|
): AppError {
|
|
|
|
|
return new AppError(
|
|
|
|
|
'COMMAND_FAILED',
|
|
|
|
|
`Failed to write stdin for ${executable}`,
|
|
|
|
|
{ cmd, args },
|
|
|
|
|
cause instanceof Error ? cause : undefined,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createCommandCanceledError(executable: string, cmd: string, args: string[]): AppError {
|
|
|
|
|
return new AppError('COMMAND_FAILED', 'request canceled', {
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
executable,
|
|
|
|
|
reason: 'request_canceled',
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createTimeoutError(
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
timeoutMs: number,
|
|
|
|
|
exitCode: number,
|
|
|
|
|
stdout: string,
|
|
|
|
|
stderr: string,
|
|
|
|
|
): AppError {
|
|
|
|
|
return new AppError('COMMAND_FAILED', `${executable} timed out after ${timeoutMs}ms`, {
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
stdout,
|
|
|
|
|
stderr,
|
|
|
|
|
exitCode,
|
|
|
|
|
timeoutMs,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createExitError(
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
exitCode: number,
|
|
|
|
|
stdout: string,
|
|
|
|
|
stderr: string,
|
|
|
|
|
): AppError {
|
|
|
|
|
return new AppError('COMMAND_FAILED', `${executable} exited with code ${exitCode}`, {
|
|
|
|
|
cmd,
|
|
|
|
|
args,
|
|
|
|
|
stdout,
|
|
|
|
|
stderr,
|
|
|
|
|
exitCode,
|
|
|
|
|
processExitError: true,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
feat: e2e command perf benchmark harness + nightly CI (#630)
* feat: add e2e command perf benchmark harness + nightly CI
Adds scripts/perf, a cheap end-to-end perf benchmark that drives the built
CLI through an ordered Settings tour of ~24 commands for N rounds, on a fully
isolated daemon/state-dir and self-cleaning device, and emits JSON + Markdown
reports. Per-command timing comes from wrapping each batchable command in its
own single-step batch (daemon durationMs) plus wall-clock around the process.
Wires a scheduled + workflow_dispatch CI job (perf-nightly.yml) that reuses the
cached iOS XCUITest runner (setup-apple-replay) and the Android replay host, and
runs the CLI from source via --experimental-strip-types (no dist build).
* refactor(perf): drive the harness CLI via runCmdSync, not spawnSync
Review (P2): repo rule is to spawn processes through src/utils/exec.ts, not
node:child_process directly. Switch the perf harness's invokeCli to runCmdSync
(allowFailure so non-zero exits are recorded as samples) and add a maxBuffer
option to ExecOptions/runCmdSync (snapshot payloads exceed Node's ~1MB default).
* perf(harness): warm the runner after open so the first measured command is clean
The first interaction after open/relaunch pays the one-time iOS XCUITest runner
startup (~10s+ cold) and a per-relaunch first-AX-query settle cost (~4s). That was
landing on the first measured command each round (snapshot -i), inflating it ~10x
vs the next snapshot. Run an untimed warmup snapshot -i after establishSession, after
each round's reset-open, and after every freshRoot relaunch, so no measured command
absorbs runner startup. Noted in the report header.
* refactor(perf): address review + fix Fallow CI
- exec.ts: extract spawnRejectionError + commandCloseFailure helpers, deduping the
error/close handler clones (Fallow duplication ✗ that surfaced once the maxBuffer
change pulled exec.ts into the audit scope).
- .fallowrc: exclude scripts/perf/** (non-shipped benchmark tooling, like examples/
test-app) so its naturally-moderate functions don't trip the complexity gate.
- config.ts: drop unused exports CLI_BIN/DEFAULT_OUT_DIR; add readIntValue so
--n/--rounds/--warmup report the actual flag + reject non-integers clearly.
- harness.ts: extract toSample(); type sampleError param as CliResult.
- scenario.ts: ScenarioStep is now a discriminated union on execMode (removes step.step!/
step.args ?? []).
- comment/legend rewords (platform defaults are local-convenience/CI-overridden;
elements = node count). check:fallow now green; typecheck/lint/unit pass.
* perf(harness): downgrade sample ok when a batch step reports ok:false
Defensive belt-and-suspenders for the Codex review note: stop-only batch already
surfaces a failed step as a top-level failure (caught by invokeCli), but if an
on-error=continue mode ever keeps the batch ok while a step fails, don't silently
count that step as a successful sample — derive ok from the step's own result.ok.
2026-05-31 14:37:59 +02:00
|
|
|
type CommandAbort = { readonly didAbort: boolean };
|
|
|
|
|
|
|
|
|
|
// Error to reject a spawned child's `error` event with: canceled if we aborted, else a spawn error.
|
|
|
|
|
function spawnRejectionError(
|
|
|
|
|
abort: CommandAbort,
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
err: Error,
|
|
|
|
|
): AppError {
|
|
|
|
|
return abort.didAbort
|
|
|
|
|
? createCommandCanceledError(executable, cmd, args)
|
|
|
|
|
: createSpawnError(executable, cmd, args, err);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Failure (if any) for a spawned child's `close` event: canceled if we aborted, an exit error on
|
|
|
|
|
// a non-zero code unless allowed, otherwise null (the command resolves successfully).
|
|
|
|
|
function commandCloseFailure(
|
|
|
|
|
abort: CommandAbort,
|
|
|
|
|
executable: string,
|
|
|
|
|
cmd: string,
|
|
|
|
|
args: string[],
|
|
|
|
|
exitCode: number,
|
|
|
|
|
allowFailure: boolean | undefined,
|
|
|
|
|
stdout: string,
|
|
|
|
|
stderr: string,
|
|
|
|
|
): AppError | null {
|
|
|
|
|
if (abort.didAbort) return createCommandCanceledError(executable, cmd, args);
|
|
|
|
|
if (exitCode !== 0 && !allowFailure) {
|
|
|
|
|
return createExitError(executable, cmd, args, exitCode, stdout, stderr);
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-11 21:36:54 +02:00
|
|
|
function normalizeOverridePath(
|
|
|
|
|
rawPath: string | undefined,
|
|
|
|
|
envName: string,
|
|
|
|
|
kind: 'executable' | 'file',
|
|
|
|
|
): string | undefined {
|
|
|
|
|
const candidate = rawPath?.trim();
|
|
|
|
|
if (!candidate) return undefined;
|
|
|
|
|
if (!path.isAbsolute(candidate) || candidate.includes('\0')) {
|
|
|
|
|
throw new AppError(
|
|
|
|
|
'INVALID_ARGS',
|
|
|
|
|
`${envName} must be an absolute ${kind} path, not ${JSON.stringify(rawPath)}`,
|
|
|
|
|
{ envName, path: rawPath },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return candidate;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function normalizeExecutableLookup(cmd: string): string | null {
|
2026-04-07 20:19:26 +02:00
|
|
|
const candidate = cmd.trim();
|
|
|
|
|
if (!candidate || candidate.includes('\0')) return null;
|
|
|
|
|
if (path.isAbsolute(candidate)) return candidate;
|
|
|
|
|
if (candidate.includes('/') || candidate.includes('\\')) {
|
2026-04-11 21:36:54 +02:00
|
|
|
return null;
|
2026-04-07 20:19:26 +02:00
|
|
|
}
|
|
|
|
|
return BARE_COMMAND_RE.test(candidate) ? candidate : null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolvePathExtensions(): string[] {
|
|
|
|
|
if (process.platform !== 'win32') return [''];
|
|
|
|
|
const rawPathExt = process.env.PATHEXT;
|
|
|
|
|
if (!rawPathExt) return WINDOWS_PATH_EXTENSIONS;
|
|
|
|
|
const extensions = rawPathExt
|
|
|
|
|
.split(';')
|
|
|
|
|
.map((value) => value.trim().toLowerCase())
|
|
|
|
|
.filter((value) => value.length > 0);
|
|
|
|
|
return extensions.length > 0 ? extensions : WINDOWS_PATH_EXTENSIONS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolveExecutableCandidates(cmd: string, pathExtensions: string[]): string[] {
|
|
|
|
|
if (process.platform !== 'win32') return [cmd];
|
|
|
|
|
const lowered = cmd.toLowerCase();
|
|
|
|
|
if (pathExtensions.some((extension) => lowered.endsWith(extension))) {
|
|
|
|
|
return [cmd];
|
|
|
|
|
}
|
|
|
|
|
return pathExtensions.map((extension) => `${cmd}${extension}`);
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-11 21:36:54 +02:00
|
|
|
export async function isExecutablePath(filePath: string): Promise<boolean> {
|
2026-04-07 20:19:26 +02:00
|
|
|
try {
|
2026-04-11 21:36:54 +02:00
|
|
|
if (!(await isFilePath(filePath))) return false;
|
2026-04-07 20:19:26 +02:00
|
|
|
await access(filePath, process.platform === 'win32' ? constants.F_OK : constants.X_OK);
|
|
|
|
|
return true;
|
|
|
|
|
} catch {
|
|
|
|
|
return false;
|
2026-01-30 19:41:00 +01:00
|
|
|
}
|
|
|
|
|
}
|
2026-02-12 17:17:56 +01:00
|
|
|
|
2026-04-11 21:36:54 +02:00
|
|
|
async function isFilePath(filePath: string): Promise<boolean> {
|
|
|
|
|
try {
|
|
|
|
|
const fileStat = await stat(filePath);
|
|
|
|
|
return fileStat.isFile();
|
|
|
|
|
} catch {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-12 17:17:56 +01:00
|
|
|
function normalizeTimeoutMs(value: number | undefined): number | undefined {
|
|
|
|
|
if (!Number.isFinite(value)) return undefined;
|
|
|
|
|
const timeout = Math.floor(value as number);
|
|
|
|
|
if (timeout <= 0) return undefined;
|
|
|
|
|
return timeout;
|
|
|
|
|
}
|
2026-04-27 17:19:08 -04:00
|
|
|
|
2026-05-13 13:05:25 +02:00
|
|
|
function watchCommandAbort(
|
|
|
|
|
child: ChildProcess,
|
|
|
|
|
options: Pick<ExecOptions, 'detached' | 'signal'>,
|
|
|
|
|
): { readonly didAbort: boolean; dispose: () => void } {
|
|
|
|
|
let didAbort = false;
|
|
|
|
|
const onAbort = () => {
|
|
|
|
|
didAbort = true;
|
|
|
|
|
killProcessTree(child, options.detached);
|
|
|
|
|
};
|
|
|
|
|
if (options.signal?.aborted) {
|
|
|
|
|
onAbort();
|
|
|
|
|
} else {
|
|
|
|
|
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
get didAbort() {
|
|
|
|
|
return didAbort;
|
|
|
|
|
},
|
|
|
|
|
dispose: () => {
|
|
|
|
|
options.signal?.removeEventListener('abort', onAbort);
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2026-04-27 17:19:08 -04:00
|
|
|
function killProcessTree(child: ChildProcess, detached: boolean | undefined): void {
|
|
|
|
|
if (detached && child.pid && process.platform !== 'win32') {
|
|
|
|
|
try {
|
|
|
|
|
process.kill(-child.pid, 'SIGKILL');
|
|
|
|
|
return;
|
|
|
|
|
} catch {}
|
|
|
|
|
}
|
|
|
|
|
child.kill('SIGKILL');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function writeChildStdin(
|
|
|
|
|
child: ChildProcess,
|
|
|
|
|
stdin: string | Buffer | undefined,
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
if (!child.stdin) return;
|
|
|
|
|
if (stdin === undefined) {
|
|
|
|
|
child.stdin?.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
await pipeline(Readable.from([stdin]), child.stdin);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isEpipeError(error: unknown): boolean {
|
|
|
|
|
return (
|
|
|
|
|
error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'EPIPE'
|
|
|
|
|
);
|
|
|
|
|
}
|