mirror of
https://github.com/callstack/agent-device.git
synced 2026-09-14 20:06:34 +08:00
refactor(replay-test): make an attempt failure's warnings channel always present
An empty array and an absent key said the same thing two ways, so every consumer needed both guards. The failed outcome now always carries the warnings array, like the passed outcome already did, and the result builder narrows the attempt once instead of per field. This also removes the clone the shared candidate-view formatter had grown between the CLI and MCP error renderers — the MCP error text gained the same warnings lines the CLI stderr prints.
This commit is contained in:
@@ -106,6 +106,7 @@ test('materializeReplayTestAttemptArtifacts writes failure manifest and copies l
|
||||
details: { reason: 'timeout', artifactPaths: [screenshotPath] },
|
||||
},
|
||||
artifactPaths: [screenshotPath],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
},
|
||||
filePath: replayPath,
|
||||
@@ -189,6 +190,7 @@ test('materialization preserves replay sources and diagnostics named after attem
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'original failure' },
|
||||
artifactPaths,
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
},
|
||||
filePath: replayPath,
|
||||
@@ -267,6 +269,7 @@ test('materialization copies a log listed in both the outcome and error only onc
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'failed', logPath },
|
||||
artifactPaths: [logPath, logPath],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
},
|
||||
filePath: replayPath,
|
||||
@@ -300,6 +303,7 @@ test.each(['result.txt', 'failure.txt', 'RESULT.TXT'])(
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'original failure' },
|
||||
artifactPaths: [diagnosticPath],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
},
|
||||
filePath: replayPath,
|
||||
|
||||
@@ -22,6 +22,7 @@ const FAILED_WITHOUT_WARNINGS: ReplayTestAttemptOutcome = {
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'tap failed' },
|
||||
artifactPaths: [],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
};
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'request canceled' },
|
||||
artifactPaths: [],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
});
|
||||
await replaySettled;
|
||||
@@ -142,6 +143,7 @@ test('runReplayTestAttempt keeps a passing replay passed when finalization fails
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'failed to stop recording' },
|
||||
artifactPaths: [],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
}),
|
||||
cleanupSession,
|
||||
@@ -167,6 +169,7 @@ test('runReplayTestAttempt marks a failed cleanup as infrastructure so the sched
|
||||
status: 'failed',
|
||||
error: { code: 'COMMAND_FAILED', message: 'open "System Settings" failed' },
|
||||
artifactPaths: [],
|
||||
warnings: [],
|
||||
infrastructure: false,
|
||||
}),
|
||||
cleanupSession,
|
||||
|
||||
@@ -388,16 +388,21 @@ function buildReplayTestFailedResult(
|
||||
attempts: outcome.attempts,
|
||||
artifactsDir: context.testArtifactsDir,
|
||||
error,
|
||||
...(attemptOutcome?.status === 'failed' && (attemptOutcome.warnings?.length ?? 0) > 0
|
||||
? { warnings: [...attemptOutcome.warnings!] }
|
||||
: {}),
|
||||
...(attemptOutcome?.status === 'failed' && attemptOutcome.infrastructure
|
||||
? { infrastructure: true as const }
|
||||
: {}),
|
||||
...replayTestFailedAttemptFields(attemptOutcome),
|
||||
...replayTestShardResultMetadata(shard),
|
||||
};
|
||||
}
|
||||
|
||||
function replayTestFailedAttemptFields(
|
||||
attemptOutcome: ReplayTestCaseOutcome['finalOutcome'],
|
||||
): Pick<ReplaySuiteTestFailed, 'warnings' | 'infrastructure' | 'snapshotDiagnostics'> {
|
||||
const failed = attemptOutcome?.status === 'failed' ? attemptOutcome : undefined;
|
||||
return {
|
||||
...(failed && failed.warnings.length > 0 ? { warnings: [...failed.warnings] } : {}),
|
||||
...(failed?.infrastructure ? { infrastructure: true as const } : {}),
|
||||
...(attemptOutcome?.snapshotDiagnostics
|
||||
? { snapshotDiagnostics: attemptOutcome.snapshotDiagnostics }
|
||||
: {}),
|
||||
...replayTestShardResultMetadata(shard),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ export type ReplayTestAttemptFailed = {
|
||||
error: ReplayTestAttemptError;
|
||||
artifactPaths: readonly string[];
|
||||
/** Warnings accumulated before the failing step (skipped `optional` steps, capture degradations). */
|
||||
warnings?: readonly string[];
|
||||
warnings: readonly string[];
|
||||
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
|
||||
/**
|
||||
* The host's verdict that this failure is environmental (device/runner/boot) rather than a
|
||||
@@ -276,6 +276,7 @@ export type ReplayTestExecutionDependencies = Omit<
|
||||
export function replayTestAttemptFailure(params: {
|
||||
error: ReplayTestAttemptError;
|
||||
artifactPaths?: readonly string[];
|
||||
warnings?: readonly string[];
|
||||
infrastructure?: boolean;
|
||||
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
|
||||
}): ReplayTestAttemptFailed {
|
||||
@@ -283,6 +284,7 @@ export function replayTestAttemptFailure(params: {
|
||||
status: 'failed',
|
||||
error: params.error,
|
||||
artifactPaths: params.artifactPaths ?? [],
|
||||
warnings: params.warnings ?? [],
|
||||
infrastructure: params.infrastructure ?? false,
|
||||
...(params.snapshotDiagnostics ? { snapshotDiagnostics: params.snapshotDiagnostics } : {}),
|
||||
};
|
||||
|
||||
@@ -54,14 +54,14 @@ export function printHumanError(
|
||||
}
|
||||
}
|
||||
|
||||
function readResponseWarnings(details: Record<string, unknown> | undefined): string[] {
|
||||
export function readResponseWarnings(details: Record<string, unknown> | undefined): string[] {
|
||||
const warnings = details?.warnings;
|
||||
return Array.isArray(warnings)
|
||||
? warnings.filter((warning): warning is string => typeof warning === 'string')
|
||||
: [];
|
||||
}
|
||||
|
||||
function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
|
||||
export function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
|
||||
return views.flatMap((view) => {
|
||||
if (view.kind === 'element-match') {
|
||||
const remaining = view.matches - view.candidates.length;
|
||||
|
||||
@@ -25,18 +25,18 @@ test('failed attempt outcome carries warnings from the error details (#2560)', (
|
||||
});
|
||||
});
|
||||
|
||||
test('failed attempt outcome omits absent or empty warnings', () => {
|
||||
test('failed attempt outcome reads an empty warnings array when absent or non-string', () => {
|
||||
const withoutWarnings = toReplayTestAttemptOutcome({
|
||||
ok: false,
|
||||
error: { code: 'COMMAND_FAILED', message: 'step failed' },
|
||||
});
|
||||
expect('warnings' in withoutWarnings).toBe(false);
|
||||
expect(withoutWarnings.status === 'failed' && withoutWarnings.warnings).toEqual([]);
|
||||
|
||||
const emptyWarnings = toReplayTestAttemptOutcome({
|
||||
ok: false,
|
||||
error: { code: 'COMMAND_FAILED', message: 'step failed', details: { warnings: [7] } },
|
||||
});
|
||||
expect('warnings' in emptyWarnings).toBe(false);
|
||||
expect(emptyWarnings.status === 'failed' && emptyWarnings.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
test('passed attempt outcome keeps reading warnings from response data', () => {
|
||||
|
||||
@@ -14,12 +14,11 @@ import type { ReplayTestAttemptFailed, ReplayTestAttemptOutcome } from '@agent-d
|
||||
*/
|
||||
export function toReplayTestAttemptOutcome(response: DaemonResponse): ReplayTestAttemptOutcome {
|
||||
if (!response.ok) {
|
||||
const warnings = readStringArray(response.error.details?.warnings);
|
||||
return {
|
||||
status: 'failed',
|
||||
error: response.error,
|
||||
artifactPaths: readArtifactPaths(response.error.details?.artifactPaths),
|
||||
...(warnings.length > 0 ? { warnings } : {}),
|
||||
warnings: readStringArray(response.error.details?.warnings),
|
||||
infrastructure: isReplayInfrastructureFailure(response),
|
||||
...snapshotDiagnostics(response.error.details?.snapshotDiagnostics),
|
||||
};
|
||||
|
||||
@@ -36,6 +36,22 @@ test('formatToolErrorText omits the candidates block for non-ambiguous errors',
|
||||
const text = formatToolErrorText(normalizeToolError(err));
|
||||
|
||||
assert.equal(text.includes('Candidates:'), false);
|
||||
assert.equal(text.includes('Warning:'), false);
|
||||
});
|
||||
|
||||
// #2560: a failed `replay` carries the run's accumulated warnings at error level;
|
||||
// the MCP reader must see them too, not only --json consumers.
|
||||
test('formatToolErrorText renders run-level warnings carried in error details', () => {
|
||||
const err = new AppError('REPLAY_DIVERGENCE', 'Replay failed at step 2 (tapOn "Save")', {
|
||||
warnings: ['Optional Maestro assertVisible skipped at line 1: no match'],
|
||||
});
|
||||
|
||||
const text = formatToolErrorText(normalizeToolError(err));
|
||||
|
||||
assert.match(
|
||||
text,
|
||||
/^Error \(REPLAY_DIVERGENCE\)[\s\S]*\nWarning: Optional Maestro assertVisible skipped at line 1: no match/,
|
||||
);
|
||||
});
|
||||
|
||||
test('formatToolErrorText renders a structured cause', () => {
|
||||
|
||||
+5
-22
@@ -1,10 +1,11 @@
|
||||
import {
|
||||
normalizeError,
|
||||
readErrorCandidateViews,
|
||||
type ErrorCandidateView,
|
||||
type NormalizedError,
|
||||
} from '@agent-device/kernel/errors';
|
||||
import { formatReplayDivergenceReport } from '@agent-device/ad-replay/divergence';
|
||||
import { formatErrorCandidateViews, readResponseWarnings } from '../commands/output/error.ts';
|
||||
import { collapseWarningText } from '../commands/output-common.ts';
|
||||
|
||||
export function normalizeToolError(error: unknown): NormalizedError {
|
||||
return normalizeError(error);
|
||||
@@ -17,30 +18,12 @@ export function formatToolErrorText(normalized: NormalizedError): string {
|
||||
lines.push(`Cause: ${code}${normalized.cause.message}`);
|
||||
}
|
||||
if (normalized.hint) lines.push(`Hint: ${normalized.hint}`);
|
||||
for (const warning of readResponseWarnings(normalized.details)) {
|
||||
lines.push(`Warning: ${collapseWarningText(warning)}`);
|
||||
}
|
||||
lines.push(...formatErrorCandidateViews(readErrorCandidateViews(normalized.details)));
|
||||
if (normalized.supportedOn) lines.push(`Supported on: ${normalized.supportedOn}`);
|
||||
const divergence = formatReplayDivergenceReport(normalized.details);
|
||||
if (divergence) lines.push(divergence);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function formatErrorCandidateViews(views: ErrorCandidateView[]): string[] {
|
||||
return views.flatMap((view) => {
|
||||
if (view.kind === 'element-match') {
|
||||
const remaining = view.matches - view.candidates.length;
|
||||
return [
|
||||
'Candidates:',
|
||||
...view.candidates.map(
|
||||
(candidate) => ` ${pinCandidateLine(candidate, view.refsGeneration)}`,
|
||||
),
|
||||
...(remaining > 0 ? [` +${remaining} more`] : []),
|
||||
];
|
||||
}
|
||||
return ['Devices:', ...view.devices.map((device) => ` ${device.id} ${device.name}`)];
|
||||
});
|
||||
}
|
||||
|
||||
function pinCandidateLine(candidate: string, generation: number | undefined): string {
|
||||
if (generation === undefined) return candidate;
|
||||
return candidate.replace(/^@(e\d+)(?=\s|$)/, `@$1~s${generation}`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user