test(ios): own snapshot differential on macOS

This commit is contained in:
Michał Pierzchała
2026-09-01 14:31:19 +02:00
parent 5cd3ce8eb3
commit cbb3dae97a
5 changed files with 67 additions and 40 deletions
+4
View File
@@ -70,6 +70,10 @@ jobs:
uses: ./.github/actions/run-gate
with: { gate: macos-coverage }
- name: Run iOS snapshot Swift/TypeScript differential
uses: ./.github/actions/run-gate
with: { gate: ios-snapshot-differential }
- name: Restore and build macOS XCTest runner
uses: ./.github/actions/setup-apple-runner-build
with:
+1
View File
@@ -117,6 +117,7 @@
"maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts",
"maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs",
"maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts",
"test:ios-snapshot-differential": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts",
"size": "node scripts/size-report.mjs",
"perf": "node --experimental-strip-types scripts/perf/run.ts",
"mutation:run": "node --experimental-strip-types scripts/mutation/run.ts",
+6
View File
@@ -46,6 +46,12 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [
gate('package', 'Published package (publint, attw, clean-install resolution)', 'check:package'),
gate('integration-node', 'Node integration smoke', 'test:integration:node'),
gate('macos-coverage', 'macOS command coverage manifest', 'test:integration:macos-coverage'),
gate(
'ios-snapshot-differential',
'iOS snapshot Swift/TypeScript differential',
'test:ios-snapshot-differential',
false,
),
{
id: 'vitest-related',
label: 'Tests related by Vitest module graph',
+11
View File
@@ -46,6 +46,7 @@ export type CheckId =
| 'provider-integration'
| 'integration-node'
| 'macos-coverage'
| 'ios-snapshot-differential'
| 'integration-progress'
| 'swift-runner-ios'
| 'swift-runner-macos'
@@ -105,6 +106,7 @@ export const ALL_CHECKS: readonly CheckId[] = [
// run before the related-project workload heats the host.
'integration-node',
'macos-coverage',
'ios-snapshot-differential',
'vitest-related',
'unit',
'unit-ci',
@@ -435,6 +437,15 @@ const BUILD_OWNERSHIP: ReadonlyArray<{
detail: string;
owns: (file: string) => boolean;
}> = [
{
check: 'ios-snapshot-differential',
rule: 'own:ios-snapshot-differential',
detail: 'the required macOS lane runs the Swift/TypeScript snapshot differential',
owns: (file) =>
file.startsWith('packages/capture-kit/src/ios-snapshot-engine/') ||
file.startsWith('apple/snapshot-presentation/') ||
file === 'contracts/fixtures/ios-snapshot-engine-conformance.json',
},
// Both platform builds compile the same runner sources, and each is a separate
// gate in a separate lane, so a Swift change owns both.
{
@@ -1,14 +1,14 @@
import assert from 'node:assert/strict';
import { test } from 'node:test';
import fc from 'fast-check';
import { test } from 'vitest';
import {
compareDifferentialCases,
swiftToolchainAvailable,
SWIFT_RUN_TIMEOUT_MS,
writeDifferentialFailureArtifact,
} from './conformance-harness.ts';
import { differentialBatchArbitrary } from './conformance-generator.ts';
import { readIosSnapshotEngineFixture } from './conformance-fixture.ts';
} from '../packages/capture-kit/src/ios-snapshot-engine/conformance-harness.ts';
import { differentialBatchArbitrary } from '../packages/capture-kit/src/ios-snapshot-engine/conformance-generator.ts';
import { readIosSnapshotEngineFixture } from '../packages/capture-kit/src/ios-snapshot-engine/conformance-fixture.ts';
type DifferentialCase = Parameters<typeof compareDifferentialCases>[0][number];
@@ -16,9 +16,11 @@ const FUZZ_SEEDS = [219101, 219102, 219103, 219104];
const RUNS_PER_SEED = 8;
const MAX_TOTAL_DURATION_MS = 60_000;
const differentialTest = swiftToolchainAvailable() ? test : test.skip;
if (!swiftToolchainAvailable()) {
throw new Error('iOS snapshot differential requires the macOS Swift toolchain');
}
differentialTest('authored Swift and TypeScript golden cases agree', () => {
test('authored Swift and TypeScript golden cases agree', { timeout: SWIFT_RUN_TIMEOUT_MS }, () => {
const fixture = readIosSnapshotEngineFixture();
const cases = fixture.cases
.filter((testCase) => testCase.swift && !testCase.interactiveOnly)
@@ -36,41 +38,44 @@ differentialTest('authored Swift and TypeScript golden cases agree', () => {
assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : '');
});
differentialTest('raw unscoped depth compares the same acquisition frontier', () => {
const fixture = readIosSnapshotEngineFixture();
const depthCase = fixture.cases.find(
(testCase) => testCase.name === 'raw unscoped depth uses the acquisition frontier',
);
assert.ok(depthCase);
const deepNode = depthCase.nodes.at(-1);
assert.ok(deepNode);
const mismatch = compareDifferentialCases([
{
name: 'raw-depth-frontier-with-malformed-tail',
projection: 'raw',
interactiveOnly: false,
depth: 1,
scope: null,
foldPolicy: 'cursor-projected',
viewport: fixture.viewport,
nodes: [...depthCase.nodes, { ...deepNode, index: 1, parentIndex: 1, depth: 2 }],
},
]);
assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : '');
});
test(
'raw unscoped depth compares the same acquisition frontier',
{ timeout: SWIFT_RUN_TIMEOUT_MS },
() => {
const fixture = readIosSnapshotEngineFixture();
const depthCase = fixture.cases.find(
(testCase) => testCase.name === 'raw unscoped depth uses the acquisition frontier',
);
assert.ok(depthCase);
const deepNode = depthCase.nodes.at(-1);
assert.ok(deepNode);
const mismatch = compareDifferentialCases([
{
name: 'raw-depth-frontier-with-malformed-tail',
projection: 'raw',
interactiveOnly: false,
depth: 1,
scope: null,
foldPolicy: 'cursor-projected',
viewport: fixture.viewport,
nodes: [...depthCase.nodes, { ...deepNode, index: 1, parentIndex: 1, depth: 2 }],
},
]);
assert.equal(mismatch, undefined, mismatch ? JSON.stringify(mismatch, null, 2) : '');
},
);
differentialTest('deterministic Swift/TypeScript differential fuzz stays under 60000ms', () => {
assertDifferentialFuzz();
});
function assertDifferentialFuzz(): void {
const startedAt = performance.now();
for (const seed of FUZZ_SEEDS) {
assertDifferentialSeed(seed);
assertWithinKillCriterion(startedAt, seed);
}
assert.ok(true);
}
test(
'deterministic Swift/TypeScript differential fuzz stays under 60000ms',
{ timeout: SWIFT_RUN_TIMEOUT_MS * FUZZ_SEEDS.length },
() => {
const startedAt = performance.now();
for (const seed of FUZZ_SEEDS) {
assertDifferentialSeed(seed);
assertWithinKillCriterion(startedAt, seed);
}
},
);
function assertDifferentialSeed(seed: number): void {
const result = fc.check(