From 4257671ceefaa4825094c684250151acf388897e Mon Sep 17 00:00:00 2001 From: Charles Wiltgen Date: Wed, 16 Sep 2026 14:54:16 -0700 Subject: [PATCH] fix(ci): skip the dev-state guard tests in a checkout that has no dev state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Test Suite workflow has failed on every push since it was added. Three tests in scripts/methodology-leak.test.ts read `.claude/rules/skill-development.md` and `.claude/skills/preflight/skills/behavioral-testing.md` and die with ENOENT in CI, because `.claude/` is gitignored: those files exist on a maintainer's machine and nowhere else. Locally the suite is 518/518; in a fresh checkout it was 515/518. The tests themselves are sound — they guard a real invariant, that behavioral-test methodology stays out of the file the harness appends to every skill-file Read, so it cannot reach a GREEN arm and confound a behavioral test. The subjects are just local dev state, so the precondition is "this checkout has dev state" and the tests skip when it does not. They still run wherever the state exists, which is the only place the leak they guard against could actually occur. The second test deliberately gates on the presence of the RULES file rather than its own subject: deleting the canonical file while the rules file remains is the "fixed it by deleting the content" regression it exists to catch, and it must fail then, not skip. Verified in three checkouts: dev state present 3 pass / 0 skip; no dev state 3 skip / 0 fail; rules file present with the canonical file deleted 1 fail. Before the change, the first two were 3 fail and the third was 3 fail. The workflow did its job here — it is the first thing to run this suite outside a maintainer's machine, and it found a suite that could not pass outside one. --- scripts/methodology-leak.test.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/methodology-leak.test.ts b/scripts/methodology-leak.test.ts index c421da30..8fe53cb7 100644 --- a/scripts/methodology-leak.test.ts +++ b/scripts/methodology-leak.test.ts @@ -29,6 +29,21 @@ const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const LEAKING_FILE = ".claude/rules/skill-development.md"; const CANONICAL_FILE = ".claude/skills/preflight/skills/behavioral-testing.md"; +/** + * Both subjects live under `.claude/`, which is gitignored local dev state. In a fresh + * checkout — CI, or anyone else's clone — neither file exists, so there is nothing to + * guard and these tests skip instead of failing on ENOENT. They run wherever the dev + * state is present, which is the only place a behavioral test could be exposed to the + * leak they exist to catch. + * + * Skipped per test, not as a group: deleting the canonical file while the rules file + * remains is exactly the "fixed it by deleting the content" regression the second test + * is here to catch, and it must still fail in that case. + */ +const has = (rel: string) => fs.existsSync(path.join(root, rel)); +const absent = (rel: string) => + has(rel) ? false : `${rel} is not present in this checkout (gitignored dev state)`; + /** Payload that lets a subject recognise the manipulation being applied to it. */ const FORBIDDEN_IN_LEAKING_FILE: ReadonlyArray = [ ["pressure-scenario type names", /\b(sunk cost|scope creep|existential threat)\b/i], @@ -38,7 +53,7 @@ const FORBIDDEN_IN_LEAKING_FILE: ReadonlyArray { +test("the auto-surfaced rules file carries no behavioral-test methodology", { skip: absent(LEAKING_FILE) }, () => { const text = fs.readFileSync(path.join(root, LEAKING_FILE), "utf8"); for (const [label, pattern] of FORBIDDEN_IN_LEAKING_FILE) { assert.equal( @@ -51,7 +66,7 @@ test("the auto-surfaced rules file carries no behavioral-test methodology", () = } }); -test("the canonical protocol file still holds that methodology", () => { +test("the canonical protocol file still holds that methodology", { skip: absent(LEAKING_FILE) }, () => { // The other half of the invariant: relocation, not deletion. If someone "fixes" // the test above by deleting the content outright, this fails. const text = fs.readFileSync(path.join(root, CANONICAL_FILE), "utf8"); @@ -64,7 +79,7 @@ test("the canonical protocol file still holds that methodology", () => { } }); -test("pointers in the rules file stay bare", () => { +test("pointers in the rules file stay bare", { skip: absent(LEAKING_FILE) }, () => { // Explaining the fix in the leaking file re-creates the leak: a subject that reads // why the protocol moved learns that arms exist and how they differ. Verified — // the first attempt at this fix did exactly that and was caught by re-measurement.