From 73bcab39b07a06bbf57ce079f72b8cb064236bb3 Mon Sep 17 00:00:00 2001 From: Charles Wiltgen Date: Sat, 5 Sep 2026 10:49:43 -0700 Subject: [PATCH] fix(scripts): regenerate Codex on version bump and close the version-parity class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit set-version.js regenerated the Cursor variant but never Codex, so axiom-codex/.codex-plugin/plugin.json kept the PREVIOUS version through a version bump — invisibly. Gate 12f compares skills/ and agents/ mtimes against that manifest and a pure version bump touches neither, while the version-parity gate (check 8) covered Cursor's manifest but not Codex's. Confirmed by reverting the manifest to the prior version and watching Phase 1 pass green. Root package.json had the identical hole: written by set-version.js, but read by pre-deploy only for the `pi` manifest block, never into the parity map. Also confirmed empirically. `pi install` resolves against that file. - set-version.js now runs build-cursor.ts AND build-codex.ts - --tag preflight absolves axiom-codex/ output via a new codex-output.js, mirroring cursor-output.js (regression the fix itself introduced: the preflight would have refused on ~350 files the same run had just written) - check 8 reads every version carrier from one table; parse wrapped in try/catch because build-codex rmSync's and rebuilds in place with no staging - version-parity.test.ts fails if a carrier is added without a matching read - methodology-leak.test.ts keeps behavioral-test methodology out of .claude/rules/skill-development.md, which the harness appends to every skill-file read and therefore reaches a test's GREEN arm but never its control Parity gate now reports 9 files, up from 7. Unit suite 488/488 (was 478). Reviewed with code-reviewer; the root package.json hole was its finding. --- scripts/codex-output.js | 29 ++++++++ scripts/codex-output.test.ts | 54 +++++++++++++++ scripts/cursor-output.js | 8 ++- scripts/cursor-output.test.ts | 3 + scripts/methodology-leak.test.ts | 80 ++++++++++++++++++++++ scripts/pre-deploy.ts | 42 ++++++++++-- scripts/set-version.js | 36 +++++++--- scripts/version-parity.test.ts | 113 +++++++++++++++++++++++++++++++ 8 files changed, 345 insertions(+), 20 deletions(-) create mode 100644 scripts/codex-output.js create mode 100644 scripts/codex-output.test.ts create mode 100644 scripts/methodology-leak.test.ts create mode 100644 scripts/version-parity.test.ts diff --git a/scripts/codex-output.js b/scripts/codex-output.js new file mode 100644 index 00000000..3fe5a608 --- /dev/null +++ b/scripts/codex-output.js @@ -0,0 +1,29 @@ +// Which repository paths the Codex build owns. +// +// `build-codex.ts` writes exactly one root: the `axiom-codex/` plugin tree. It +// does NOT touch `.agents/plugins/marketplace.json` (the Codex marketplace +// manifest is version-free and hand-maintained). Kept in one place so the two +// sites that must agree cannot drift: +// - set-version.js — the --tag preflight, which since 2026-09-05 runs +// build-codex and must not treat the output it just wrote as an unrelated +// dirty-tree change +// - this module's test, which pins the boundary +// +// Deliberately narrow. The preflight's job is refusing to tag a dirty tree, so +// this absolves only paths the script itself regenerates — never a stray sibling +// like `axiom-codex-notes.md`. + +const CODEX_PLUGIN_ROOT = 'axiom-codex/'; + +/** + * True when `relativePath` is regenerated by `npm run build:codex`. + * + * @param {string} relativePath repository-relative path, either separator style + * @returns {boolean} + */ +export function isCodexGeneratedPath(relativePath) { + if (typeof relativePath !== 'string' || relativePath === '') return false; + return relativePath.replaceAll('\\', '/').startsWith(CODEX_PLUGIN_ROOT); +} + +export { CODEX_PLUGIN_ROOT }; diff --git a/scripts/codex-output.test.ts b/scripts/codex-output.test.ts new file mode 100644 index 00000000..f35405d1 --- /dev/null +++ b/scripts/codex-output.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { isCodexGeneratedPath } from "./codex-output.js"; +import { isCursorGeneratedPath } from "./cursor-output.js"; + +test("recognises every path the Codex build regenerates", () => { + for (const generated of [ + "axiom-codex/README.md", + "axiom-codex/.codex-plugin/plugin.json", + "axiom-codex/skills/axiom-swiftui/SKILL.md", + "axiom-codex/skills/axiom-media/skills/music-library.md", + "axiom-codex/hooks/user-prompt-submit.py", + ]) { + assert.equal(isCodexGeneratedPath(generated), true, generated); + } +}); + +test("does not absolve unrelated working-tree changes", () => { + // The --tag preflight exists to refuse tagging a dirty tree. Widening it for + // generated output must not turn it into a blanket pass. + for (const unrelated of [ + "scripts/set-version.js", + "docs/start/codex-install.md", + ".claude-plugin/plugins/axiom/agents/build-fixer.md", + "axiom-codex-notes.md", + // build-codex.ts does NOT write the Codex marketplace manifest — it is + // version-free and hand-maintained (see .claude/rules/version-management.md). + ".agents/plugins/marketplace.json", + "", + ]) { + assert.equal(isCodexGeneratedPath(unrelated), false, unrelated); + } +}); + +test("normalises Windows separators", () => { + assert.equal(isCodexGeneratedPath("axiom-codex\\README.md"), true); +}); + +test("the two generated-output predicates are disjoint", () => { + // set-version's --tag preflight ORs these. If they ever overlapped, a path + // could be absolved by the wrong owner and the narrowing intent would rot. + for (const p of [ + "axiom-cursor/.cursor-plugin/plugin.json", + ".cursor-plugin/marketplace.json", + "axiom-codex/.codex-plugin/plugin.json", + "axiom-codex/skills/axiom-media/SKILL.md", + ]) { + assert.equal( + isCursorGeneratedPath(p) && isCodexGeneratedPath(p), + false, + `${p} claimed by both predicates`, + ); + } +}); diff --git a/scripts/cursor-output.js b/scripts/cursor-output.js index c556f6d3..14220480 100644 --- a/scripts/cursor-output.js +++ b/scripts/cursor-output.js @@ -8,8 +8,12 @@ // - this module's test, which pins the boundary // // Deliberately narrow. The preflight's job is refusing to tag a dirty tree, so -// this absolves only paths the script itself regenerates — never a sibling like -// `axiom-codex/` or a stray `axiom-cursor-notes.md`. +// this absolves only paths the script itself regenerates — never a stray sibling +// like `axiom-cursor-notes.md`. +// +// `axiom-codex/` is NOT this module's business, but it is no longer "unrelated" +// either: since 2026-09-05 set-version also runs build-codex, so the preflight +// composes this predicate with `isCodexGeneratedPath` from `codex-output.js`. const CURSOR_MARKETPLACE_PATH = '.cursor-plugin/marketplace.json'; const CURSOR_PLUGIN_ROOT = 'axiom-cursor/'; diff --git a/scripts/cursor-output.test.ts b/scripts/cursor-output.test.ts index 85b030a0..a1c7b7ae 100644 --- a/scripts/cursor-output.test.ts +++ b/scripts/cursor-output.test.ts @@ -23,6 +23,9 @@ test("does not absolve unrelated working-tree changes", () => { "docs/start/cursor-install.md", "CURSOR-MARKETPLACE-SUBMISSION.md", ".claude-plugin/plugins/axiom/agents/build-fixer.md", + // Not Cursor output, so false here is correct. Since 2026-09-05 it IS + // regenerated by set-version, but that is `isCodexGeneratedPath`'s job — + // the --tag preflight composes the two predicates. See codex-output.test.ts. "axiom-codex/skills/axiom-swiftui/SKILL.md", "axiom-cursor-notes.md", ".cursor-plugin/other.json", diff --git a/scripts/methodology-leak.test.ts b/scripts/methodology-leak.test.ts new file mode 100644 index 00000000..c421da30 --- /dev/null +++ b/scripts/methodology-leak.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +/** + * `.claude/rules/skill-development.md` is appended to the tool result of any Read + * under `.claude-plugin/plugins/axiom/skills/` by the harness. Only the GREEN arm of + * a behavioral test reads a skill file, so anything in that file reaches the treatment + * arm and never the control. + * + * Asymmetric delivery is what turns a leak into a confound — content sensitivity is + * secondary. `router-validation.md` carries RED/GREEN language too, but it arrives via + * the claudeMd block to BOTH arms, so it biases nothing and is deliberately not checked + * here. + * + * Measured 2026-09-05 (`Axiom-kud`): with the scenario templates and the tester + * watch-list in this file, GREEN agents disclosed having seen the exact scenario type + * they were about to face. After relocating both to + * `.claude/skills/preflight/skills/behavioral-testing.md`, a verification agent + * confirmed neither reaches a subject, and rated inferable-test-awareness from the + * Read result alone at 1/3 (protocol named by location only) rather than 3/3. + * + * This test fails if that payload comes back. + */ +const LEAKING_FILE = ".claude/rules/skill-development.md"; +const CANONICAL_FILE = ".claude/skills/preflight/skills/behavioral-testing.md"; + +/** Payload that lets a subject recognise the manipulation being applied to it. */ +const FORBIDDEN_IN_LEAKING_FILE: ReadonlyArray = [ + ["pressure-scenario type names", /\b(sunk cost|scope creep|existential threat)\b/i], + ["authority/time pressure scenario rows", /\|\s*(authority|time) pressure\s*\|/i], + ["tester watch-list phrases", /nonisolated\(unsafe\)|Ship quick fix|Use sleep\(\) for now/i], + ["letter-grade rubric", /\|\s*A\+?\s*\|.*\|/], + ["scenario template scaffold", /\*\*Pressure\*\*:|\*\*Expected with skill\*\*:|Anti-pattern without skill/i], +]; + +test("the auto-surfaced rules file carries no behavioral-test methodology", () => { + const text = fs.readFileSync(path.join(root, LEAKING_FILE), "utf8"); + for (const [label, pattern] of FORBIDDEN_IN_LEAKING_FILE) { + assert.equal( + pattern.test(text), + false, + `${LEAKING_FILE} contains ${label}. It is appended to every skill-file Read, so ` + + `this reaches a behavioral test's GREEN arm and never its RED control. ` + + `Move it to ${CANONICAL_FILE}.`, + ); + } +}); + +test("the canonical protocol file still holds that methodology", () => { + // The other half of the invariant: relocation, not deletion. If someone "fixes" + // the test above by deleting the content outright, this fails. + const text = fs.readFileSync(path.join(root, CANONICAL_FILE), "utf8"); + for (const [label, pattern] of [ + ["pressure-scenario type names", /sunk cost/i], + ["tester watch-list phrases", /nonisolated\(unsafe\)/i], + ["letter-grade rubric", /\|\s*A\+\s*\|/], + ] as const) { + assert.ok(pattern.test(text), `${CANONICAL_FILE} lost its ${label} — relocate, do not delete`); + } +}); + +test("pointers in the rules file stay bare", () => { + // Explaining the fix in the leaking file re-creates the leak: a subject that reads + // why the protocol moved learns that arms exist and how they differ. Verified — + // the first attempt at this fix did exactly that and was caught by re-measurement. + const text = fs.readFileSync(path.join(root, LEAKING_FILE), "utf8"); + for (const forbidden of [/GREEN arm/i, /RED arm/i, /treatment arm/i, /control arm/i, /contaminat/i]) { + assert.equal( + forbidden.test(text), + false, + `${LEAKING_FILE} explains the test-arm structure. Keep pointers bare; ` + + `put rationale in ${CANONICAL_FILE}.`, + ); + } +}); diff --git a/scripts/pre-deploy.ts b/scripts/pre-deploy.ts index 02853430..77d82af2 100644 --- a/scripts/pre-deploy.ts +++ b/scripts/pre-deploy.ts @@ -616,13 +616,41 @@ if (fs.existsSync(mcpPkgPath)) { versions["axiom-mcp/package.json"] = mcpPkg.version; } -const cursorPluginManifestPath = path.join(root, "axiom-cursor/.cursor-plugin/plugin.json"); -if (fs.existsSync(cursorPluginManifestPath)) { - const cursorPlugin = JSON.parse(fs.readFileSync(cursorPluginManifestPath, "utf8")); - if (typeof cursorPlugin.version === "string") versions["axiom-cursor/.cursor-plugin/plugin.json"] = cursorPlugin.version; - else error("version", "Cursor plugin manifest has no string version"); -} else { - error("version", "Cursor plugin manifest not found — run: npm run build:cursor"); +// Every generated variant manifest that CARRIES a version must be listed here. +// +// Codex was missing until 2026-09-05 and the omission was invisible: gate 12f +// (Codex staleness) compares skill/agent mtimes against the manifest, and a pure +// version bump touches neither, so the Codex manifest could sit at the PREVIOUS +// version through a fully green Phase 1. Confirmed by reverting it and watching +// the suite pass. Root package.json had the identical hole — it is written by +// set-version.js:448 but was only ever read here for the `pi` manifest block, +// and `pi install` resolves against it. +// +// Adding a version-carrying file? Add it to this list AND to the expected-key +// assertion in scripts/version-parity.test.ts, which fails if they diverge. +const VERSION_CARRYING_FILES: ReadonlyArray = [ + ["Cursor plugin manifest", "axiom-cursor/.cursor-plugin/plugin.json", "npm run build:cursor"], + ["Codex plugin manifest", "axiom-codex/.codex-plugin/plugin.json", "npm run build:codex"], + ["root package.json", "package.json", "node scripts/set-version.js "], +]; + +for (const [label, relPath, rebuild] of VERSION_CARRYING_FILES) { + const abs = path.join(root, relPath); + if (!fs.existsSync(abs)) { + error("version", `${label} not found at ${relPath} — run: ${rebuild}`); + continue; + } + // build-codex.ts rmSync's its output tree and rebuilds in place with no + // staging, so a half-written manifest is a live failure mode. Report it as a + // structured error naming the file rather than aborting the gate on a bare + // SyntaxError (CLAUDE.md E-2/E-4). + try { + const parsed = JSON.parse(fs.readFileSync(abs, "utf8")); + if (typeof parsed.version === "string") versions[relPath] = parsed.version; + else error("version", `${label} (${relPath}) has no string version`); + } catch (e: unknown) { + error("json", `${label} (${relPath}) unreadable: ${(e as Error).message}`); + } } const versionValues = Object.values(versions); diff --git a/scripts/set-version.js b/scripts/set-version.js index 8c1dbc01..916dad00 100755 --- a/scripts/set-version.js +++ b/scripts/set-version.js @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'; import { execSync } from 'node:child_process'; import { VERSION_RE, VERSION_CORE } from './version-regex.js'; import { isCursorGeneratedPath } from './cursor-output.js'; +import { isCodexGeneratedPath } from './codex-output.js'; import { DOC_STAT_FILES, docStatValues, applyDocStats, checkMarkerSpec } from './doc-stats.js'; import { isGeneratedSubSkill } from './inline-auditors.ts'; import { manifestSkillsFromDisk } from './skill-listing.ts'; @@ -543,10 +544,14 @@ try { throw new Error(`--tag requires a git repository: ${err.message}`); } const dirtyFiles = status.split('\n').filter(Boolean).map(l => l.slice(3)); - // The Cursor distribution is regenerated below, after these writes, so the - // preflight would otherwise refuse on output this script is about to produce - // itself. Only paths build:cursor owns are absolved; everything else still blocks. - const unexpected = dirtyFiles.filter(f => !expectedRelative.has(f) && !isCursorGeneratedPath(f)); + // The Cursor AND Codex distributions are regenerated below, after these writes, + // so the preflight would otherwise refuse on output this script is about to + // produce itself. Only paths those two builds own are absolved; everything else + // still blocks. (Codex joined this carve-out on 2026-09-05, when set-version + // started regenerating it — before that it was correctly treated as a sibling.) + const unexpected = dirtyFiles.filter( + (f) => !expectedRelative.has(f) && !isCursorGeneratedPath(f) && !isCodexGeneratedPath(f), + ); if (unexpected.length) { throw new Error( `--tag refused: working tree has unrelated changes. Commit or stash them first:\n ` + @@ -587,13 +592,22 @@ try { throw err; } - // Cursor manifests and reports are generated from the canonical manifest. - // Keep this after the atomic canonical writes: a generation failure leaves a - // truthful diagnostic rather than silently shipping a stale Cursor release. - try { - execSync('node scripts/build-cursor.ts', { cwd: root, stdio: 'inherit' }); - } catch (err) { - throw new Error(`canonical version changed but Cursor output is stale: ${err.message}`); + // Cursor and Codex variants embed the version in their own plugin manifests, so + // both go stale on a bump and would ship version-mismatched. Keep these after the + // atomic canonical writes: a generation failure leaves a truthful diagnostic + // rather than silently shipping a stale variant. + // + // Codex was previously omitted here. Nothing caught it: pre-deploy's Codex + // staleness gate (12f) compares skill/agent mtimes against the manifest, and a + // pure version bump touches neither — so `axiom-codex/.codex-plugin/plugin.json` + // sat at the OLD version through a fully green `npm test`. Verified 2026-09-05 by + // reverting the manifest to the prior version and watching Phase 1 pass. + for (const [label, script] of [['Cursor', 'build-cursor.ts'], ['Codex', 'build-codex.ts']]) { + try { + execSync(`node scripts/${script}`, { cwd: root, stdio: 'inherit' }); + } catch (err) { + throw new Error(`canonical version changed but ${label} output is stale: ${err.message}`); + } } // Create annotated tag (after successful writes) if --tag passed diff --git a/scripts/version-parity.test.ts b/scripts/version-parity.test.ts new file mode 100644 index 00000000..53e65f84 --- /dev/null +++ b/scripts/version-parity.test.ts @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +/** + * Every file in the repo that carries the Axiom plugin version. + * + * This list is the CLASS, and it exists because the class kept leaking one + * member at a time. Two independent instances were found on 2026-09-05, both + * fully green under the whole gate suite while desynced: + * + * - `axiom-codex/.codex-plugin/plugin.json` — written only by build-codex.ts, + * which set-version.js did not run. Gate 12f (Codex staleness) compares + * skill/agent mtimes against the manifest and a version bump touches + * neither, so it could not fire. + * - `package.json` (root) — written by set-version.js, but pre-deploy read it + * only for the `pi` manifest block, never into the version-parity map. + * `pi install git:github.com/CharlesWiltgen/Axiom` resolves against it. + * + * A file that carries the version but is absent from pre-deploy's check 8 is + * invisible to every gate. This test fails when the two drift. + */ +const VERSION_CARRYING_FILES = [ + ".claude-plugin/plugins/axiom/claude-code.json", + ".claude-plugin/plugins/axiom/.claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + ".claude-plugin/plugins/axiom/hooks/metadata.txt", + "docs/.vitepress/config.ts", + "axiom-mcp/package.json", + "axiom-cursor/.cursor-plugin/plugin.json", + "axiom-codex/.codex-plugin/plugin.json", + "package.json", +] as const; + +function canonicalVersion(): string { + const manifest = JSON.parse( + fs.readFileSync(path.join(root, ".claude-plugin/plugins/axiom/claude-code.json"), "utf8"), + ); + return manifest.version; +} + +/** Read the version out of a file, whatever shape it stores it in. */ +function versionOf(relPath: string): string | undefined { + const abs = path.join(root, relPath); + if (!fs.existsSync(abs)) return undefined; + const raw = fs.readFileSync(abs, "utf8"); + + if (relPath.endsWith("metadata.txt")) return raw.trim().split("\n")[0]; + if (relPath.endsWith("config.ts")) return raw.match(/• v([0-9][^\s"'`<]*)/)?.[1]; + if (relPath === ".claude-plugin/marketplace.json") { + return JSON.parse(raw).plugins?.find((p: { name: string }) => p.name === "axiom")?.version; + } + return JSON.parse(raw).version; +} + +test("every version-carrying file exists", () => { + for (const relPath of VERSION_CARRYING_FILES) { + assert.ok( + fs.existsSync(path.join(root, relPath)), + `${relPath} is missing — regenerate the variant that owns it`, + ); + } +}); + +test("every version-carrying file matches the canonical version", () => { + const canonical = canonicalVersion(); + for (const relPath of VERSION_CARRYING_FILES) { + assert.equal( + versionOf(relPath), + canonical, + `${relPath} is out of sync with claude-code.json (${canonical})`, + ); + } +}); + +test("pre-deploy check 8 covers exactly the version-carrying files", () => { + // The guard that actually closes the class: a carrier added to the list above + // without a matching read in check 8 — or dropped from check 8 — fails here + // rather than shipping green. + // + // Check 8 populates `versions` two ways, so count both: direct literal-key + // assignments (`versions["claude-code.json"] = …`, which use short display + // labels rather than repo paths, so a path-substring match cannot see them), + // plus the VERSION_CARRYING_FILES table it loops over. + const preDeploy = fs.readFileSync(path.join(root, "scripts/pre-deploy.ts"), "utf8"); + + const directKeys = [...preDeploy.matchAll(/\bversions\[\s*"([^"]+)"\s*\]\s*=/g)].map((m) => m[1]); + const tableBlock = preDeploy.match( + /const VERSION_CARRYING_FILES[\s\S]*?\n\];/, + )?.[0]; + assert.ok(tableBlock, "pre-deploy.ts no longer declares VERSION_CARRYING_FILES"); + const tableEntries = [...tableBlock.matchAll(/\[\s*"[^"]+",\s*"([^"]+)"/g)].map((m) => m[1]); + + const covered = new Set([...directKeys, ...tableEntries]); + assert.equal( + covered.size, + VERSION_CARRYING_FILES.length, + `check 8 reads ${covered.size} version carriers but this test lists ` + + `${VERSION_CARRYING_FILES.length}. Covered: ${[...covered].sort().join(", ")}`, + ); + + // The table half must match by real repo path; the direct half uses labels. + for (const relPath of tableEntries) { + assert.ok( + (VERSION_CARRYING_FILES as readonly string[]).includes(relPath), + `check 8 reads ${relPath}, which this test does not list as a version carrier`, + ); + } +});