fix(scripts): regenerate Codex on version bump and close the version-parity class

set-version.js regenerated the Cursor variant but never Codex, so
axiom-codex/.codex-plugin/plugin.json kept the PREVIOUS version through a
version bump — invisibly. Gate 12f compares skills/ and agents/ mtimes against
that manifest and a pure version bump touches neither, while the version-parity
gate (check 8) covered Cursor's manifest but not Codex's. Confirmed by reverting
the manifest to the prior version and watching Phase 1 pass green.

Root package.json had the identical hole: written by set-version.js, but read by
pre-deploy only for the `pi` manifest block, never into the parity map. Also
confirmed empirically. `pi install` resolves against that file.

- set-version.js now runs build-cursor.ts AND build-codex.ts
- --tag preflight absolves axiom-codex/ output via a new codex-output.js,
  mirroring cursor-output.js (regression the fix itself introduced: the
  preflight would have refused on ~350 files the same run had just written)
- check 8 reads every version carrier from one table; parse wrapped in
  try/catch because build-codex rmSync's and rebuilds in place with no staging
- version-parity.test.ts fails if a carrier is added without a matching read
- methodology-leak.test.ts keeps behavioral-test methodology out of
  .claude/rules/skill-development.md, which the harness appends to every
  skill-file read and therefore reaches a test's GREEN arm but never its control

Parity gate now reports 9 files, up from 7. Unit suite 488/488 (was 478).
Reviewed with code-reviewer; the root package.json hole was its finding.
This commit is contained in:
Charles Wiltgen
2026-09-05 10:49:43 -07:00
parent 6f552b66ae
commit 73bcab39b0
8 changed files with 345 additions and 20 deletions
+29
View File
@@ -0,0 +1,29 @@
// Which repository paths the Codex build owns.
//
// `build-codex.ts` writes exactly one root: the `axiom-codex/` plugin tree. It
// does NOT touch `.agents/plugins/marketplace.json` (the Codex marketplace
// manifest is version-free and hand-maintained). Kept in one place so the two
// sites that must agree cannot drift:
// - set-version.js — the --tag preflight, which since 2026-09-05 runs
// build-codex and must not treat the output it just wrote as an unrelated
// dirty-tree change
// - this module's test, which pins the boundary
//
// Deliberately narrow. The preflight's job is refusing to tag a dirty tree, so
// this absolves only paths the script itself regenerates — never a stray sibling
// like `axiom-codex-notes.md`.
const CODEX_PLUGIN_ROOT = 'axiom-codex/';
/**
* True when `relativePath` is regenerated by `npm run build:codex`.
*
* @param {string} relativePath repository-relative path, either separator style
* @returns {boolean}
*/
export function isCodexGeneratedPath(relativePath) {
if (typeof relativePath !== 'string' || relativePath === '') return false;
return relativePath.replaceAll('\\', '/').startsWith(CODEX_PLUGIN_ROOT);
}
export { CODEX_PLUGIN_ROOT };
+54
View File
@@ -0,0 +1,54 @@
import assert from "node:assert/strict";
import test from "node:test";
import { isCodexGeneratedPath } from "./codex-output.js";
import { isCursorGeneratedPath } from "./cursor-output.js";
test("recognises every path the Codex build regenerates", () => {
for (const generated of [
"axiom-codex/README.md",
"axiom-codex/.codex-plugin/plugin.json",
"axiom-codex/skills/axiom-swiftui/SKILL.md",
"axiom-codex/skills/axiom-media/skills/music-library.md",
"axiom-codex/hooks/user-prompt-submit.py",
]) {
assert.equal(isCodexGeneratedPath(generated), true, generated);
}
});
test("does not absolve unrelated working-tree changes", () => {
// The --tag preflight exists to refuse tagging a dirty tree. Widening it for
// generated output must not turn it into a blanket pass.
for (const unrelated of [
"scripts/set-version.js",
"docs/start/codex-install.md",
".claude-plugin/plugins/axiom/agents/build-fixer.md",
"axiom-codex-notes.md",
// build-codex.ts does NOT write the Codex marketplace manifest — it is
// version-free and hand-maintained (see .claude/rules/version-management.md).
".agents/plugins/marketplace.json",
"",
]) {
assert.equal(isCodexGeneratedPath(unrelated), false, unrelated);
}
});
test("normalises Windows separators", () => {
assert.equal(isCodexGeneratedPath("axiom-codex\\README.md"), true);
});
test("the two generated-output predicates are disjoint", () => {
// set-version's --tag preflight ORs these. If they ever overlapped, a path
// could be absolved by the wrong owner and the narrowing intent would rot.
for (const p of [
"axiom-cursor/.cursor-plugin/plugin.json",
".cursor-plugin/marketplace.json",
"axiom-codex/.codex-plugin/plugin.json",
"axiom-codex/skills/axiom-media/SKILL.md",
]) {
assert.equal(
isCursorGeneratedPath(p) && isCodexGeneratedPath(p),
false,
`${p} claimed by both predicates`,
);
}
});
+6 -2
View File
@@ -8,8 +8,12 @@
// - this module's test, which pins the boundary
//
// Deliberately narrow. The preflight's job is refusing to tag a dirty tree, so
// this absolves only paths the script itself regenerates — never a sibling like
// `axiom-codex/` or a stray `axiom-cursor-notes.md`.
// this absolves only paths the script itself regenerates — never a stray sibling
// like `axiom-cursor-notes.md`.
//
// `axiom-codex/` is NOT this module's business, but it is no longer "unrelated"
// either: since 2026-09-05 set-version also runs build-codex, so the preflight
// composes this predicate with `isCodexGeneratedPath` from `codex-output.js`.
const CURSOR_MARKETPLACE_PATH = '.cursor-plugin/marketplace.json';
const CURSOR_PLUGIN_ROOT = 'axiom-cursor/';
+3
View File
@@ -23,6 +23,9 @@ test("does not absolve unrelated working-tree changes", () => {
"docs/start/cursor-install.md",
"CURSOR-MARKETPLACE-SUBMISSION.md",
".claude-plugin/plugins/axiom/agents/build-fixer.md",
// Not Cursor output, so false here is correct. Since 2026-09-05 it IS
// regenerated by set-version, but that is `isCodexGeneratedPath`'s job —
// the --tag preflight composes the two predicates. See codex-output.test.ts.
"axiom-codex/skills/axiom-swiftui/SKILL.md",
"axiom-cursor-notes.md",
".cursor-plugin/other.json",
+80
View File
@@ -0,0 +1,80 @@
import assert from "node:assert/strict";
import test from "node:test";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
/**
* `.claude/rules/skill-development.md` is appended to the tool result of any Read
* under `.claude-plugin/plugins/axiom/skills/` by the harness. Only the GREEN arm of
* a behavioral test reads a skill file, so anything in that file reaches the treatment
* arm and never the control.
*
* Asymmetric delivery is what turns a leak into a confound — content sensitivity is
* secondary. `router-validation.md` carries RED/GREEN language too, but it arrives via
* the claudeMd block to BOTH arms, so it biases nothing and is deliberately not checked
* here.
*
* Measured 2026-09-05 (`Axiom-kud`): with the scenario templates and the tester
* watch-list in this file, GREEN agents disclosed having seen the exact scenario type
* they were about to face. After relocating both to
* `.claude/skills/preflight/skills/behavioral-testing.md`, a verification agent
* confirmed neither reaches a subject, and rated inferable-test-awareness from the
* Read result alone at 1/3 (protocol named by location only) rather than 3/3.
*
* This test fails if that payload comes back.
*/
const LEAKING_FILE = ".claude/rules/skill-development.md";
const CANONICAL_FILE = ".claude/skills/preflight/skills/behavioral-testing.md";
/** Payload that lets a subject recognise the manipulation being applied to it. */
const FORBIDDEN_IN_LEAKING_FILE: ReadonlyArray<readonly [label: string, pattern: RegExp]> = [
["pressure-scenario type names", /\b(sunk cost|scope creep|existential threat)\b/i],
["authority/time pressure scenario rows", /\|\s*(authority|time) pressure\s*\|/i],
["tester watch-list phrases", /nonisolated\(unsafe\)|Ship quick fix|Use sleep\(\) for now/i],
["letter-grade rubric", /\|\s*A\+?\s*\|.*\|/],
["scenario template scaffold", /\*\*Pressure\*\*:|\*\*Expected with skill\*\*:|Anti-pattern without skill/i],
];
test("the auto-surfaced rules file carries no behavioral-test methodology", () => {
const text = fs.readFileSync(path.join(root, LEAKING_FILE), "utf8");
for (const [label, pattern] of FORBIDDEN_IN_LEAKING_FILE) {
assert.equal(
pattern.test(text),
false,
`${LEAKING_FILE} contains ${label}. It is appended to every skill-file Read, so ` +
`this reaches a behavioral test's GREEN arm and never its RED control. ` +
`Move it to ${CANONICAL_FILE}.`,
);
}
});
test("the canonical protocol file still holds that methodology", () => {
// The other half of the invariant: relocation, not deletion. If someone "fixes"
// the test above by deleting the content outright, this fails.
const text = fs.readFileSync(path.join(root, CANONICAL_FILE), "utf8");
for (const [label, pattern] of [
["pressure-scenario type names", /sunk cost/i],
["tester watch-list phrases", /nonisolated\(unsafe\)/i],
["letter-grade rubric", /\|\s*A\+\s*\|/],
] as const) {
assert.ok(pattern.test(text), `${CANONICAL_FILE} lost its ${label} — relocate, do not delete`);
}
});
test("pointers in the rules file stay bare", () => {
// Explaining the fix in the leaking file re-creates the leak: a subject that reads
// why the protocol moved learns that arms exist and how they differ. Verified —
// the first attempt at this fix did exactly that and was caught by re-measurement.
const text = fs.readFileSync(path.join(root, LEAKING_FILE), "utf8");
for (const forbidden of [/GREEN arm/i, /RED arm/i, /treatment arm/i, /control arm/i, /contaminat/i]) {
assert.equal(
forbidden.test(text),
false,
`${LEAKING_FILE} explains the test-arm structure. Keep pointers bare; ` +
`put rationale in ${CANONICAL_FILE}.`,
);
}
});
+35 -7
View File
@@ -616,13 +616,41 @@ if (fs.existsSync(mcpPkgPath)) {
versions["axiom-mcp/package.json"] = mcpPkg.version;
}
const cursorPluginManifestPath = path.join(root, "axiom-cursor/.cursor-plugin/plugin.json");
if (fs.existsSync(cursorPluginManifestPath)) {
const cursorPlugin = JSON.parse(fs.readFileSync(cursorPluginManifestPath, "utf8"));
if (typeof cursorPlugin.version === "string") versions["axiom-cursor/.cursor-plugin/plugin.json"] = cursorPlugin.version;
else error("version", "Cursor plugin manifest has no string version");
} else {
error("version", "Cursor plugin manifest not found — run: npm run build:cursor");
// Every generated variant manifest that CARRIES a version must be listed here.
//
// Codex was missing until 2026-09-05 and the omission was invisible: gate 12f
// (Codex staleness) compares skill/agent mtimes against the manifest, and a pure
// version bump touches neither, so the Codex manifest could sit at the PREVIOUS
// version through a fully green Phase 1. Confirmed by reverting it and watching
// the suite pass. Root package.json had the identical hole — it is written by
// set-version.js:448 but was only ever read here for the `pi` manifest block,
// and `pi install` resolves against it.
//
// Adding a version-carrying file? Add it to this list AND to the expected-key
// assertion in scripts/version-parity.test.ts, which fails if they diverge.
const VERSION_CARRYING_FILES: ReadonlyArray<readonly [label: string, relPath: string, rebuild: string]> = [
["Cursor plugin manifest", "axiom-cursor/.cursor-plugin/plugin.json", "npm run build:cursor"],
["Codex plugin manifest", "axiom-codex/.codex-plugin/plugin.json", "npm run build:codex"],
["root package.json", "package.json", "node scripts/set-version.js <version>"],
];
for (const [label, relPath, rebuild] of VERSION_CARRYING_FILES) {
const abs = path.join(root, relPath);
if (!fs.existsSync(abs)) {
error("version", `${label} not found at ${relPath} — run: ${rebuild}`);
continue;
}
// build-codex.ts rmSync's its output tree and rebuilds in place with no
// staging, so a half-written manifest is a live failure mode. Report it as a
// structured error naming the file rather than aborting the gate on a bare
// SyntaxError (CLAUDE.md E-2/E-4).
try {
const parsed = JSON.parse(fs.readFileSync(abs, "utf8"));
if (typeof parsed.version === "string") versions[relPath] = parsed.version;
else error("version", `${label} (${relPath}) has no string version`);
} catch (e: unknown) {
error("json", `${label} (${relPath}) unreadable: ${(e as Error).message}`);
}
}
const versionValues = Object.values(versions);
+25 -11
View File
@@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
import { execSync } from 'node:child_process';
import { VERSION_RE, VERSION_CORE } from './version-regex.js';
import { isCursorGeneratedPath } from './cursor-output.js';
import { isCodexGeneratedPath } from './codex-output.js';
import { DOC_STAT_FILES, docStatValues, applyDocStats, checkMarkerSpec } from './doc-stats.js';
import { isGeneratedSubSkill } from './inline-auditors.ts';
import { manifestSkillsFromDisk } from './skill-listing.ts';
@@ -543,10 +544,14 @@ try {
throw new Error(`--tag requires a git repository: ${err.message}`);
}
const dirtyFiles = status.split('\n').filter(Boolean).map(l => l.slice(3));
// The Cursor distribution is regenerated below, after these writes, so the
// preflight would otherwise refuse on output this script is about to produce
// itself. Only paths build:cursor owns are absolved; everything else still blocks.
const unexpected = dirtyFiles.filter(f => !expectedRelative.has(f) && !isCursorGeneratedPath(f));
// The Cursor AND Codex distributions are regenerated below, after these writes,
// so the preflight would otherwise refuse on output this script is about to
// produce itself. Only paths those two builds own are absolved; everything else
// still blocks. (Codex joined this carve-out on 2026-09-05, when set-version
// started regenerating it — before that it was correctly treated as a sibling.)
const unexpected = dirtyFiles.filter(
(f) => !expectedRelative.has(f) && !isCursorGeneratedPath(f) && !isCodexGeneratedPath(f),
);
if (unexpected.length) {
throw new Error(
`--tag refused: working tree has unrelated changes. Commit or stash them first:\n ` +
@@ -587,13 +592,22 @@ try {
throw err;
}
// Cursor manifests and reports are generated from the canonical manifest.
// Keep this after the atomic canonical writes: a generation failure leaves a
// truthful diagnostic rather than silently shipping a stale Cursor release.
try {
execSync('node scripts/build-cursor.ts', { cwd: root, stdio: 'inherit' });
} catch (err) {
throw new Error(`canonical version changed but Cursor output is stale: ${err.message}`);
// Cursor and Codex variants embed the version in their own plugin manifests, so
// both go stale on a bump and would ship version-mismatched. Keep these after the
// atomic canonical writes: a generation failure leaves a truthful diagnostic
// rather than silently shipping a stale variant.
//
// Codex was previously omitted here. Nothing caught it: pre-deploy's Codex
// staleness gate (12f) compares skill/agent mtimes against the manifest, and a
// pure version bump touches neither — so `axiom-codex/.codex-plugin/plugin.json`
// sat at the OLD version through a fully green `npm test`. Verified 2026-09-05 by
// reverting the manifest to the prior version and watching Phase 1 pass.
for (const [label, script] of [['Cursor', 'build-cursor.ts'], ['Codex', 'build-codex.ts']]) {
try {
execSync(`node scripts/${script}`, { cwd: root, stdio: 'inherit' });
} catch (err) {
throw new Error(`canonical version changed but ${label} output is stale: ${err.message}`);
}
}
// Create annotated tag (after successful writes) if --tag passed
+113
View File
@@ -0,0 +1,113 @@
import assert from "node:assert/strict";
import test from "node:test";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
/**
* Every file in the repo that carries the Axiom plugin version.
*
* This list is the CLASS, and it exists because the class kept leaking one
* member at a time. Two independent instances were found on 2026-09-05, both
* fully green under the whole gate suite while desynced:
*
* - `axiom-codex/.codex-plugin/plugin.json` — written only by build-codex.ts,
* which set-version.js did not run. Gate 12f (Codex staleness) compares
* skill/agent mtimes against the manifest and a version bump touches
* neither, so it could not fire.
* - `package.json` (root) — written by set-version.js, but pre-deploy read it
* only for the `pi` manifest block, never into the version-parity map.
* `pi install git:github.com/CharlesWiltgen/Axiom` resolves against it.
*
* A file that carries the version but is absent from pre-deploy's check 8 is
* invisible to every gate. This test fails when the two drift.
*/
const VERSION_CARRYING_FILES = [
".claude-plugin/plugins/axiom/claude-code.json",
".claude-plugin/plugins/axiom/.claude-plugin/plugin.json",
".claude-plugin/marketplace.json",
".claude-plugin/plugins/axiom/hooks/metadata.txt",
"docs/.vitepress/config.ts",
"axiom-mcp/package.json",
"axiom-cursor/.cursor-plugin/plugin.json",
"axiom-codex/.codex-plugin/plugin.json",
"package.json",
] as const;
function canonicalVersion(): string {
const manifest = JSON.parse(
fs.readFileSync(path.join(root, ".claude-plugin/plugins/axiom/claude-code.json"), "utf8"),
);
return manifest.version;
}
/** Read the version out of a file, whatever shape it stores it in. */
function versionOf(relPath: string): string | undefined {
const abs = path.join(root, relPath);
if (!fs.existsSync(abs)) return undefined;
const raw = fs.readFileSync(abs, "utf8");
if (relPath.endsWith("metadata.txt")) return raw.trim().split("\n")[0];
if (relPath.endsWith("config.ts")) return raw.match(/• v([0-9][^\s"'`<]*)/)?.[1];
if (relPath === ".claude-plugin/marketplace.json") {
return JSON.parse(raw).plugins?.find((p: { name: string }) => p.name === "axiom")?.version;
}
return JSON.parse(raw).version;
}
test("every version-carrying file exists", () => {
for (const relPath of VERSION_CARRYING_FILES) {
assert.ok(
fs.existsSync(path.join(root, relPath)),
`${relPath} is missing — regenerate the variant that owns it`,
);
}
});
test("every version-carrying file matches the canonical version", () => {
const canonical = canonicalVersion();
for (const relPath of VERSION_CARRYING_FILES) {
assert.equal(
versionOf(relPath),
canonical,
`${relPath} is out of sync with claude-code.json (${canonical})`,
);
}
});
test("pre-deploy check 8 covers exactly the version-carrying files", () => {
// The guard that actually closes the class: a carrier added to the list above
// without a matching read in check 8 — or dropped from check 8 — fails here
// rather than shipping green.
//
// Check 8 populates `versions` two ways, so count both: direct literal-key
// assignments (`versions["claude-code.json"] = …`, which use short display
// labels rather than repo paths, so a path-substring match cannot see them),
// plus the VERSION_CARRYING_FILES table it loops over.
const preDeploy = fs.readFileSync(path.join(root, "scripts/pre-deploy.ts"), "utf8");
const directKeys = [...preDeploy.matchAll(/\bversions\[\s*"([^"]+)"\s*\]\s*=/g)].map((m) => m[1]);
const tableBlock = preDeploy.match(
/const VERSION_CARRYING_FILES[\s\S]*?\n\];/,
)?.[0];
assert.ok(tableBlock, "pre-deploy.ts no longer declares VERSION_CARRYING_FILES");
const tableEntries = [...tableBlock.matchAll(/\[\s*"[^"]+",\s*"([^"]+)"/g)].map((m) => m[1]);
const covered = new Set([...directKeys, ...tableEntries]);
assert.equal(
covered.size,
VERSION_CARRYING_FILES.length,
`check 8 reads ${covered.size} version carriers but this test lists ` +
`${VERSION_CARRYING_FILES.length}. Covered: ${[...covered].sort().join(", ")}`,
);
// The table half must match by real repo path; the direct half uses labels.
for (const relPath of tableEntries) {
assert.ok(
(VERSION_CARRYING_FILES as readonly string[]).includes(relPath),
`check 8 reads ${relPath}, which this test does not list as a version carrier`,
);
}
});