mirror of
https://github.com/zernie/vigiles.git
synced 2026-09-14 20:53:57 +08:00
feat(site): sell the test and eval tiers with measured OSS findings (#218)
Render the test and eval tiers as measured findings rather than feature bullets: a real vendored OSS guard's own verdicts through runHook, and measureTriggerRate on superpowers' TDD skill at 83% over 30 runs. Both render from committed fixtures, one regenerated from the engine and pinned by a browser test, the other copied from the lock so CI spends no model quota. Adds a /comparison page rendered from a probe snapshot, retires four unmeasured claims plus two shipped-but-"planned" rows, and type-checks rendered snippets against the real .d.ts instead of scanning them with a regex. Committing this repo's first eval lock also revealed that the CI eval-check step had never once executed — it short-circuited on anyLocksCommitted and exited 0. Two defects behind it are fixed: a quota consent gate that refused a model-free verify, and a dogfood step that now names the one eval holding a lock.
This commit is contained in:
zernie
+10
-10
@@ -51,22 +51,22 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": "Edit|Write|MultiEdit",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx vigiles hook-runtime run-program .vigiles/hooks/test-tier-nudge.hook.mjs"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": "Edit|Write|MultiEdit",
|
||||
"matcher": "Edit|Write",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx vigiles hook-runtime run-program .vigiles/hooks/docs-drift-nudge.hook.mjs"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": "Edit|Write",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx vigiles hook-runtime run-program .vigiles/hooks/test-tier-nudge.hook.mjs"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PreToolUse": [
|
||||
|
||||
@@ -352,10 +352,19 @@ jobs:
|
||||
fi
|
||||
echo "vigiles action output: valid=$got"
|
||||
|
||||
# Dogfood the eval staleness gate via the composite Action. This repo's
|
||||
# real-model evals run locally (`npm run test:eval`), so no lock is
|
||||
# committed yet — `eval-check` is a green no-op here, but it proves the
|
||||
# action's `command: eval-check` → `eval --check` path works in CI.
|
||||
# Dogfood the eval staleness gate via the composite Action. It now VERIFIES
|
||||
# a real committed lock instead of no-opping: `paths` names the one eval
|
||||
# that has one, which is what the gate's own error message prescribes for
|
||||
# an eval without a lock ("exclude it from the --check run").
|
||||
#
|
||||
# 🔴 It was a NO-OP FOR ITS WHOLE LIFE, and nobody could tell. With no lock
|
||||
# committed anywhere, `eval --check` returned early on `anyLocksCommitted`
|
||||
# and printed "nothing to verify" — exit 0. The first commit of a lock got
|
||||
# past that early return and the step failed twice over: once on a quota
|
||||
# consent that should never gate a verify (fixed in run-scripts.ts), then
|
||||
# on the 22 evals that are unnamed and unlocked. Both were invisible while
|
||||
# the step never ran. Widen `paths` as evals earn locks; a bare `--check`
|
||||
# here fails until every discovered eval has a name AND a committed lock.
|
||||
#
|
||||
# 🔴 Kept OUT of `npm run check` on purpose, and nearly lost by accident when
|
||||
# the consolidation below was written: it sat between two command-steps, so a
|
||||
@@ -365,6 +374,7 @@ jobs:
|
||||
uses: ./
|
||||
with:
|
||||
command: eval-check
|
||||
paths: examples/harness/skill-trigger-rate.eval.mjs
|
||||
version: local
|
||||
comment: "false"
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ site/blob-report/
|
||||
# gitignored by site/.gitignore) — same class: building the site locally would
|
||||
# otherwise redden `fmt:check` on files nobody commits.
|
||||
site/checks/
|
||||
site/comparison/
|
||||
|
||||
# Ecosystem-benchmark run artifacts (machine-written JSON) + vendored upstream
|
||||
# SKILL.md snapshots (pinned by SHA, formatted by their upstreams, not us).
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "superpowers-tdd-trigger-rate",
|
||||
"inputsHash": "b45c96b31c5a0b45",
|
||||
"model": "sonnet",
|
||||
"harnessVersionKey": "2.1",
|
||||
"evalApiVersion": 1,
|
||||
"builtAt": "2026-09-09T01:38:23.831Z",
|
||||
"report": {
|
||||
"rate": 0.8333333333333334,
|
||||
"n": 30,
|
||||
"perPrompt": [
|
||||
{
|
||||
"prompt": "Add an `isEven(n)` function to utils.js — write it test-first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Implement a stack class in stack.js. Use TDD.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Fix the off-by-one in paginate(); add a regression test first.",
|
||||
"fired": 0,
|
||||
"trials": 3,
|
||||
"rate": 0
|
||||
},
|
||||
{
|
||||
"prompt": "Build a small LRU cache, driving it with failing tests first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Add a `slugify` helper — red/green/refactor please.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Write a rate limiter; start from the tests and work outward.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Implement currency rounding with a test-first approach.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Add retry-with-backoff to the API client, tests leading.",
|
||||
"fired": 1,
|
||||
"trials": 3,
|
||||
"rate": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"prompt": "Create a debounce utility; specify behaviour as tests first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Parse ISO durations into seconds — write the spec before the code.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
}
|
||||
],
|
||||
"competitors": 1,
|
||||
"namespace": "superpowers",
|
||||
"usage": {
|
||||
"totalCostUsd": 1.6588511999999997,
|
||||
"meanCostUsd": 0.05529503999999999,
|
||||
"meanDurationMs": 8713.566666666668,
|
||||
"totalInputTokens": 150,
|
||||
"totalOutputTokens": 12317,
|
||||
"totalCacheCreationTokens": 233925,
|
||||
"totalCacheReadTokens": 2998406
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+3
-1
@@ -101,7 +101,7 @@ declare module "vigiles/generated" {
|
||||
| "internal:check"
|
||||
| "docs:api";
|
||||
|
||||
/** 433 project files. */
|
||||
/** 434 project files. */
|
||||
export type ProjectFile =
|
||||
| "src/CLAUDE.md"
|
||||
| "src/CLAUDE.md.spec.ts"
|
||||
@@ -209,6 +209,7 @@ declare module "vigiles/generated" {
|
||||
| "src/colocated-execution.test.ts"
|
||||
| "src/community-skills.test.ts"
|
||||
| "src/community-skills.ts"
|
||||
| "src/comparison-snapshot.test.ts"
|
||||
| "src/core/CLAUDE.md"
|
||||
| "src/core/CLAUDE.md.spec.ts"
|
||||
| "src/core/adapter.ts"
|
||||
@@ -714,6 +715,7 @@ declare module "vigiles/spec" {
|
||||
| "src/colocated-execution.test.ts"
|
||||
| "src/community-skills.test.ts"
|
||||
| "src/community-skills.ts"
|
||||
| "src/comparison-snapshot.test.ts"
|
||||
| "src/core/CLAUDE.md"
|
||||
| "src/core/CLAUDE.md.spec.ts"
|
||||
| "src/core/adapter.ts"
|
||||
|
||||
@@ -93,7 +93,7 @@ const REMINDER =
|
||||
|
||||
export default experimental_defineReact({
|
||||
on: "PostToolUse",
|
||||
match: tools("Edit", "Write", "MultiEdit"),
|
||||
match: tools("Edit", "Write"),
|
||||
needs: [state("docs.followed"), state("docs.nudged")],
|
||||
react: (e) => {
|
||||
// `rel` is the repo-relative answer; `raw` is the fallback for a path with
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
"file": ".vigiles/hooks/docs-drift-nudge.hook.mjs",
|
||||
"stamp": "36477a2399726428"
|
||||
"stamp": "c618516d6e73d671"
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ function reminder(tier, path) {
|
||||
|
||||
export default experimental_defineReact({
|
||||
on: "PostToolUse",
|
||||
match: tools("Edit", "Write", "MultiEdit"),
|
||||
match: tools("Edit", "Write"),
|
||||
needs: [state("tier.reminded")],
|
||||
react: (e) => {
|
||||
// `rel` is the repo-relative answer `under()` would have used; `raw` is the
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{
|
||||
"file": ".vigiles/hooks/test-tier-nudge.hook.mjs",
|
||||
"stamp": "a97af128e29a60e7"
|
||||
"stamp": "125432160514c839"
|
||||
}
|
||||
|
||||
+6
-1
@@ -44,5 +44,10 @@
|
||||
"orphans": {
|
||||
"include": ["docs/**/*.md"]
|
||||
},
|
||||
"exclude": ["bench/**", "startup/**", "rule-enforcer/**"]
|
||||
"exclude": [
|
||||
".claude/worktrees/**",
|
||||
"bench/**",
|
||||
"startup/**",
|
||||
"rule-enforcer/**"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
<!-- vigiles:sha256:318853f246869cd2 compiled from CLAUDE.md.spec.ts -->
|
||||
<!-- vigiles:sha256:3c6c2542fa6a49c8 compiled from CLAUDE.md.spec.ts -->
|
||||
|
||||
# CLAUDE.md
|
||||
|
||||
## Positioning
|
||||
|
||||
vigiles verifies the rule references in agent instruction files — that each linter rule exists AND is enabled, that file paths and scripts are real. ⚠️ ADOPTION DIRECTION (committed 2026-07-15): audit-FIRST, not spec-first. The markdown the user hand-edits is the SOURCE OF TRUTH; enforcement of code-quality rules lands in the repo's NATIVE linter config (ruff/eslint), not the spec; the typed `.spec.ts` is an OPTIONAL authoring layer for harness-STRUCTURE rules only (subagent contracts, purity, railway, composition — what no linter can express). `init` is the GRADUATION to a spec, not the front door (`audit` is — see below); `adopt`/`strengthen` are skills, not verbs. The first pass writes nothing and asserts nothing false (ref-verification is advisory, never written into the file). `eject` always reverses. The rule: the spec AUTHORS, the native linter RUNS (same architecture as `@vigiles/rule-enforcer`) — Rule of Least Power applied to enforcement homes. [LEGACY, being retired: the earlier spec-first three-level ladder — inline/frontmatter/typed — and `init` adopting a CLAUDE.md into a spec-as-source-of-truth with the markdown as a build artifact. Treat as current-shipped, NOT the target.] Nobody else does this — other tools lint markdown after the fact. See `docs/markdown-mode.md`.
|
||||
vigiles verifies the rule references in agent instruction files — that each linter rule exists AND is enabled, that file paths and scripts are real. ⚠️ ADOPTION DIRECTION (committed 2026-07-15): audit-FIRST, not spec-first. The markdown the user hand-edits is the SOURCE OF TRUTH; enforcement of code-quality rules lands in the repo's NATIVE linter config (ruff/eslint), not the spec; the typed `.spec.ts` is an OPTIONAL authoring layer for harness-STRUCTURE rules only (subagent contracts, purity, railway, composition — what no linter can express). `init` is the GRADUATION to a spec, not the front door (`audit` is — see below); `adopt`/`strengthen` are skills, not verbs. The first pass writes nothing and asserts nothing false (ref-verification is advisory, never written into the file). `eject` always reverses. The rule: the spec AUTHORS, the native linter RUNS (same architecture as `@vigiles/rule-enforcer`) — Rule of Least Power applied to enforcement homes. [LEGACY, being retired: the earlier spec-first three-level ladder — inline/frontmatter/typed — and `init` adopting a CLAUDE.md into a spec-as-source-of-truth with the markdown as a build artifact. Treat as current-shipped, NOT the target.] See `docs/markdown-mode.md`.
|
||||
|
||||
Positioned in the harness engineering frame coined early 2026: Agent = Model + Harness. The harness has two enforcement modes — probabilistic compliance (prompts, instructions) and deterministic constraints (linters, types, hooks). vigiles is the deterministic-constraints layer for instruction files.
|
||||
|
||||
@@ -12,7 +12,7 @@ The sharper, structural form of "deterministic constraints" is the CATEGORY line
|
||||
|
||||
THE ADOPTION FRONT DOOR is `vigiles audit` — "Lighthouse for your harness": one zero-config command → four DETERMINISTIC category RINGS (Truthfulness/Triggering/Structure/Tested, weighted A–F) + each finding's fix inline + a shareable HTML report. A plain `audit` is a DETERMINISTIC READ — safe to run anywhere (even a prod-wired repo), IDENTICAL on every OS, nothing executes. It is a LOCAL report (like Lighthouse), NOT a CI step — CI uses `vigiles lint`. The TWO executing checks — live MCP resolution (do referenced tools resolve on the real server?) + trigger-rate (do your skills FIRE?) — sit behind ONE CONSENT (the read-vs-run axis, `src/scan-trigger-suggest.ts` `decideExecute`): at a TTY `audit` ASKS ONCE (a bundled prompt that DISCLOSES confinement + cost) and REMEMBERS in `.vigilesrc.json` (`audit.measure`); headless (`--json`/CI/non-interactive/agent) it stays a read + a one-line nudge (never hangs, never silently executes). There is deliberately NO execution flag: `audit` is a LOCAL report (like Lighthouse), NOT a CI step — CI uses `vigiles lint` (the deterministic gate). It runs the executing checks only when a human can consent; AUTOMATION tests the harness via the `vigiles` testing API + skills, never the report verb. The founder-driven simplification (2026-06-27) collapsed the earlier `--deep`/`--measure`/`--fast` flag sprawl into this ONE read-vs-run consent, because a uniform interactive choice beats per-tier toggles. WHY these two are opt-in: live MCP STARTS your own servers (a real backend connection) and trigger-rate spends model quota — neither is unsafe, but both DO something beyond a read, so a plain `audit` never does them without a human's yes. On consent: LIVE MCP is own-repo only (never a stranger's server) because STARTING a server connects to a backend and deny-all-net would break the `tools/list` it performs; trigger-rate STUBS skill bodies so no skill PROCEDURE runs (`hasModelAccess`/`isMeteredAccess` only shape the disclosure wording, sub=$0 vs metered=credits). THE SAFETY BATTERY (do your hooks actually block?) is DELIBERATELY NOT an audit ring (narrowed 2026-06-27, founder "no half-made shit pre-release"): running ARBITRARY hooks safely needs cross-platform confinement and that's parked (bubblewrap is Linux-only; env-scrub ephemeral floor + macOS sandbox-exec are the unbuilt exit criterion) — so rather than ship a Linux-confined/Mac-unconfined ring, the battery lives in the `vigiles` testing API (`guardrail-check`/`assertBlocksDisasters`) where you opt in EXPLICITLY (a test you wrote, no zero-config-safety promise to break). audit re-promotes a Safety ring only once one confinement works the same on macOS+Linux. Everything renders FROM the versioned `AuditReport` JSON (`src/audit-report.ts`, `schemaVersion`), never from the HTML: the local React/shadcn single-file report and `audit --json` for CI. The report UI is a real Vite + React + shadcn app (`report/`) built to ONE self-contained file the CLI fills with the JSON (React runs in the reader's browser; the CLI ships only the built template + stays runtime-dep-light), and its components are presentational so any other renderer works off the same JSON contract.
|
||||
|
||||
The cross-referencing engine is what the tool is built on: `enforce("@typescript-eslint/no-floating-promises")` verifies the rule exists AND is enabled in your linter config. Same for ESLint, Ruff, Clippy, Pylint, RuboCop, Stylelint, Cedar policies (for AWS Bedrock AgentCore and other Cedar-using runtimes), and the JVM/Go ecosystem — detekt, ktlint, Checkstyle, golangci-lint. The rule name is resolved against each linter's own catalog rather than matched as a string, and the enabled state is read from the project's config. (An exclusivity claim stood here until 2026-09-08; it rested on a documentation-checked competitor matrix, and `no-product-strategy-here` forbids competitive positioning in this public repo.)
|
||||
The cross-referencing engine is what the tool is built on: `enforce("@typescript-eslint/no-floating-promises")` verifies the rule exists AND is enabled in your linter config. Same for ESLint, Ruff, Clippy, Pylint, RuboCop, Stylelint, Cedar policies (for AWS Bedrock AgentCore and other Cedar-using runtimes), and the JVM/Go ecosystem — detekt, ktlint, Checkstyle, golangci-lint. The rule name is resolved against each linter's own catalog rather than matched as a string, and the enabled state is read from the project's config. (Two exclusivity claims stood in this section until 2026-09-08 — one here, one in the adoption-direction paragraph above. Both rested on a documentation-checked competitor matrix rather than a run, and `no-product-strategy-here` forbids competitive positioning in this public repo. Recorded rather than silently dropped, so neither is reintroduced.)
|
||||
|
||||
Authoring-time feedback comes two ways: `generate-types` emits a `.d.ts` so the TS compiler PROVES `.spec.ts` references at edit time, and `generate-schema` emits a JSON Schema so a YAML LSP autocompletes and squiggles `vigiles:` frontmatter rule names — same guarantee, no TypeScript required. Both scan all 11 catalog APIs, package.json, and project files.
|
||||
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ import { claude, enforce, guidance, guard } from "./src/core/spec.js";
|
||||
|
||||
export default claude({
|
||||
sections: {
|
||||
positioning: `vigiles verifies the rule references in agent instruction files — that each linter rule exists AND is enabled, that file paths and scripts are real. ⚠️ ADOPTION DIRECTION (committed 2026-07-15): audit-FIRST, not spec-first. The markdown the user hand-edits is the SOURCE OF TRUTH; enforcement of code-quality rules lands in the repo's NATIVE linter config (ruff/eslint), not the spec; the typed \`.spec.ts\` is an OPTIONAL authoring layer for harness-STRUCTURE rules only (subagent contracts, purity, railway, composition — what no linter can express). \`init\` is the GRADUATION to a spec, not the front door (\`audit\` is — see below); \`adopt\`/\`strengthen\` are skills, not verbs. The first pass writes nothing and asserts nothing false (ref-verification is advisory, never written into the file). \`eject\` always reverses. The rule: the spec AUTHORS, the native linter RUNS (same architecture as \`@vigiles/rule-enforcer\`) — Rule of Least Power applied to enforcement homes. [LEGACY, being retired: the earlier spec-first three-level ladder — inline/frontmatter/typed — and \`init\` adopting a CLAUDE.md into a spec-as-source-of-truth with the markdown as a build artifact. Treat as current-shipped, NOT the target.] Nobody else does this — other tools lint markdown after the fact. See \`docs/markdown-mode.md\`.
|
||||
positioning: `vigiles verifies the rule references in agent instruction files — that each linter rule exists AND is enabled, that file paths and scripts are real. ⚠️ ADOPTION DIRECTION (committed 2026-07-15): audit-FIRST, not spec-first. The markdown the user hand-edits is the SOURCE OF TRUTH; enforcement of code-quality rules lands in the repo's NATIVE linter config (ruff/eslint), not the spec; the typed \`.spec.ts\` is an OPTIONAL authoring layer for harness-STRUCTURE rules only (subagent contracts, purity, railway, composition — what no linter can express). \`init\` is the GRADUATION to a spec, not the front door (\`audit\` is — see below); \`adopt\`/\`strengthen\` are skills, not verbs. The first pass writes nothing and asserts nothing false (ref-verification is advisory, never written into the file). \`eject\` always reverses. The rule: the spec AUTHORS, the native linter RUNS (same architecture as \`@vigiles/rule-enforcer\`) — Rule of Least Power applied to enforcement homes. [LEGACY, being retired: the earlier spec-first three-level ladder — inline/frontmatter/typed — and \`init\` adopting a CLAUDE.md into a spec-as-source-of-truth with the markdown as a build artifact. Treat as current-shipped, NOT the target.] See \`docs/markdown-mode.md\`.
|
||||
|
||||
Positioned in the harness engineering frame coined early 2026: Agent = Model + Harness. The harness has two enforcement modes — probabilistic compliance (prompts, instructions) and deterministic constraints (linters, types, hooks). vigiles is the deterministic-constraints layer for instruction files.
|
||||
|
||||
@@ -16,7 +16,7 @@ The sharper, structural form of "deterministic constraints" is the CATEGORY line
|
||||
|
||||
THE ADOPTION FRONT DOOR is \`vigiles audit\` — "Lighthouse for your harness": one zero-config command → four DETERMINISTIC category RINGS (Truthfulness/Triggering/Structure/Tested, weighted A–F) + each finding's fix inline + a shareable HTML report. A plain \`audit\` is a DETERMINISTIC READ — safe to run anywhere (even a prod-wired repo), IDENTICAL on every OS, nothing executes. It is a LOCAL report (like Lighthouse), NOT a CI step — CI uses \`vigiles lint\`. The TWO executing checks — live MCP resolution (do referenced tools resolve on the real server?) + trigger-rate (do your skills FIRE?) — sit behind ONE CONSENT (the read-vs-run axis, \`src/scan-trigger-suggest.ts\` \`decideExecute\`): at a TTY \`audit\` ASKS ONCE (a bundled prompt that DISCLOSES confinement + cost) and REMEMBERS in \`.vigilesrc.json\` (\`audit.measure\`); headless (\`--json\`/CI/non-interactive/agent) it stays a read + a one-line nudge (never hangs, never silently executes). There is deliberately NO execution flag: \`audit\` is a LOCAL report (like Lighthouse), NOT a CI step — CI uses \`vigiles lint\` (the deterministic gate). It runs the executing checks only when a human can consent; AUTOMATION tests the harness via the \`vigiles\` testing API + skills, never the report verb. The founder-driven simplification (2026-06-27) collapsed the earlier \`--deep\`/\`--measure\`/\`--fast\` flag sprawl into this ONE read-vs-run consent, because a uniform interactive choice beats per-tier toggles. WHY these two are opt-in: live MCP STARTS your own servers (a real backend connection) and trigger-rate spends model quota — neither is unsafe, but both DO something beyond a read, so a plain \`audit\` never does them without a human's yes. On consent: LIVE MCP is own-repo only (never a stranger's server) because STARTING a server connects to a backend and deny-all-net would break the \`tools/list\` it performs; trigger-rate STUBS skill bodies so no skill PROCEDURE runs (\`hasModelAccess\`/\`isMeteredAccess\` only shape the disclosure wording, sub=\$0 vs metered=credits). THE SAFETY BATTERY (do your hooks actually block?) is DELIBERATELY NOT an audit ring (narrowed 2026-06-27, founder "no half-made shit pre-release"): running ARBITRARY hooks safely needs cross-platform confinement and that's parked (bubblewrap is Linux-only; env-scrub ephemeral floor + macOS sandbox-exec are the unbuilt exit criterion) — so rather than ship a Linux-confined/Mac-unconfined ring, the battery lives in the \`vigiles\` testing API (\`guardrail-check\`/\`assertBlocksDisasters\`) where you opt in EXPLICITLY (a test you wrote, no zero-config-safety promise to break). audit re-promotes a Safety ring only once one confinement works the same on macOS+Linux. Everything renders FROM the versioned \`AuditReport\` JSON (\`src/audit-report.ts\`, \`schemaVersion\`), never from the HTML: the local React/shadcn single-file report and \`audit --json\` for CI. The report UI is a real Vite + React + shadcn app (\`report/\`) built to ONE self-contained file the CLI fills with the JSON (React runs in the reader's browser; the CLI ships only the built template + stays runtime-dep-light), and its components are presentational so any other renderer works off the same JSON contract.
|
||||
|
||||
The cross-referencing engine is what the tool is built on: \`enforce("@typescript-eslint/no-floating-promises")\` verifies the rule exists AND is enabled in your linter config. Same for ESLint, Ruff, Clippy, Pylint, RuboCop, Stylelint, Cedar policies (for AWS Bedrock AgentCore and other Cedar-using runtimes), and the JVM/Go ecosystem — detekt, ktlint, Checkstyle, golangci-lint. The rule name is resolved against each linter's own catalog rather than matched as a string, and the enabled state is read from the project's config. (An exclusivity claim stood here until 2026-09-08; it rested on a documentation-checked competitor matrix, and \`no-product-strategy-here\` forbids competitive positioning in this public repo.)
|
||||
The cross-referencing engine is what the tool is built on: \`enforce("@typescript-eslint/no-floating-promises")\` verifies the rule exists AND is enabled in your linter config. Same for ESLint, Ruff, Clippy, Pylint, RuboCop, Stylelint, Cedar policies (for AWS Bedrock AgentCore and other Cedar-using runtimes), and the JVM/Go ecosystem — detekt, ktlint, Checkstyle, golangci-lint. The rule name is resolved against each linter's own catalog rather than matched as a string, and the enabled state is read from the project's config. (Two exclusivity claims stood in this section until 2026-09-08 — one here, one in the adoption-direction paragraph above. Both rested on a documentation-checked competitor matrix rather than a run, and \`no-product-strategy-here\` forbids competitive positioning in this public repo. Recorded rather than silently dropped, so neither is reintroduced.)
|
||||
|
||||
Authoring-time feedback comes two ways: \`generate-types\` emits a \`.d.ts\` so the TS compiler PROVES \`.spec.ts\` references at edit time, and \`generate-schema\` emits a JSON Schema so a YAML LSP autocompletes and squiggles \`vigiles:\` frontmatter rule names — same guarantee, no TypeScript required. Both scan all 11 catalog APIs, package.json, and project files.
|
||||
|
||||
|
||||
+12
-12
@@ -12,22 +12,22 @@
|
||||
| **Linter config changes** | CLAUDE.md drifts out of sync | PostToolUse hook auto-regenerates types |
|
||||
| **Spec edits** | N/A | PostToolUse hook auto-compiles to markdown |
|
||||
| **guidance → enforce upgrades** | Manual guesswork | `/strengthen` reads per-linter docs, suggests upgrades |
|
||||
| **New lint rules from PR feedback** | Copy-paste from review | _(planned)_ a rule-synthesis skill generates rule + tests + spec entry |
|
||||
| **New lint rules from PR feedback** | Copy-paste from review | The `pr-to-lint-rule` skill generates the rule + tests via `@vigiles/rule-enforcer` |
|
||||
| **Does a skill/plugin actually help?** | Unknown — stars + vibes | A/B measured on real tasks: bill + correctness ([measuring-skills](measuring-skills.md)) |
|
||||
| **CI** | Nothing to verify | `vigiles lint` catches hash drift, disabled rules, stale refs |
|
||||
|
||||
## Codex
|
||||
|
||||
| | Without vigiles | With vigiles |
|
||||
| ----------------------------- | -------------------------------- | ------------------------------------------------------- |
|
||||
| **Instructions** | Hand-written AGENTS.md | Compiled from `.spec.ts` |
|
||||
| **Linter rule references** | Trust-based | Verified at compile time |
|
||||
| **File paths / commands** | Rot silently | Checked at compile time |
|
||||
| **Direct edits to AGENTS.md** | Undetected | CI catches hash mismatch |
|
||||
| **Hooks / auto-compile** | Not available (no plugin system) | Not available — run `vigiles compile` manually or in CI |
|
||||
| **CI** | Nothing to verify | Same `vigiles lint` pipeline as Claude |
|
||||
| | Without vigiles | With vigiles |
|
||||
| ----------------------------- | --------------------------------------- | -------------------------------------- |
|
||||
| **Instructions** | Hand-written AGENTS.md | Compiled from `.spec.ts` |
|
||||
| **Linter rule references** | Trust-based | Verified at compile time |
|
||||
| **File paths / commands** | Rot silently | Checked at compile time |
|
||||
| **Direct edits to AGENTS.md** | Undetected | CI catches hash mismatch |
|
||||
| **Hooks** | Hand-written `[hooks]` in `config.toml` | Compiled to that same native TOML |
|
||||
| **CI** | Nothing to verify | Same `vigiles lint` pipeline as Claude |
|
||||
|
||||
Codex has no hook or plugin system. The compile-time verification and CI enforcement still work — the difference is there's no auto-recompilation on edit. You run `vigiles compile` before committing, and CI catches drift.
|
||||
Codex reads hooks from `[hooks]` in `config.toml`, and vigiles compiles to that native format — `src/adapters/codex/codex.test.ts` asserts such a block loads with `${PLUGIN_ROOT}` expanded. What is _not_ claimed here is auto-recompilation on edit: run `vigiles compile` before committing, and CI catches drift. (This row read "Codex has no hook or plugin system" until 2026-09-09, contradicted by this repo's own adapter, which declares `shellHooks: true`.)
|
||||
|
||||
## What's Deterministic vs What's Not
|
||||
|
||||
@@ -44,7 +44,7 @@ Codex has no hook or plugin system. The compile-time verification and CI enforce
|
||||
| Duplicate rule detection | ✅ Yes | Normalized Compression Distance (NCD) with fixed threshold |
|
||||
| Orphan docs detection | ✅ Yes | Scan configured doc directories for `.md` files no other markdown references |
|
||||
| guidance → enforce suggestion | ❌ No | Agent reads linter docs, reasons about intent — `/strengthen` skill |
|
||||
| PR comment → lint rule | ❌ No | _(planned)_ agent generates custom rule code via a rule-synthesis skill |
|
||||
| PR comment → lint rule | ❌ No | `pr-to-lint-rule` drives `@vigiles/rule-enforcer` behind its blind-gold trust gate |
|
||||
| Spec content authoring | ❌ No | Agent or human writes the spec — vigiles verifies it |
|
||||
|
||||
## What vigiles Does and Doesn't Validate in Markdown
|
||||
@@ -112,7 +112,7 @@ Specs compile to `CLAUDE.md` by default; set `target: "AGENTS.md"` or
|
||||
.spec.ts ──────┤ vigiles compile │ │ /strengthen │
|
||||
│ │ ✓ linter rules exist │ │ guidance → enforce │
|
||||
│ │ ✓ rules enabled │ │ │
|
||||
│ │ ✓ file paths valid │ │ (planned) │
|
||||
│ │ ✓ file paths valid │ │ /pr-to-lint-rule │
|
||||
│ │ ✓ commands valid │ │ rule synthesis skill │
|
||||
│ │ → CLAUDE.md + hash │ │ │
|
||||
│ └─────────────────────────┘ │ /edit-spec │
|
||||
|
||||
@@ -6,8 +6,10 @@ all three is a **prompt-injection exfiltration path with no exploit code**:
|
||||
attacker-controllable content flows in, reads your private data, and ships it out
|
||||
— all driven by the model, no bug required. Same detectors `vigiles audit` uses
|
||||
(`lethalTrifectaIssues` / `skillTrifectaIssue` in
|
||||
`src/core/lethal-trifecta.ts`); no other plugin linter checks the tool **set** for
|
||||
this — competitors lint a single tool's effect, never the dangerous combination.
|
||||
`src/core/lethal-trifecta.ts`). The check is over the tool **set** a unit holds, not
|
||||
any single tool's effect. (A claim that no other plugin linter does this stood here
|
||||
until 2026-09-08. Nobody had run one; it is removed rather than reworded so it is
|
||||
not reintroduced.)
|
||||
|
||||
> ⚠️ **Subagents and skills are read from DIFFERENT fields, because they are
|
||||
> different mechanisms.** A subagent's `tools:` really does bound the unit. A
|
||||
|
||||
@@ -29,8 +29,16 @@ The **invalid-value** half cross-references the `model:` against the alias set
|
||||
(`inherit`/`sonnet`/`opus`/`haiku`) and `color:` against the color enum, flagging
|
||||
only a **close typo** (≤2 edits) — a full/dated model id (`claude-sonnet-4-5`) is
|
||||
an explicit form and left alone, and an unrecognized far-off value is suppressed
|
||||
(high-precision, no cry-wolf). This matches Anthropic's own `claude plugin
|
||||
validate` + cclint.
|
||||
(high-precision, no cry-wolf).
|
||||
|
||||
> **Correction, 2026-09-08.** This read "matches Anthropic's own `claude plugin
|
||||
validate` + cclint." **Measured:** `node tools/measure-validate-overlap.mjs`
|
||||
> against Claude Code 2.1.263 plants `model: sonnnet` in a subagent and
|
||||
> `claude plugin validate` passes it — repo-local `.claude/` and packaged plugin,
|
||||
> default and `--strict`. The **missing-required** half above IS shared (it warns on
|
||||
> a subagent with no description); the **invalid-value** half is not. cclint was
|
||||
> never run by anything in this repo, so it should not have been named at all.
|
||||
> **What would invalidate this:** a new Claude Code minor. Re-run the probe.
|
||||
|
||||
This is the rule that catches the real bug the plugin sweep found: a marketplace
|
||||
shipping subagents (`changelog-generator`, `content-creator`, …) with **no
|
||||
|
||||
@@ -28,6 +28,12 @@ const skill = "superpowers:test-driven-development";
|
||||
|
||||
export default defineEval({
|
||||
measureTriggerRate: {
|
||||
// NAMED so `vigiles eval --update` writes a committed lock. It belongs HERE,
|
||||
// on the trigger-rate spec, NOT top-level on defineEval — measured 2026-09-09:
|
||||
// a top-level `name` is silently ignored (defineEval takes unknown keys without
|
||||
// complaint) and --update prints "skipped the lock for an unnamed eval", which
|
||||
// is the same message you get for having set nothing at all.
|
||||
name: "superpowers-tdd-trigger-rate",
|
||||
pluginDir,
|
||||
stubSkillBodies: true, // trigger = frontmatter only; stub the body to stop at selection
|
||||
// Aim for >= 10 varied phrasings — measureTriggerRate runs a deterministic
|
||||
|
||||
Generated
+20
@@ -4451,6 +4451,12 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/prismjs": {
|
||||
"version": "1.26.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.6.tgz",
|
||||
"integrity": "sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/react": {
|
||||
"version": "19.2.17",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||
@@ -12747,6 +12753,19 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/prism-react-renderer": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/prism-react-renderer/-/prism-react-renderer-2.4.1.tgz",
|
||||
"integrity": "sha512-ey8Ls/+Di31eqzUxC46h8MksNuGx/n0AAC8uKpwFau4RPDYLuE3EXTp8N8G2vX2N7UC/+IXeNUnlWBGGcAG+Ig==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/prismjs": "^1.26.0",
|
||||
"clsx": "^2.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
@@ -15832,6 +15851,7 @@
|
||||
"idb-keyval": "^6.3.0",
|
||||
"lucide-react": "^0.469.0",
|
||||
"pako": "^2.2.0",
|
||||
"prism-react-renderer": "^2.4.1",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"tailwind-merge": "^2.6.0"
|
||||
|
||||
+2
-1
@@ -8,6 +8,7 @@ blob-report/
|
||||
.vitest-attachments/
|
||||
**/__screenshots__/
|
||||
|
||||
# Generated static per-check pages (built by prebuild → scripts/gen-check-pages.ts).
|
||||
# Generated static pages (built by prebuild → scripts/gen-check-pages.ts).
|
||||
# Leading slash anchors to the site root so src/checks/ (the SOURCE) stays tracked.
|
||||
/checks/
|
||||
/comparison/
|
||||
|
||||
+4
-1
@@ -16,7 +16,9 @@
|
||||
"gen:parity": "node scripts/gen-parity-expected.mjs",
|
||||
"gen:og": "node scripts/gen-og.mjs",
|
||||
"gen:battery": "node scripts/gen-battery-expected.mjs",
|
||||
"pretest:browser": "node scripts/gen-battery-expected.mjs"
|
||||
"pretest:browser": "node scripts/gen-battery-expected.mjs && node scripts/gen-davila7-expected.mjs && node scripts/gen-eval-fixture.mjs",
|
||||
"gen:davila7": "node scripts/gen-davila7-expected.mjs",
|
||||
"gen:evalfixture": "node scripts/gen-eval-fixture.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vigiles/report-view": "*",
|
||||
@@ -25,6 +27,7 @@
|
||||
"idb-keyval": "^6.3.0",
|
||||
"lucide-react": "^0.469.0",
|
||||
"pako": "^2.2.0",
|
||||
"prism-react-renderer": "^2.4.1",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"tailwind-merge": "^2.6.0"
|
||||
|
||||
@@ -17,6 +17,9 @@ import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import { allChecks } from "../src/checks/checks.ts";
|
||||
|
||||
const COMPARISON_GIST =
|
||||
"Every harness failure vigiles checks for — a dropped tool, a hook on an event that does not exist, a script nobody committed — and whether `claude plugin validate` catches it. Measured, with the command to re-run.";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const siteRoot = join(here, "..");
|
||||
|
||||
@@ -53,13 +56,38 @@ const page = (slug: string, title: string, gist: string): string => {
|
||||
`;
|
||||
};
|
||||
|
||||
/** The comparison page is the same shape as a check page — a static MPA entry with
|
||||
* its own title/OG/canonical — so it is generated here rather than hand-committed,
|
||||
* keeping ONE place that knows how such a page is spelled. */
|
||||
const comparison = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/png" href="../logo.png" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>What breaks in an agent harness — and what notices | vigiles</title>
|
||||
<meta name="description" content="${esc(COMPARISON_GIST)}" />
|
||||
<meta property="og:title" content="What breaks in an agent harness — and what notices" />
|
||||
<meta property="og:description" content="${esc(COMPARISON_GIST)}" />
|
||||
<meta property="og:type" content="article" />
|
||||
<link rel="canonical" href="https://vigiles.sh/comparison/" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/comparison/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
|
||||
// Clean regen so a removed check drops its page.
|
||||
rmSync(join(siteRoot, "checks"), { recursive: true, force: true });
|
||||
mkdirSync(join(siteRoot, "comparison"), { recursive: true });
|
||||
writeFileSync(join(siteRoot, "comparison", "index.html"), comparison);
|
||||
for (const c of allChecks) {
|
||||
const dir = join(siteRoot, "checks", c.slug);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(join(dir, "index.html"), page(c.slug, c.title, c.gist));
|
||||
}
|
||||
console.log(
|
||||
`[gen-check-pages] wrote ${String(allChecks.length)} check page(s)`,
|
||||
`[gen-check-pages] wrote ${String(allChecks.length)} check page(s) + comparison`,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Regenerate the fixture the "does the guard actually block?" section is pinned to.
|
||||
*
|
||||
* The landing shows THREE rows measured against a REAL, vendored, MIT, SHA-pinned
|
||||
* OSS guard (test/dogfood/davila7-force-push-blocker@869640b) plus the count of shell
|
||||
* re-spellings it withstands. The site cannot import the engine that produces them —
|
||||
* `verify-plugin-guards` pulls in `run-hook`, which spawns a process, and there is no
|
||||
* browser shim for that. So the rows are retyped in the section, and a retyped row is
|
||||
* a row that drifts.
|
||||
*
|
||||
* This writes the measured outcome to a JSON fixture; the browser test asserts the
|
||||
* rendered rows still match it. Same shape and the same reason as
|
||||
* gen-battery-expected.mjs, and it runs in the same `pretest:browser` step so CI
|
||||
* regenerates BEFORE asserting — a change in the vendored slice or in the engine
|
||||
* fails the test instead of passing quietly.
|
||||
*
|
||||
* WHY THIS PLUGIN. Its hook command is a bare `echo` of a deny decision, so a checker
|
||||
* that merely pipes commands through it certifies it as blocking `rm -rf /` and
|
||||
* `cat ~/.ssh/id_rsa` too. The honest reading is that Claude Code would never invoke
|
||||
* it for those, because its `if:` condition is `Bash(git push *--force*)`. This
|
||||
* fixture carries BOTH halves — what it really blocks, and what it is never asked
|
||||
* about — so the page can show a false green being refused.
|
||||
*
|
||||
* node scripts/gen-davila7-expected.mjs # or: npm run gen:davila7
|
||||
*/
|
||||
import { createRequire } from "node:module";
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { format } from "prettier";
|
||||
|
||||
const here = (p) => fileURLToPath(new URL(p, import.meta.url));
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const enginePath = here("../../dist/test.js");
|
||||
if (!existsSync(enginePath)) {
|
||||
console.error(
|
||||
`no built engine at ${enginePath} — run \`npm run build\` at the repo root first`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const engine = require(enginePath);
|
||||
const PLUGIN = "test/dogfood/davila7-force-push-blocker@869640b";
|
||||
const dir = here(`../../${PLUGIN}`);
|
||||
|
||||
const report = engine.experimental_verifyPluginGuards(dir);
|
||||
|
||||
/** The three rows the section shows: one real block, two never-asked. */
|
||||
const WANTED = ["force-push", "rm-rf", "read-ssh-key"];
|
||||
const byId = new Map();
|
||||
for (const hook of report.hooks) {
|
||||
for (const r of hook.results ?? []) {
|
||||
const prev = byId.get(r.event.id);
|
||||
// A disaster counts as blocked if ANY declared hook blocked it.
|
||||
if (!prev || (r.blocked && !prev.blocked)) byId.set(r.event.id, r);
|
||||
}
|
||||
}
|
||||
const rows = WANTED.map((id) => {
|
||||
const r = byId.get(id);
|
||||
if (!r)
|
||||
throw new Error(`no measured result for "${id}" — did the catalog change?`);
|
||||
return {
|
||||
id,
|
||||
label: r.event.label,
|
||||
command: String(r.event.input.command ?? ""),
|
||||
blocked: Boolean(r.blocked),
|
||||
ran: Boolean(r.ran),
|
||||
reason: String(r.reason ?? ""),
|
||||
};
|
||||
});
|
||||
|
||||
/** Fairness half: every shell re-spelling of a force push this guard withstands. */
|
||||
const forcePush = report.events.filter((e) => e.id.startsWith("force-push"));
|
||||
const spellings = engine.experimental_alternateSpellings(forcePush);
|
||||
const all = [...forcePush, ...spellings];
|
||||
const spellReport = engine.experimental_verifyPluginGuards(dir, {
|
||||
events: all,
|
||||
});
|
||||
let blockedCount = 0;
|
||||
const seen = new Set();
|
||||
for (const hook of spellReport.hooks) {
|
||||
for (const r of hook.results ?? []) {
|
||||
if (r.blocked) seen.add(r.event.id);
|
||||
}
|
||||
}
|
||||
blockedCount = seen.size;
|
||||
|
||||
const fixture = {
|
||||
source: `experimental_verifyPluginGuards("${PLUGIN}")`,
|
||||
regenerate: "node scripts/gen-davila7-expected.mjs",
|
||||
plugin: PLUGIN,
|
||||
rows,
|
||||
spellings: { fed: all.length, blocked: blockedCount },
|
||||
};
|
||||
|
||||
const out = here("../src/components/sections/__fixtures__/davila7-guard.json");
|
||||
writeFileSync(
|
||||
out,
|
||||
await format(JSON.stringify(fixture, null, 2), { parser: "json" }),
|
||||
);
|
||||
console.log(
|
||||
`[gen-davila7-expected] ${String(rows.length)} row(s), ${String(blockedCount)}/${String(all.length)} spellings blocked → ${out}`,
|
||||
);
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Copy the committed eval LOCK into the site's fixtures.
|
||||
*
|
||||
* The eval tier costs real model quota, so unlike the guard fixture this CANNOT be
|
||||
* regenerated by re-running the measurement in CI — that would spend money on every
|
||||
* push. The lock (`.vigiles/eval-locks/<name>.lock.json`, written by
|
||||
* `vigiles eval --update`) IS the committed measurement; this only copies it where
|
||||
* the site can import it, so the page still renders from a machine-written artifact
|
||||
* rather than a number somebody retyped.
|
||||
*
|
||||
* Refresh the measurement itself with:
|
||||
* node dist/cli.js eval examples/harness/skill-trigger-rate.eval.mjs --trials=3 --update
|
||||
*/
|
||||
import { copyFileSync, existsSync, mkdirSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = (p) => fileURLToPath(new URL(p, import.meta.url));
|
||||
const LOCK = here(
|
||||
"../../.vigiles/eval-locks/superpowers-tdd-trigger-rate.lock.json",
|
||||
);
|
||||
const OUT = here(
|
||||
"../src/components/sections/__fixtures__/tdd-trigger-rate.json",
|
||||
);
|
||||
|
||||
if (!existsSync(LOCK)) {
|
||||
console.error(
|
||||
`no eval lock at ${LOCK} — it is committed, so this means it was deleted. ` +
|
||||
`Re-measure: node dist/cli.js eval examples/harness/skill-trigger-rate.eval.mjs --trials=3 --update`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
mkdirSync(here("../src/components/sections/__fixtures__"), { recursive: true });
|
||||
copyFileSync(LOCK, OUT);
|
||||
console.log(`[gen-eval-fixture] ${LOCK} → ${OUT}`);
|
||||
@@ -2,6 +2,7 @@ import { StickyCTA } from "@/components/StickyCTA";
|
||||
import { Toaster } from "@/components/ui/toaster";
|
||||
import { Hero } from "@/components/sections/Hero";
|
||||
import { Wedge } from "@/components/sections/Wedge";
|
||||
import { Measure } from "@/components/sections/Measure";
|
||||
import { Guard } from "@/components/sections/Guard";
|
||||
import { VerbMap } from "@/components/sections/VerbMap";
|
||||
import { Adoption } from "@/components/sections/Adoption";
|
||||
@@ -17,6 +18,7 @@ export function App() {
|
||||
<main className="min-h-screen">
|
||||
<Hero />
|
||||
<Wedge />
|
||||
<Measure />
|
||||
<Guard />
|
||||
<VerbMap />
|
||||
<Adoption />
|
||||
|
||||
@@ -149,7 +149,7 @@ export const CHECKS: Record<string, CheckDoc> = {
|
||||
"Unlike a skill, a subagent REQUIRES `name` + `description` frontmatter to register at all — without them the harness has no agent to dispatch. A `model:` or `color:` that's a close typo of a real value silently falls back to the default.",
|
||||
],
|
||||
why: "A subagent that won't register can never be dispatched — the whole agent is dead weight. A silently-wrong model means it runs on a model you didn't choose.",
|
||||
fix: "Add the required `name` + `description`; fix any misspelled `model`/`color` value. vigiles matches Anthropic's own `claude plugin validate` here and flags close typos of real aliases.",
|
||||
fix: "Add the required `name` + `description`; fix any misspelled `model`/`color` value. `claude plugin validate` warns about a missing description too; a typo'd `model:` passes it clean \u2014 measured against Claude Code 2.1.263.",
|
||||
category: "Structure",
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { checkSlugsWithPages } from "../checks/checks";
|
||||
import { ROWS, type ComparisonRow } from "./rows";
|
||||
import snapshot from "./validate-overlap.json";
|
||||
|
||||
/**
|
||||
* The `/comparison` page — what breaks in an agent harness, and which tool notices.
|
||||
*
|
||||
* EVERY COMPETITOR CELL IS RENDERED FROM `validate-overlap.json`, the snapshot written
|
||||
* by `node tools/measure-validate-overlap.mjs --json`. Nothing here is typed by hand,
|
||||
* because a hand-typed cell is how this repository accumulated four false claims about
|
||||
* another tool before 2026-09-09. A row with no measured case renders "not probed" and
|
||||
* never an ✗: absence of a run is not a defect in somebody else's product.
|
||||
*
|
||||
* Static Vite MPA entry, like the check pages — real HTML, own <title>/OG/canonical,
|
||||
* no router, no SSR. Links are `../`-relative (one level deep) to hold under `base: "./"`.
|
||||
*/
|
||||
|
||||
interface Case {
|
||||
id: string;
|
||||
rule: string;
|
||||
what: string;
|
||||
flagged: boolean;
|
||||
per: Record<string, string[]>;
|
||||
}
|
||||
const CASES = new Map(
|
||||
(snapshot.cases as Case[]).map((c) => [c.id, c] as const),
|
||||
);
|
||||
|
||||
/** Which shapes flagged it — the plugin-vs-repo-local distinction is load-bearing. */
|
||||
function verdict(kase: Case | undefined): {
|
||||
label: string;
|
||||
tone: "caught" | "missed" | "unprobed";
|
||||
} {
|
||||
if (!kase) return { label: "not probed", tone: "unprobed" };
|
||||
const hits = Object.entries(kase.per).filter(([, f]) => f.length > 0);
|
||||
if (hits.length === 0) return { label: "passes it", tone: "missed" };
|
||||
const everywhere = hits.length === Object.keys(kase.per).length;
|
||||
return {
|
||||
label: everywhere ? "caught" : "caught, packaged plugins only",
|
||||
tone: "caught",
|
||||
};
|
||||
}
|
||||
|
||||
function Row({ row }: { row: ComparisonRow }) {
|
||||
const kase = row.probeCase ? CASES.get(row.probeCase) : undefined;
|
||||
const v = verdict(kase);
|
||||
return (
|
||||
<div className="border-t border-border/60 py-5 first:border-t-0">
|
||||
<div className="flex flex-col gap-1 sm:flex-row sm:items-baseline sm:justify-between sm:gap-6">
|
||||
<p className="text-base font-medium text-foreground">{row.what}</p>
|
||||
{row.zone === "config" && (
|
||||
<p
|
||||
className={`shrink-0 font-mono text-sm ${
|
||||
v.tone === "caught"
|
||||
? "text-good"
|
||||
: v.tone === "missed"
|
||||
? "text-muted-foreground"
|
||||
: "text-muted-foreground/60"
|
||||
}`}
|
||||
>
|
||||
{v.label}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<p className="mt-1 max-w-2xl text-sm leading-relaxed text-muted-foreground">
|
||||
{row.gloss}
|
||||
</p>
|
||||
{row.slug && checkSlugsWithPages.has(row.slug) && (
|
||||
<a
|
||||
className="mt-2 inline-block text-sm text-primary hover:underline"
|
||||
href={`../checks/${row.slug}/`}
|
||||
>
|
||||
How vigiles checks it →
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ComparisonPage() {
|
||||
const config = ROWS.filter((r) => r.zone === "config");
|
||||
const behaviour = ROWS.filter((r) => r.zone === "behaviour");
|
||||
const groups: [string, ComparisonRow[]][] = [];
|
||||
for (const r of config) {
|
||||
const name = r.group ?? "Other";
|
||||
const found = groups.find(([n]) => n === name);
|
||||
if (found) found[1].push(r);
|
||||
else groups.push([name, [r]]);
|
||||
}
|
||||
const caught = config.filter(
|
||||
(r) => verdict(CASES.get(r.probeCase ?? "")).tone === "caught",
|
||||
).length;
|
||||
|
||||
return (
|
||||
<main className="mx-auto w-full max-w-3xl px-6 py-16">
|
||||
<a
|
||||
href="../"
|
||||
className="inline-flex items-center gap-2 text-sm text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" /> vigiles
|
||||
</a>
|
||||
|
||||
<h1 className="mt-8 text-3xl font-bold tracking-tight sm:text-4xl">
|
||||
What breaks in an agent harness — and what notices
|
||||
</h1>
|
||||
<p className="mt-4 text-lg leading-relaxed text-muted-foreground">
|
||||
Config that parses perfectly and still does nothing: a tool the harness
|
||||
drops, a hook wired to an event that does not exist, a script nobody
|
||||
committed. Below is every failure we check for, and whether the
|
||||
validator you already have catches it.
|
||||
</p>
|
||||
|
||||
<section id="config" className="mt-14 scroll-mt-8">
|
||||
<h2 className="text-xl font-semibold">
|
||||
Is your config actually wired?
|
||||
</h2>
|
||||
<p className="mt-2 text-sm leading-relaxed text-muted-foreground">
|
||||
Measured, not asserted. Each row was planted as a real defect in a
|
||||
throwaway plugin and run through{" "}
|
||||
<code className="rounded bg-muted/60 px-1 py-0.5 font-mono">
|
||||
{snapshot.tool}
|
||||
</code>{" "}
|
||||
{snapshot.version}, in both shapes a user has — a repo-local{" "}
|
||||
<code className="rounded bg-muted/60 px-1 py-0.5 font-mono">
|
||||
.claude/
|
||||
</code>{" "}
|
||||
harness and a packaged plugin, with and without{" "}
|
||||
<code className="rounded bg-muted/60 px-1 py-0.5 font-mono">
|
||||
--strict
|
||||
</code>
|
||||
. It flagged <strong>{caught}</strong> of {config.length}. Re-run it
|
||||
yourself:{" "}
|
||||
<code className="rounded bg-muted/60 px-1 py-0.5 font-mono">
|
||||
node tools/measure-validate-overlap.mjs
|
||||
</code>
|
||||
.
|
||||
</p>
|
||||
{/* ONE line, not a two-column header: at 390px the two-column version
|
||||
collapsed into a four-line stack ("VIGILES / CATCHES / EVERY / ROW"). */}
|
||||
<p className="mt-8 text-sm text-foreground">
|
||||
vigiles catches every row below. The verdict on each is{" "}
|
||||
<code className="rounded bg-muted/60 px-1 py-0.5 font-mono text-xs">
|
||||
{snapshot.tool}
|
||||
</code>{" "}
|
||||
{snapshot.version}.
|
||||
</p>
|
||||
|
||||
{/* COLLAPSED BY DEFAULT. Thirteen expanded rows read as a lint-rule dump on
|
||||
a marketing page; four groups, each carrying its own measured tally, read
|
||||
as an argument you can skim in seconds and open only where you care.
|
||||
Native <details> — no JS, works without hydration, keyboard-accessible. */}
|
||||
<div className="mt-6 space-y-3">
|
||||
{groups.map(([name, rows]) => {
|
||||
const hit = rows.filter(
|
||||
(r) => verdict(CASES.get(r.probeCase ?? "")).tone === "caught",
|
||||
).length;
|
||||
return (
|
||||
<details
|
||||
key={name}
|
||||
className="group rounded-xl border border-border/60 bg-card/30 px-5"
|
||||
>
|
||||
<summary className="flex cursor-pointer list-none items-baseline justify-between gap-4 py-4">
|
||||
<span className="text-base font-medium text-foreground">
|
||||
{name}
|
||||
</span>
|
||||
<span className="shrink-0 font-mono text-xs text-muted-foreground">
|
||||
{rows.length} {rows.length === 1 ? "check" : "checks"} ·{" "}
|
||||
{hit === 0 ? "none caught" : `${String(hit)} caught`}
|
||||
</span>
|
||||
</summary>
|
||||
<div className="pb-2">
|
||||
{rows.map((r) => (
|
||||
<Row key={r.what} row={r} />
|
||||
))}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="behaviour" className="mt-16 scroll-mt-8">
|
||||
<h2 className="text-xl font-semibold">
|
||||
Does the agent actually behave?
|
||||
</h2>
|
||||
<p className="mt-2 text-sm leading-relaxed text-muted-foreground">
|
||||
These have no column above, and that is the honest answer rather than
|
||||
a gap: a config validator is not attempting them. Whether a skill
|
||||
fires cannot be decided by reading the file at all — it depends on a
|
||||
model choosing, so it has to be measured by running one.
|
||||
</p>
|
||||
<div className="mt-6">
|
||||
{behaviour.map((r) => (
|
||||
<Row key={r.what} row={r} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="mt-16 rounded-xl border border-border/60 bg-card/40 p-6">
|
||||
<h2 className="text-base font-semibold">Grade your own harness</h2>
|
||||
<p className="mt-2 text-sm text-muted-foreground">
|
||||
One command, nothing uploaded, nothing executed — a deterministic
|
||||
read.
|
||||
</p>
|
||||
<pre className="mt-4 whitespace-pre-wrap break-words rounded-lg border border-border/60 bg-background p-3 font-mono text-sm">
|
||||
npx vigiles audit
|
||||
</pre>
|
||||
</section>
|
||||
|
||||
<p className="mt-10 text-xs leading-relaxed text-muted-foreground">
|
||||
Every measured cell on this page comes from{" "}
|
||||
<code className="font-mono">{snapshot.command}</code>, run against{" "}
|
||||
{snapshot.tool} {snapshot.version} on {snapshot.measuredAt}. A blank
|
||||
cell means we have not run that tool against that defect — not that it
|
||||
fails. Only tools this repository actually runs appear here.
|
||||
</p>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import "../index.css";
|
||||
import { ComparisonPage } from "./ComparisonPage";
|
||||
|
||||
/** MPA entry for /comparison — one static HTML page, no router (see gen-check-pages.ts). */
|
||||
const root = document.getElementById("root");
|
||||
if (root) {
|
||||
createRoot(root).render(
|
||||
<StrictMode>
|
||||
<ComparisonPage />
|
||||
</StrictMode>,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* The COMPARISON rows — what can go wrong in an agent harness, in a plain sentence.
|
||||
*
|
||||
* WHY ROWS ARE DEFECTS, NOT PRODUCTS. A grid with product columns invites a cell to
|
||||
* be filled from a competitor's documentation. Every row here is a thing that breaks
|
||||
* in a real harness; a column may only answer it if a script in `tools/` actually RAN
|
||||
* that tool against a planted instance of it.
|
||||
*
|
||||
* TWO ZONES, and keeping them apart is the honest part. `claude plugin validate` is a
|
||||
* genuine peer on CONFIG (zone "config") and is not attempting BEHAVIOUR (zone
|
||||
* "behaviour") at all — it is a manifest checker, not a test runner. Scoring it on
|
||||
* whether it measures a skill's trigger rate would be a rigged row, so behaviour rows
|
||||
* carry no competitor cell and say why.
|
||||
*
|
||||
* `probeCase` keys into `validate-overlap.json`, written by
|
||||
* `node tools/measure-validate-overlap.mjs --json`. A row without one renders
|
||||
* "not probed" — never an ✗ against somebody else's product.
|
||||
*/
|
||||
|
||||
export type Zone = "config" | "behaviour";
|
||||
|
||||
export interface ComparisonRow {
|
||||
/** The defect as a sentence a plugin author would recognise. */
|
||||
what: string;
|
||||
/** The plain gloss — shown inline, because a hover tooltip is invisible on a phone. */
|
||||
gloss: string;
|
||||
/** The vigiles rule slug, when one check owns this row (links to its page). */
|
||||
slug?: string;
|
||||
/** Key into the measured snapshot. Absent ⇒ no competitor cell is rendered. */
|
||||
probeCase?: string;
|
||||
/** Collapsing bucket. Thirteen flat rows read as a lint-rule dump on a marketing
|
||||
* page; four named groups read as an argument. Behaviour rows need none. */
|
||||
group?: string;
|
||||
zone: Zone;
|
||||
}
|
||||
|
||||
export const ROWS: readonly ComparisonRow[] = [
|
||||
// ── zone: config — measured against a real run of `claude plugin validate` ──
|
||||
{
|
||||
zone: "config",
|
||||
what: "A subagent asks for a tool that is spelled wrong",
|
||||
gloss:
|
||||
"The harness drops the tool silently. Your agent quietly cannot use it.",
|
||||
slug: "subagent-tool-contract",
|
||||
probeCase: "subagent-typod-tool",
|
||||
group: "Tools the harness silently drops",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A subagent asks for a tool no subagent can ever have",
|
||||
gloss:
|
||||
"AskUserQuestion is not available to subagents. It is dropped without a word.",
|
||||
slug: "subagent-tool-contract",
|
||||
probeCase: "subagent-never-available-tool",
|
||||
group: "Tools the harness silently drops",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A skill asks for a tool that does not exist",
|
||||
gloss: "Same silent drop, on the skill side of the harness.",
|
||||
slug: "subagent-tool-contract",
|
||||
probeCase: "skill-tool-does-not-exist",
|
||||
group: "Tools the harness silently drops",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A skill names an MCP server the plugin never declares",
|
||||
gloss:
|
||||
"The tool can never resolve, so the step that needed it fails at runtime.",
|
||||
slug: "mcp-tool-resolves",
|
||||
probeCase: "skill-undeclared-mcp-server",
|
||||
group: "Paths and names that point at nothing",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A subagent names a model that does not exist",
|
||||
gloss:
|
||||
"It silently falls back to the default — you are billed for a model you did not choose.",
|
||||
slug: "subagent-frontmatter",
|
||||
probeCase: "subagent-typod-model",
|
||||
group: "Units that never register",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A subagent has no name or description",
|
||||
gloss: "It cannot register at all, so it can never be dispatched.",
|
||||
slug: "subagent-frontmatter",
|
||||
probeCase: "subagent-missing-frontmatter",
|
||||
group: "Units that never register",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A deny-list entry is misspelled, so it blocks nothing",
|
||||
gloss:
|
||||
"You believe a tool is forbidden. The typo means it stays available.",
|
||||
slug: "disallowed-tools-contract",
|
||||
probeCase: "subagent-disallowed-tools-typo",
|
||||
group: "Tools the harness silently drops",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A hook is registered on an event that does not exist",
|
||||
gloss:
|
||||
"One letter off and the hook is never wired to anything. The config is still valid.",
|
||||
slug: "hook-events",
|
||||
probeCase: "hook-typod-event",
|
||||
group: "Paths and names that point at nothing",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A hook points at a script that was never committed",
|
||||
gloss:
|
||||
"The path parses fine. The guard you think protects you runs nothing.",
|
||||
slug: "hook-script-exists",
|
||||
probeCase: "hook-script-missing",
|
||||
group: "Paths and names that point at nothing",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A skill's settings block is not valid YAML",
|
||||
gloss:
|
||||
"Fields may not parse as you intended, so the skill loads with the wrong metadata.",
|
||||
slug: "frontmatter-valid",
|
||||
probeCase: "skill-malformed-frontmatter",
|
||||
group: "Units that never register",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A skill has no name or description",
|
||||
gloss:
|
||||
"It falls back to the directory name and first paragraph — a weak trigger surface.",
|
||||
slug: "skill-frontmatter",
|
||||
probeCase: "skill-missing-frontmatter",
|
||||
group: "Units that never register",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "Two skills describe themselves almost identically",
|
||||
gloss: "The model cannot tell them apart, so the wrong one fires.",
|
||||
slug: "description-overlap",
|
||||
probeCase: "skill-description-overlap",
|
||||
group: "The model picks the wrong one",
|
||||
},
|
||||
{
|
||||
zone: "config",
|
||||
what: "A skill links to a file that is not there",
|
||||
gloss: "The step that told the agent to read it silently does nothing.",
|
||||
slug: "skill-resource-resolves",
|
||||
probeCase: "skill-resource-missing",
|
||||
group: "Paths and names that point at nothing",
|
||||
},
|
||||
|
||||
// ── zone: behaviour — no competitor cell, and the reason is the point ──
|
||||
{
|
||||
zone: "behaviour",
|
||||
what: "Does this skill actually fire when it should?",
|
||||
gloss:
|
||||
"Undecidable by reading the file — it depends on a model choosing. vigiles measures it across varied prompts and reports recall and precision.",
|
||||
},
|
||||
{
|
||||
zone: "behaviour",
|
||||
what: "Does your safety hook actually block a destructive command?",
|
||||
gloss:
|
||||
"Measured against a catalogue of disaster commands. The hand-written guard the ecosystem copies blocks 2 of 7; a compiled hook blocks 7 of 7.",
|
||||
},
|
||||
{
|
||||
zone: "behaviour",
|
||||
what: "Does a subagent stay inside the tools it declared?",
|
||||
gloss:
|
||||
"`tools:` is documentation, not a fence. vigiles turns the declared contract into a rail enforced while the agent runs.",
|
||||
},
|
||||
{
|
||||
zone: "behaviour",
|
||||
what: "Does the lint rule your CLAUDE.md claims to enforce exist — and is it still on?",
|
||||
gloss:
|
||||
"Your instructions say you enforce a rule. vigiles resolves it against your real linter config and fails CI when someone switches it off.",
|
||||
},
|
||||
] as const;
|
||||
@@ -0,0 +1,197 @@
|
||||
{
|
||||
"tool": "claude plugin validate",
|
||||
"version": "2.1.266",
|
||||
"measuredAt": "2026-09-09",
|
||||
"command": "node tools/measure-validate-overlap.mjs --json <path>",
|
||||
"shapes": ["plain", "plain --strict", "plugin", "plugin --strict"],
|
||||
"cases": [
|
||||
{
|
||||
"id": "skill-tool-does-not-exist",
|
||||
"rule": "subagent-tool-contract",
|
||||
"what": "a skill declares a tool that does not exist (`Bahs`)",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "skill-undeclared-mcp-server",
|
||||
"rule": "mcp-tool-resolves",
|
||||
"what": "a skill names an MCP server the plugin never declares (`mcp__ghost__thing`)",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "subagent-typod-tool",
|
||||
"rule": "subagent-tool-contract",
|
||||
"what": "a subagent declares a typo'd tool (`Grpe`)",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "subagent-never-available-tool",
|
||||
"rule": "subagent-tool-contract",
|
||||
"what": "a subagent declares a tool a subagent can never have (`AskUserQuestion`)",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "subagent-typod-model",
|
||||
"rule": "subagent-frontmatter",
|
||||
"what": "a subagent declares a typo'd model (`sonnnet`) — silently falls back",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "subagent-missing-frontmatter",
|
||||
"rule": "subagent-frontmatter",
|
||||
"what": "a subagent with no name/description — cannot register at all",
|
||||
"flagged": true,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
],
|
||||
"plugin --strict": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "subagent-disallowed-tools-typo",
|
||||
"rule": "disallowed-tools-contract",
|
||||
"what": "a subagent's deny-list entry is a typo (`Bahs`) — so it blocks nothing",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "hook-typod-event",
|
||||
"rule": "hook-events",
|
||||
"what": "a hook is registered on a typo'd event (`PreToolUze`) — never fires",
|
||||
"flagged": true,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [
|
||||
"hooks: hooks.PreToolUze: unknown hook event; entry ignored at runtime"
|
||||
],
|
||||
"plugin --strict": [
|
||||
"hooks: hooks.PreToolUze: unknown hook event; entry ignored at runtime"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "hook-script-missing",
|
||||
"rule": "hook-script-exists",
|
||||
"what": "a hook command names a script that is not on disk — silently runs nothing",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "skill-malformed-frontmatter",
|
||||
"rule": "frontmatter-valid",
|
||||
"what": "a skill's `---` block exists but is not valid YAML",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "skill-missing-frontmatter",
|
||||
"rule": "skill-frontmatter",
|
||||
"what": "a skill with no name/description — falls back to dir name + first paragraph",
|
||||
"flagged": true,
|
||||
"per": {
|
||||
"plain": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
],
|
||||
"plain --strict": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
],
|
||||
"plugin": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
],
|
||||
"plugin --strict": [
|
||||
"frontmatter: No frontmatter block found. Add YAML frontmatter between --- delimiters at the top of the file to set description and other metadata."
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "skill-description-overlap",
|
||||
"rule": "description-overlap",
|
||||
"what": "two model-invocable skills with near-identical descriptions — the selector cannot tell them apart",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "skill-resource-missing",
|
||||
"rule": "skill-resource-resolves",
|
||||
"what": "a skill body links a file that is not on disk",
|
||||
"flagged": false,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [],
|
||||
"plugin --strict": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "all-at-once",
|
||||
"rule": "(combined)",
|
||||
"what": "the original seven-defects-in-one-fixture run, kept so the published 0/7 and 1/7 figures stay reproducible",
|
||||
"flagged": true,
|
||||
"per": {
|
||||
"plain": [],
|
||||
"plain --strict": [],
|
||||
"plugin": [
|
||||
"hooks: hooks.PreToolUze: unknown hook event; entry ignored at runtime"
|
||||
],
|
||||
"plugin --strict": [
|
||||
"hooks: hooks.PreToolUze: unknown hook event; entry ignored at runtime"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { Highlight, type PrismTheme } from "prism-react-renderer";
|
||||
|
||||
/**
|
||||
* The one code block on the site — highlighted, and wrapping.
|
||||
*
|
||||
* THEME: Oceanic Material, defined here as data rather than pulled from a theme
|
||||
* package. prism-react-renderer takes a plain object, so the palette IS the config;
|
||||
* a second dependency would buy nothing. Palette is the canonical Oceanic set
|
||||
* (#1B2B34 ground, #6699CC blue, #99C794 green, #C594C5 purple, #F99157 orange,
|
||||
* #5FB3B3 cyan, #65737E comment).
|
||||
*
|
||||
* 🔴 WRAPPING IS NOT COSMETIC — it is the mobile contract. `site/e2e/mobile.spec.ts`
|
||||
* asserts no visible <pre> overflows its box at 390px, and a highlighter's default is
|
||||
* to emit a wide non-wrapping <pre>. Both `whitespace-pre-wrap` and `break-words` are
|
||||
* load-bearing here: the first wraps at spaces, the second breaks the long unbroken
|
||||
* tokens (paths, URLs, `measureTriggerRate:`) that otherwise push the page sideways.
|
||||
* A previous version of these blocks used `overflow-x-auto` and bled 258px.
|
||||
*/
|
||||
const oceanic: PrismTheme = {
|
||||
plain: { color: "#CDD3DE", backgroundColor: "#1B2B34" },
|
||||
styles: [
|
||||
{
|
||||
types: ["comment", "prolog", "doctype", "cdata"],
|
||||
style: { color: "#65737E", fontStyle: "italic" },
|
||||
},
|
||||
{ types: ["punctuation"], style: { color: "#A7ADBA" } },
|
||||
{
|
||||
types: ["property", "tag", "constant", "symbol", "deleted"],
|
||||
style: { color: "#EC5f67" },
|
||||
},
|
||||
{ types: ["boolean", "number"], style: { color: "#F99157" } },
|
||||
{
|
||||
types: ["selector", "attr-name", "string", "char", "builtin", "inserted"],
|
||||
style: { color: "#99C794" },
|
||||
},
|
||||
{
|
||||
types: ["operator", "entity", "url", "variable"],
|
||||
style: { color: "#5FB3B3" },
|
||||
},
|
||||
{
|
||||
types: ["atrule", "attr-value", "function", "class-name"],
|
||||
style: { color: "#6699CC" },
|
||||
},
|
||||
{ types: ["keyword"], style: { color: "#C594C5" } },
|
||||
{ types: ["regex", "important"], style: { color: "#FAC863" } },
|
||||
],
|
||||
};
|
||||
|
||||
export function CodeBlock({
|
||||
code,
|
||||
language = "tsx",
|
||||
className = "",
|
||||
}: {
|
||||
code: string;
|
||||
language?: string;
|
||||
className?: string;
|
||||
}) {
|
||||
return (
|
||||
<Highlight theme={oceanic} code={code} language={language}>
|
||||
{({ style, tokens, getLineProps, getTokenProps }) => (
|
||||
<pre
|
||||
style={style}
|
||||
className={`whitespace-pre-wrap break-words rounded-xl border border-border/60 p-5 font-mono text-xs leading-relaxed ${className}`}
|
||||
>
|
||||
{tokens.map((line, i) => (
|
||||
<div key={i} {...getLineProps({ line })}>
|
||||
{line.map((token, k) => (
|
||||
<span key={k} {...getTokenProps({ token })} />
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</pre>
|
||||
)}
|
||||
</Highlight>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
import { CodeBlock } from "@/components/CodeBlock";
|
||||
import RUN_HOOK from "@/snippets/hook-test.ts?raw";
|
||||
import fixture from "./__fixtures__/davila7-guard.json";
|
||||
import lock from "./__fixtures__/tdd-trigger-rate.json";
|
||||
|
||||
/**
|
||||
* TEST and EVAL — the two beats `audit` cannot play.
|
||||
*
|
||||
* WHY IT EXISTS. The landing led with `audit` (a graded read) and named the other
|
||||
* three verbs once each in a map. A visitor came away thinking vigiles is a linter.
|
||||
* The founder, 2026-09-09: "audit is a starting point and then people going through
|
||||
* the website should think wow this stuff like testing and evals really solves my
|
||||
* problem."
|
||||
*
|
||||
* THE ARGUMENT, and why it is on THIS repo. The demo above grades
|
||||
* davila7/claude-code-templates. So these sections stay on the same repo the reader
|
||||
* just watched get graded, instead of introducing a hypothetical. Its force-push
|
||||
* guard is a real, MIT, SHA-pinned vendored plugin, and its hook command is a bare
|
||||
* `echo` of a deny decision — so a checker that merely pipes commands through it
|
||||
* certifies it as blocking `rm -rf /` and `cat ~/.ssh/id_rsa`. It does not: Claude
|
||||
* Code only ever invokes it for a force push. Refusing that false green IS the pitch.
|
||||
*
|
||||
* 🔴 THE COPY STANDARD FOR THIS SECTION AND ANY LIKE IT (Ernie, 2026-09-09).
|
||||
* A reader must look at the code and instantly think "wait — yeah, I have seen
|
||||
* this." Write the BUG they have already lived through, not the mechanism we
|
||||
* built. "A hook is a process, pipe it an event" describes plumbing; "the guard
|
||||
* you think protects you runs nothing" is the thing that happened to them.
|
||||
* Every beat should name a concrete failure a plugin author recognises: a guard
|
||||
* that looks unconditional and only ever fires on one command; a tool that is
|
||||
* silently dropped; a skill that never fires because a sibling took the prompt.
|
||||
* If a paragraph would still read fine in a competitor's docs, it is describing
|
||||
* a feature and has not earned its place — rewrite it as the failure.
|
||||
*
|
||||
* EVERY NUMBER IS PINNED. The rows and the 44/44 come from
|
||||
* __fixtures__/davila7-guard.json, regenerated by scripts/gen-davila7-expected.mjs
|
||||
* in `pretest:browser` and asserted by Measure.browser.test.tsx — the same mechanism
|
||||
* as the disaster battery, for the same reason: a retyped number is one that drifts,
|
||||
* and this section's whole claim is that it does not need a footnote.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The snippet is a REAL FILE the type-checker compiles (`npm run check` runs
|
||||
* `tsc --noEmit -p site/tsconfig.json`), imported here as text. A hand-typed string
|
||||
* is not checked by anything: the eval snippet shipped naming two exports that do
|
||||
* not exist — `defineEval` and `skillResolved` from "vigiles/eval" — with build,
|
||||
* lint, prettier, browser and e2e all green. Now a renamed export fails the build
|
||||
* with "Did you mean 'runHook'?" instead of reaching a reader.
|
||||
*
|
||||
* MEASURED LIMIT, so nobody over-trusts this: tsc catches the wrong NAME, and
|
||||
* catches wrong types only where the API is precisely typed. `tool: 123` currently
|
||||
* passes, because runHook's options type is loose there.
|
||||
*/
|
||||
|
||||
function Verdict({ row }: { row: (typeof fixture.rows)[number] }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1 border-t border-border/60 py-4 sm:flex-row sm:items-baseline sm:justify-between sm:gap-6">
|
||||
<code className="whitespace-pre-wrap break-words font-mono text-sm text-foreground">
|
||||
{row.command}
|
||||
</code>
|
||||
<span
|
||||
className={`shrink-0 font-mono text-xs ${
|
||||
row.blocked ? "text-good" : "text-muted-foreground"
|
||||
}`}
|
||||
>
|
||||
{row.blocked ? "blocked" : "not run"}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function Measure() {
|
||||
const { rows, spellings } = fixture;
|
||||
// Only the prompts that did NOT reliably fire — the eight that always fire are
|
||||
// not the story, and listing all ten is the rule-dump shape we already cut once.
|
||||
const misses = lock.report.perPrompt.filter((p) => p.rate < 1);
|
||||
return (
|
||||
<>
|
||||
<section className="border-t border-border/60">
|
||||
<div className="mx-auto w-full max-w-4xl px-6 py-20 sm:py-24">
|
||||
<p className="font-mono text-xs text-primary">
|
||||
$ vigiles test · no model · free in CI
|
||||
</p>
|
||||
<h2 className="mt-3 text-3xl font-bold tracking-tight sm:text-4xl">
|
||||
You have a safety hook. What does it actually stop?
|
||||
</h2>
|
||||
<p className="mt-4 max-w-2xl text-lg leading-relaxed text-muted-foreground">
|
||||
The repo above ships one. It looks like a blanket deny — and for two
|
||||
of these three commands, Claude Code never even calls it.
|
||||
</p>
|
||||
|
||||
<div className="mt-8">
|
||||
{rows.map((r) => (
|
||||
<Verdict key={r.id} row={r} />
|
||||
))}
|
||||
</div>
|
||||
|
||||
<p className="mt-6 max-w-2xl text-sm leading-relaxed text-muted-foreground">
|
||||
Its hook command is a bare <code className="font-mono">echo</code>{" "}
|
||||
of a deny decision, so a checker that just pipes commands through it
|
||||
would certify it as stopping all three. It does not — Claude Code
|
||||
only ever invokes it for a force push. vigiles reports what the
|
||||
guard was actually asked, and refuses the false green.
|
||||
</p>
|
||||
<p className="mt-4 max-w-2xl text-sm leading-relaxed text-muted-foreground">
|
||||
And it does not manufacture a complaint either: fed{" "}
|
||||
<strong className="text-foreground">
|
||||
{spellings.fed} shell re-spellings
|
||||
</strong>{" "}
|
||||
of a force push — quoted flags,{" "}
|
||||
<code className="font-mono">sudo</code>, tabs, escapes — this guard
|
||||
blocked{" "}
|
||||
<strong className="text-foreground">all {spellings.blocked}</strong>
|
||||
.
|
||||
</p>
|
||||
|
||||
<CodeBlock code={RUN_HOOK} language="tsx" className="mt-8" />
|
||||
<p className="mt-3 text-sm text-muted-foreground">
|
||||
The test lives next to the thing it tests —{" "}
|
||||
<code className="font-mono text-xs">hooks/block-force-push.sh</code>{" "}
|
||||
gets{" "}
|
||||
<code className="font-mono text-xs">
|
||||
hooks/block-force-push.harness.mjs
|
||||
</code>
|
||||
. One property earns that:{" "}
|
||||
<code className="font-mono text-xs">ls</code> answers “is this
|
||||
tested?” without running anything.
|
||||
</p>
|
||||
|
||||
{/* Collapsed on purpose. A reader who has never had a test push to their
|
||||
real remote does not need this; the one who has, opens it. */}
|
||||
<details className="group mt-6 rounded-xl border border-border/60 bg-card/30 px-5">
|
||||
<summary className="cursor-pointer list-none py-4 text-base font-medium text-foreground">
|
||||
But my hook actually does things — what stops the test doing them
|
||||
for real?
|
||||
</summary>
|
||||
<div className="space-y-4 pb-5 text-sm leading-relaxed text-muted-foreground">
|
||||
<p className="text-foreground">
|
||||
You can throw away a temp directory. You cannot un-push a branch
|
||||
or un-charge an API call.
|
||||
</p>
|
||||
<p>
|
||||
<strong className="text-foreground">
|
||||
Every run is disposable.
|
||||
</strong>{" "}
|
||||
Each test gets a fresh throwaway working directory, so a hook
|
||||
that writes files leaves nothing behind.
|
||||
</p>
|
||||
<p>
|
||||
<strong className="text-foreground">
|
||||
Effects you cannot take back get intercepted, not contained.
|
||||
</strong>{" "}
|
||||
A tool call can be caught and asserted on without being allowed
|
||||
to happen — you check that the agent <em>tried</em> to push,
|
||||
while no push occurs.
|
||||
</p>
|
||||
<p>
|
||||
<strong className="text-foreground">
|
||||
Somebody else’s hook runs confined.
|
||||
</strong>{" "}
|
||||
Code you did not write executes with no network, a read-only
|
||||
filesystem and a cleared environment. That wall is Linux
|
||||
(bubblewrap); where the kernel features are absent vigiles
|
||||
refuses to run foreign code rather than quietly running it
|
||||
unconfined.
|
||||
</p>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="border-t border-border/60">
|
||||
<div className="mx-auto w-full max-w-4xl px-6 py-20 sm:py-24">
|
||||
<p className="font-mono text-xs text-primary">
|
||||
$ vigiles eval · real model · your Claude subscription
|
||||
</p>
|
||||
<h2 className="mt-3 text-3xl font-bold tracking-tight sm:text-4xl">
|
||||
Your skill has a description. Does it fire?
|
||||
</h2>
|
||||
<p className="mt-4 max-w-2xl text-lg leading-relaxed text-muted-foreground">
|
||||
We measured a popular open-source TDD skill against ten ways of
|
||||
asking for test-first work. Eight reached it. These two did not:
|
||||
</p>
|
||||
|
||||
<div className="mt-8 overflow-hidden rounded-xl border border-border/60">
|
||||
{misses.map((m) => (
|
||||
<div
|
||||
key={m.prompt}
|
||||
className="flex flex-col gap-1 border-b border-border/60 bg-card/40 p-5 sm:flex-row sm:items-baseline sm:justify-between sm:gap-6"
|
||||
>
|
||||
<p className="whitespace-pre-wrap break-words font-mono text-sm text-foreground">
|
||||
{m.prompt}
|
||||
</p>
|
||||
<span className="shrink-0 font-mono text-xs text-muted-foreground">
|
||||
fired {m.fired} of {m.trials}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
<div className="bg-card/20 p-5">
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">
|
||||
Every prompt it <em>did</em> fire on asks for something new.
|
||||
Both of these ask for tests around code that already exists —
|
||||
which is most of the work most people do.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="mt-6 max-w-2xl text-sm leading-relaxed text-muted-foreground">
|
||||
{Math.round(lock.report.rate * 100)}% over {lock.report.n} runs (
|
||||
{lock.report.perPrompt.length} prompts x{" "}
|
||||
{lock.report.perPrompt[0].trials} trials, {lock.model}, measured{" "}
|
||||
{lock.builtAt.slice(0, 10)}). Run it against your own skill and you
|
||||
get the same shape of answer: which phrasings reach it, and which
|
||||
quietly do not. Billed to the Claude subscription you already pay
|
||||
for, not a metered key.
|
||||
</p>
|
||||
</div>
|
||||
</section>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"source": "experimental_verifyPluginGuards(\"test/dogfood/davila7-force-push-blocker@869640b\")",
|
||||
"regenerate": "node scripts/gen-davila7-expected.mjs",
|
||||
"plugin": "test/dogfood/davila7-force-push-blocker@869640b",
|
||||
"rows": [
|
||||
{
|
||||
"id": "force-push",
|
||||
"label": "git push --force to a protected branch",
|
||||
"command": "git push --force origin main",
|
||||
"blocked": true,
|
||||
"ran": true,
|
||||
"reason": "matches `git push *--force*` via `git push --force origin main`"
|
||||
},
|
||||
{
|
||||
"id": "rm-rf",
|
||||
"label": "rm -rf of a broad path",
|
||||
"command": "rm -rf /",
|
||||
"blocked": false,
|
||||
"ran": false,
|
||||
"reason": "does not match `git push *--force*` — Claude Code would not run this hook"
|
||||
},
|
||||
{
|
||||
"id": "read-ssh-key",
|
||||
"label": "read a private SSH key",
|
||||
"command": "cat ~/.ssh/id_rsa",
|
||||
"blocked": false,
|
||||
"ran": false,
|
||||
"reason": "does not match `git push *--force*` — Claude Code would not run this hook"
|
||||
}
|
||||
],
|
||||
"spellings": {
|
||||
"fed": 44,
|
||||
"blocked": 44
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "superpowers-tdd-trigger-rate",
|
||||
"inputsHash": "b45c96b31c5a0b45",
|
||||
"model": "sonnet",
|
||||
"harnessVersionKey": "2.1",
|
||||
"evalApiVersion": 1,
|
||||
"builtAt": "2026-09-09T01:38:23.831Z",
|
||||
"report": {
|
||||
"rate": 0.8333333333333334,
|
||||
"n": 30,
|
||||
"perPrompt": [
|
||||
{
|
||||
"prompt": "Add an `isEven(n)` function to utils.js — write it test-first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Implement a stack class in stack.js. Use TDD.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Fix the off-by-one in paginate(); add a regression test first.",
|
||||
"fired": 0,
|
||||
"trials": 3,
|
||||
"rate": 0
|
||||
},
|
||||
{
|
||||
"prompt": "Build a small LRU cache, driving it with failing tests first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Add a `slugify` helper — red/green/refactor please.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Write a rate limiter; start from the tests and work outward.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Implement currency rounding with a test-first approach.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Add retry-with-backoff to the API client, tests leading.",
|
||||
"fired": 1,
|
||||
"trials": 3,
|
||||
"rate": 0.3333333333333333
|
||||
},
|
||||
{
|
||||
"prompt": "Create a debounce utility; specify behaviour as tests first.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
},
|
||||
{
|
||||
"prompt": "Parse ISO durations into seconds — write the spec before the code.",
|
||||
"fired": 3,
|
||||
"trials": 3,
|
||||
"rate": 1
|
||||
}
|
||||
],
|
||||
"competitors": 1,
|
||||
"namespace": "superpowers",
|
||||
"usage": {
|
||||
"totalCostUsd": 1.6588511999999997,
|
||||
"meanCostUsd": 0.05529503999999999,
|
||||
"meanDurationMs": 8713.566666666668,
|
||||
"totalInputTokens": 150,
|
||||
"totalOutputTokens": 12317,
|
||||
"totalCacheCreationTokens": 233925,
|
||||
"totalCacheReadTokens": 2998406
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { assertHookBlocked, runHook } from "vigiles";
|
||||
|
||||
// The guard your plugin already ships.
|
||||
const guard = "./hooks/block-force-push.sh";
|
||||
|
||||
const blocked = runHook(guard, {
|
||||
event: "PreToolUse",
|
||||
tool: "Bash",
|
||||
input: { command: "git push --force origin main" },
|
||||
});
|
||||
|
||||
assertHookBlocked(blocked); // fails CI the day it stops blocking
|
||||
+3
-1
@@ -18,7 +18,9 @@
|
||||
"@/*": ["./src/*"],
|
||||
"@engine/scan-files": ["../dist/scan-files.d.ts"],
|
||||
"@engine/audit-report": ["../dist/audit-report.d.ts"],
|
||||
"@engine/spec": ["../dist/core/spec.d.ts"]
|
||||
"@engine/spec": ["../dist/core/spec.d.ts"],
|
||||
"vigiles": ["../dist/test.d.ts"],
|
||||
"vigiles/eval": ["../dist/eval-surface.d.ts"]
|
||||
}
|
||||
},
|
||||
"include": ["src", "vite.config.ts", "vite.aliases.ts"]
|
||||
|
||||
+9
-1
@@ -19,8 +19,16 @@ const checkPages = existsSync(checksDir)
|
||||
.filter((e) => e.isDirectory())
|
||||
.map((e) => `checks/${e.name}/index.html`)
|
||||
: [];
|
||||
// The comparison page is a sibling static entry (site/comparison/index.html, written
|
||||
// by the same prebuild generator). Guarded like checkPages so `test:browser`, which
|
||||
// does not run the generator, still resolves to just the landing.
|
||||
const comparisonPage = existsSync(
|
||||
fileURLToPath(new URL("comparison/index.html", import.meta.url)),
|
||||
)
|
||||
? ["comparison/index.html"]
|
||||
: [];
|
||||
const inputs = Object.fromEntries(
|
||||
["index.html", ...checkPages].map((rel) => [
|
||||
["index.html", ...checkPages, ...comparisonPage].map((rel) => [
|
||||
rel.replace(/\/index\.html$/, "").replace(/\.html$/, "") || "index",
|
||||
fileURLToPath(new URL(rel, import.meta.url)),
|
||||
]),
|
||||
|
||||
@@ -434,6 +434,7 @@ const evalEnv = (o: Partial<Parameters<typeof decideRunScripts>[0]> = {}) => ({
|
||||
isTTY: false,
|
||||
all: false,
|
||||
yes: false,
|
||||
lockCheck: false,
|
||||
...o,
|
||||
});
|
||||
|
||||
@@ -488,6 +489,31 @@ test("decideRunScripts: bare eval over many, at a TTY → CONFIRM", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// Both directions, because either alone is worthless here. The gate exists to
|
||||
// stop an unbounded fan-out from spending model quota; `--check` cannot spend
|
||||
// any (decideLock in check mode returns `replay` or `stale`, never `run`), so it
|
||||
// must pass the gate while a bare run over the same set is still refused.
|
||||
//
|
||||
// This is the case CI met on 2026-09-09: the `eval-check` step had never once
|
||||
// executed, because with no lock committed anywhere `eval --check` returned
|
||||
// early on `anyLocksCommitted`. The first commit of a lock reached this gate and
|
||||
// was refused exit 2 — a step that had been green only because it never ran.
|
||||
test("decideRunScripts: --check verifies locks, so it is NOT quota-gated", () => {
|
||||
assert.deepEqual(
|
||||
decideRunScripts(
|
||||
evalEnv({ matchedCount: 23, isTTY: false, lockCheck: true }),
|
||||
),
|
||||
{ kind: "run" },
|
||||
);
|
||||
});
|
||||
|
||||
test("decideRunScripts: the same set WITHOUT --check is still refused", () => {
|
||||
assert.deepEqual(
|
||||
decideRunScripts(evalEnv({ matchedCount: 23, isTTY: false })),
|
||||
{ kind: "refuse", count: 23 },
|
||||
);
|
||||
});
|
||||
|
||||
test("the runner reads back WHICH surfaces a script exercised", async () => {
|
||||
// The channel's second job: coverage answers "tested?" from execution, and
|
||||
// this is the wire it travels on. The fixture attributes through the tier
|
||||
|
||||
@@ -434,6 +434,19 @@ export interface RunScriptsEnv {
|
||||
readonly all: boolean;
|
||||
/** `--yes` / `--no-interactive` — agent/CI mode: never prompt. */
|
||||
readonly yes: boolean;
|
||||
/**
|
||||
* `--check` — VERIFY committed eval locks rather than measure. `decideLock`
|
||||
* in check mode returns only `replay` (the recorded report, no model call) or
|
||||
* `stale` (a failure), NEVER `run` — so this path cannot spend quota, and the
|
||||
* quota consent below must not stand in its way.
|
||||
*
|
||||
* Measured 2026-09-09: the CI `eval-check` step had never once executed. With
|
||||
* no lock committed anywhere, `eval --check` short-circuited on
|
||||
* `anyLocksCommitted` and returned "skip"; the first repo to commit a lock got
|
||||
* past that early return, reached this gate, and was refused exit 2. A gate
|
||||
* that is green because it never runs is the failure this repo keeps naming.
|
||||
*/
|
||||
readonly lockCheck: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -450,6 +463,8 @@ export function decideRunScripts(o: RunScriptsEnv): RunScriptsDecision {
|
||||
if (o.kind === "test") return { kind: "run" };
|
||||
if (o.explicitTargets) return { kind: "run" };
|
||||
if (o.all || o.yes) return { kind: "run" };
|
||||
// Verifying a lock is not spending quota — see `lockCheck`.
|
||||
if (o.lockCheck) return { kind: "run" };
|
||||
// A bounded no-target run (0 = no-op, 1 = a single obviously-intended eval) is
|
||||
// not the footgun; the footgun is fanning out over the whole tree.
|
||||
if (o.matchedCount <= 1) return { kind: "run" };
|
||||
|
||||
@@ -6371,6 +6371,7 @@ async function handleRunScripts(
|
||||
isTTY: (process.stdin.isTTY ?? false) && (process.stdout.isTTY ?? false),
|
||||
all: args.includes("--all"),
|
||||
yes: args.includes("--yes") || args.includes("--no-interactive"),
|
||||
lockCheck: args.includes("--check"),
|
||||
});
|
||||
if (runDecision.kind === "refuse") {
|
||||
console.error(
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* The UPKEEP CONTRACT behind the /comparison page.
|
||||
*
|
||||
* Every measured cell on that page is a claim about SOMEBODY ELSE'S product, so it
|
||||
* rots the moment Anthropic ships. Four such claims had to be deleted from this
|
||||
* repository on 2026-09-08/09 for exactly that reason, and each had been written from
|
||||
* prose rather than a run. The snapshot removes the first failure (a hand-typed cell);
|
||||
* this test removes the second (a cell that was measured once and quietly aged).
|
||||
*
|
||||
* It does NOT re-run the probe. A red build here must never mean "Anthropic shipped a
|
||||
* release" — that is not a defect in this repo, and a gate that fires on someone else's
|
||||
* calendar gets disabled. It checks only that the STAMP is current, the same
|
||||
* distinction eval-lock.ts draws between a committed integrity stamp and a cache.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
// `__dirname`, NOT import.meta: src/ compiles to CommonJS, where import.meta is a
|
||||
// tsc error (TS1470) even though vitest — which strips types — runs it happily. The
|
||||
// idiom is legal one directory over in scripts/, which is outside the build. At
|
||||
// runtime this file lives in dist/, so the repo root is one level up.
|
||||
const SNAPSHOT = resolve(
|
||||
__dirname,
|
||||
"..",
|
||||
"site/src/comparison/validate-overlap.json",
|
||||
);
|
||||
|
||||
interface Snapshot {
|
||||
tool: string;
|
||||
version: string;
|
||||
measuredAt: string;
|
||||
cases: { id: string; rule: string; flagged: boolean }[];
|
||||
}
|
||||
const snapshot = JSON.parse(readFileSync(SNAPSHOT, "utf8")) as Snapshot;
|
||||
|
||||
/** major.minor — a patch bump is not worth a red build. */
|
||||
const minorOf = (v: string): string => /^(\d+\.\d+)/.exec(v)?.[1] ?? "";
|
||||
|
||||
function installedClaude(): string | null {
|
||||
try {
|
||||
return execFileSync("claude", ["--version"], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
describe("the /comparison snapshot", () => {
|
||||
it("carries the provenance every rendered cell cites", () => {
|
||||
expect(snapshot.tool).toBe("claude plugin validate");
|
||||
expect(snapshot.version).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
expect(snapshot.measuredAt).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
||||
expect(snapshot.cases.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("records a verdict for every case, so no cell renders from a gap", () => {
|
||||
for (const c of snapshot.cases) {
|
||||
expect(c.id, "case id").toBeTruthy();
|
||||
expect(typeof c.flagged, `flagged for ${c.id}`).toBe("boolean");
|
||||
}
|
||||
});
|
||||
|
||||
it("is not stale against the installed Claude Code", () => {
|
||||
const installed = installedClaude();
|
||||
if (installed === null) {
|
||||
// LOUD skip: no `claude` here means the freshness question was NOT answered.
|
||||
console.warn(
|
||||
"⊘ SKIPPED freshness — `claude` is not on PATH, so snapshot staleness is UNCHECKED",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const have = minorOf(installed);
|
||||
const stamped = minorOf(snapshot.version);
|
||||
expect(
|
||||
have,
|
||||
`The /comparison page cites ${snapshot.tool} ${snapshot.version}, but the installed Claude Code is ${installed}. ` +
|
||||
`Every measured cell on that page may now be false. Re-measure and commit:\n` +
|
||||
` node tools/measure-validate-overlap.mjs --json site/src/comparison/validate-overlap.json`,
|
||||
).toBe(stamped);
|
||||
});
|
||||
});
|
||||
@@ -12,79 +12,222 @@
|
||||
* it does. Both errors came from the same habit: sourcing a claim about another
|
||||
* tool from prose about that tool. This script is the alternative.
|
||||
*
|
||||
* Same shape and reason as `measure-hook-matcher-semantics.mjs`: ground truth
|
||||
* about someone else's product, measured, so a claim on a public page cites a
|
||||
* command instead of a memory.
|
||||
* ONE FIXTURE PER CASE (changed 2026-09-09). The first version planted all seven
|
||||
* defects in ONE fixture and printed the findings as a set. That answers "how
|
||||
* many of the seven?" and nothing else: with a non-empty result you cannot say
|
||||
* WHICH defect produced it, so no per-defect cell can be sourced from it. Each
|
||||
* case now builds its own isolated fixture, so every verdict is attributable to
|
||||
* exactly one planted defect. The original combined fixture survives as the
|
||||
* `all-at-once` case, so the 0/7 and 1/7 figures already published stay
|
||||
* reproducible by this same script.
|
||||
*
|
||||
* FAIRNESS — WHAT IS NOT ASKED. Only defects a general plugin validator could
|
||||
* reasonably be expected to catch are planted. vigiles rules about vigiles's OWN
|
||||
* artifacts (its SHA integrity stamp, `.spec.ts` adoption, `untested-*` coverage,
|
||||
* doc refs) are deliberately absent: asking another product whether it checks our
|
||||
* hash is a rigged row, and `gate-first-adoption` requires grading others fairly.
|
||||
*
|
||||
* NOT IN CI, deliberately. It runs a third-party binary whose behaviour is the
|
||||
* variable under test — a red build here would mean "Anthropic shipped a
|
||||
* release", which is not a defect in this repo. Run it by hand.
|
||||
*
|
||||
* RUN WHEN: before publishing any claim about what `claude plugin validate` does
|
||||
* or does not catch, and whenever Claude Code ships a new minor. The hook-EVENT
|
||||
* row is the one most likely to grow — it is the only row that already passes,
|
||||
* so it is the direction the tool is evidently moving.
|
||||
* or does not catch, and whenever Claude Code ships a new minor.
|
||||
*
|
||||
* Usage: node tools/measure-validate-overlap.mjs
|
||||
* Needs: `claude` on PATH. Writes only to a temp dir. Makes no network call.
|
||||
* Usage: node tools/measure-validate-overlap.mjs [--json <path>]
|
||||
* Needs: `claude` on PATH. Writes only to a temp dir (and <path> with --json).
|
||||
* Makes no network call.
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { dirname, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
/** The planted defects. Each is a thing vigiles reports; the question is whether
|
||||
* `claude plugin validate` reports it too. */
|
||||
const DEFECTS = [
|
||||
"skill declares a tool that does not exist (`Bahs`)",
|
||||
"skill declares an undeclared MCP server (`mcp__ghost__thing`)",
|
||||
"subagent declares a typo'd tool (`Grpe`)",
|
||||
"subagent declares a never-available tool (`AskUserQuestion`)",
|
||||
"subagent declares a typo'd model (`sonnnet`)",
|
||||
"hook command names a script that does not exist",
|
||||
"hook is registered on a typo'd event (`PreToolUze`)",
|
||||
];
|
||||
|
||||
const HOOKS = {
|
||||
PreToolUze: [
|
||||
const GHOST_HOOK = (event) => ({
|
||||
[event]: [
|
||||
{
|
||||
matcher: "Bash",
|
||||
hooks: [
|
||||
{
|
||||
type: "command",
|
||||
command: "${CLAUDE_PLUGIN_ROOT}/hooks/ghost.sh",
|
||||
},
|
||||
{ type: "command", command: "${CLAUDE_PLUGIN_ROOT}/hooks/ghost.sh" },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
const SKILL = `---
|
||||
name: demo
|
||||
description: a skill declaring a tool that does not exist and an undeclared MCP server
|
||||
allowed-tools: Read, Bahs, mcp__ghost__thing
|
||||
---
|
||||
Body.
|
||||
`;
|
||||
const skill = (body, name = "demo") => ({ kind: "skill", name, body });
|
||||
const agent = (body, name = "rev") => ({ kind: "agent", name, body });
|
||||
|
||||
const AGENT = `---
|
||||
name: rev
|
||||
description: a subagent with a typo'd tool, a never-available tool, and a typo'd model
|
||||
tools: Read, Grpe, AskUserQuestion
|
||||
model: sonnnet
|
||||
---
|
||||
Review it.
|
||||
`;
|
||||
/**
|
||||
* The planted defects. `rule` names the vigiles rule that reports the same thing,
|
||||
* so a consumer can join this snapshot to `src/core/rule-meta.ts` by key rather
|
||||
* than by matching prose.
|
||||
*/
|
||||
const CASES = [
|
||||
{
|
||||
id: "skill-tool-does-not-exist",
|
||||
rule: "subagent-tool-contract",
|
||||
what: "a skill declares a tool that does not exist (`Bahs`)",
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: demo\ndescription: declares a tool that does not exist\nallowed-tools: Read, Bahs\n---\nBody.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "skill-undeclared-mcp-server",
|
||||
rule: "mcp-tool-resolves",
|
||||
what: "a skill names an MCP server the plugin never declares (`mcp__ghost__thing`)",
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: demo\ndescription: names an undeclared MCP server\nallowed-tools: Read, mcp__ghost__thing\n---\nBody.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "subagent-typod-tool",
|
||||
rule: "subagent-tool-contract",
|
||||
what: "a subagent declares a typo'd tool (`Grpe`)",
|
||||
files: [
|
||||
agent(
|
||||
"---\nname: rev\ndescription: declares a typo'd tool\ntools: Read, Grpe\n---\nReview it.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "subagent-never-available-tool",
|
||||
rule: "subagent-tool-contract",
|
||||
what: "a subagent declares a tool a subagent can never have (`AskUserQuestion`)",
|
||||
files: [
|
||||
agent(
|
||||
"---\nname: rev\ndescription: declares a never-available tool\ntools: Read, AskUserQuestion\n---\nReview it.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "subagent-typod-model",
|
||||
rule: "subagent-frontmatter",
|
||||
what: "a subagent declares a typo'd model (`sonnnet`) — silently falls back",
|
||||
files: [
|
||||
agent(
|
||||
"---\nname: rev\ndescription: declares a typo'd model\ntools: Read\nmodel: sonnnet\n---\nReview it.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "subagent-missing-frontmatter",
|
||||
rule: "subagent-frontmatter",
|
||||
what: "a subagent with no name/description — cannot register at all",
|
||||
files: [agent("Just prose, no frontmatter block at all.\n")],
|
||||
},
|
||||
{
|
||||
id: "subagent-disallowed-tools-typo",
|
||||
rule: "disallowed-tools-contract",
|
||||
what: "a subagent's deny-list entry is a typo (`Bahs`) — so it blocks nothing",
|
||||
files: [
|
||||
agent(
|
||||
"---\nname: rev\ndescription: deny-list entry is a typo\ntools: Read, Bash\ndisallowedTools: Bahs\n---\nReview it.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "hook-typod-event",
|
||||
rule: "hook-events",
|
||||
what: "a hook is registered on a typo'd event (`PreToolUze`) — never fires",
|
||||
hooks: GHOST_HOOK("PreToolUze"),
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: "hook-script-missing",
|
||||
rule: "hook-script-exists",
|
||||
what: "a hook command names a script that is not on disk — silently runs nothing",
|
||||
hooks: GHOST_HOOK("PreToolUse"),
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: "skill-malformed-frontmatter",
|
||||
rule: "frontmatter-valid",
|
||||
what: "a skill's `---` block exists but is not valid YAML",
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: demo\ndescription: unbalanced [ bracket: and: colons\nallowed-tools: [Read\n---\nBody.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "skill-missing-frontmatter",
|
||||
rule: "skill-frontmatter",
|
||||
what: "a skill with no name/description — falls back to dir name + first paragraph",
|
||||
files: [skill("Just prose, no frontmatter block at all.\n")],
|
||||
},
|
||||
{
|
||||
id: "skill-description-overlap",
|
||||
rule: "description-overlap",
|
||||
what: "two model-invocable skills with near-identical descriptions — the selector cannot tell them apart",
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: alpha\ndescription: Review the changed code for bugs and correctness problems before merging\n---\nBody.\n",
|
||||
"alpha",
|
||||
),
|
||||
skill(
|
||||
"---\nname: beta\ndescription: Review the changed code for bugs and correctness issues before merging\n---\nBody.\n",
|
||||
"beta",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "skill-resource-missing",
|
||||
rule: "skill-resource-resolves",
|
||||
what: "a skill body links a file that is not on disk",
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: demo\ndescription: links a resource that does not exist\n---\nSee [the reference](./reference.md) for details.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "all-at-once",
|
||||
rule: "(combined)",
|
||||
what: "the original seven-defects-in-one-fixture run, kept so the published 0/7 and 1/7 figures stay reproducible",
|
||||
hooks: GHOST_HOOK("PreToolUze"),
|
||||
files: [
|
||||
skill(
|
||||
"---\nname: demo\ndescription: a skill declaring a tool that does not exist and an undeclared MCP server\nallowed-tools: Read, Bahs, mcp__ghost__thing\n---\nBody.\n",
|
||||
),
|
||||
agent(
|
||||
"---\nname: rev\ndescription: a subagent with a typo'd tool, a never-available tool, and a typo'd model\ntools: Read, Grpe, AskUserQuestion\nmodel: sonnnet\n---\nReview it.\n",
|
||||
),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
/** Build one fixture carrying ALL seven defects, in one of the two shapes a real
|
||||
* user has: a repo-local `.claude/` harness, or a packaged plugin. The shape is
|
||||
/** Build ONE isolated fixture for ONE case, in one of the two shapes a real user
|
||||
* has: a repo-local `.claude/` harness, or a packaged plugin. The shape is
|
||||
* load-bearing — the two do not score the same. */
|
||||
function build(root, shape) {
|
||||
function build(root, shape, kase) {
|
||||
const base = shape === "plugin" ? root : join(root, ".claude");
|
||||
mkdirSync(join(base, "skills", "demo"), { recursive: true });
|
||||
mkdirSync(join(base, "agents"), { recursive: true });
|
||||
writeFileSync(join(base, "skills", "demo", "SKILL.md"), SKILL);
|
||||
writeFileSync(join(base, "agents", "rev.md"), AGENT);
|
||||
mkdirSync(base, { recursive: true });
|
||||
|
||||
// A BENIGN, VALID skill in every fixture. Without it a case whose defect lives
|
||||
// only in settings.json (the hook cases) leaves the directory otherwise empty,
|
||||
// and `validate` answers "No manifest found in directory" — a complaint about
|
||||
// the FIXTURE, not about the planted defect. Measured 2026-09-09: that made
|
||||
// two hook cases look FLAGGED in the plain shape and nearly overturned a
|
||||
// published claim that was in fact correct. The baseline keeps every run a
|
||||
// question about the defect.
|
||||
mkdirSync(join(base, "skills", "baseline"), { recursive: true });
|
||||
writeFileSync(
|
||||
join(base, "skills", "baseline", "SKILL.md"),
|
||||
"---\nname: baseline\ndescription: A valid skill present in every fixture so the directory is a real harness\n---\nBody.\n",
|
||||
);
|
||||
|
||||
for (const f of kase.files ?? []) {
|
||||
const path =
|
||||
f.kind === "skill"
|
||||
? join(base, "skills", f.name, "SKILL.md")
|
||||
: join(base, "agents", `${f.name}.md`);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, f.body);
|
||||
}
|
||||
|
||||
if (shape === "plugin") {
|
||||
mkdirSync(join(root, ".claude-plugin"), { recursive: true });
|
||||
@@ -96,16 +239,16 @@ function build(root, shape) {
|
||||
description: "d",
|
||||
version: "0.0.1",
|
||||
author: { name: "t" },
|
||||
hooks: HOOKS,
|
||||
...(kase.hooks ? { hooks: kase.hooks } : {}),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
} else if (kase.hooks) {
|
||||
writeFileSync(
|
||||
join(base, "settings.json"),
|
||||
JSON.stringify({ hooks: HOOKS }, null, 2),
|
||||
JSON.stringify({ hooks: kase.hooks }, null, 2),
|
||||
);
|
||||
}
|
||||
return root;
|
||||
@@ -121,44 +264,109 @@ function run(dir, strict) {
|
||||
}
|
||||
}
|
||||
|
||||
/** A complaint about the fixture's own scaffolding is a PROBE ERROR, never a
|
||||
* result: counting it as a hit is how a finding about an empty directory gets
|
||||
* read as the tool catching a planted defect. Fail loudly instead. */
|
||||
const SCAFFOLD_NOISE = /No manifest found|marketplace\.json/i;
|
||||
|
||||
const findingsOf = (out, kase, label) => {
|
||||
const found = out
|
||||
.split("\n")
|
||||
.filter((l) => l.trim().startsWith(">"))
|
||||
.map((l) => l.trim().replace(/^>\s*/, ""));
|
||||
const noise = found.filter((f) => SCAFFOLD_NOISE.test(f));
|
||||
if (noise.length > 0) {
|
||||
console.error(
|
||||
`PROBE ERROR [${kase.id} / ${label}] — validate complained about the fixture itself, not the planted defect:\n ${noise.join("\n ")}`,
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
return found;
|
||||
};
|
||||
|
||||
const jsonIdx = process.argv.indexOf("--json");
|
||||
const jsonPath = jsonIdx === -1 ? null : process.argv[jsonIdx + 1];
|
||||
if (jsonIdx !== -1 && !jsonPath) {
|
||||
console.error("--json needs a path");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const version = execFileSync("claude", ["--version"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
const versionNumber = /^([\d.]+)/.exec(version)?.[1] ?? version;
|
||||
const tmp = mkdtempSync(join(tmpdir(), "vigiles-validate-probe-"));
|
||||
|
||||
console.log(`claude --version → ${version}`);
|
||||
console.log(`\nPlanted in EVERY fixture:`);
|
||||
for (const d of DEFECTS) console.log(` · ${d}`);
|
||||
const SHAPES = [
|
||||
["plain", false],
|
||||
["plain --strict", true],
|
||||
["plugin", false],
|
||||
["plugin --strict", true],
|
||||
];
|
||||
|
||||
const results = [];
|
||||
try {
|
||||
for (const shape of ["plain", "plugin"]) {
|
||||
const dir = build(join(tmp, shape), shape);
|
||||
for (const strict of [false, true]) {
|
||||
const label = `${shape}${strict ? " --strict" : ""}`;
|
||||
const out = run(dir, strict);
|
||||
const findings = out
|
||||
.split("\n")
|
||||
.filter((l) => l.trim().startsWith(">"))
|
||||
.map((l) => l.trim().replace(/^>\s*/, ""));
|
||||
console.log(`\n===== ${label} =====`);
|
||||
console.log(
|
||||
findings.length === 0
|
||||
? " (no findings — all seven planted defects passed)"
|
||||
: findings.map((f) => ` FLAGGED: ${f}`).join("\n"),
|
||||
for (const kase of CASES) {
|
||||
const per = {};
|
||||
for (const [label, strict] of SHAPES) {
|
||||
const shape = label.startsWith("plugin") ? "plugin" : "plain";
|
||||
const dir = build(
|
||||
mkdtempSync(join(tmp, `${kase.id}-`.replace(/[^\w-]/g, ""))),
|
||||
shape,
|
||||
kase,
|
||||
);
|
||||
per[label] = findingsOf(run(dir, strict), kase, label);
|
||||
}
|
||||
results.push({ ...kase, files: undefined, hooks: undefined, per });
|
||||
}
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const flaggedAnywhere = (r) => SHAPES.some(([l]) => r.per[l].length > 0);
|
||||
|
||||
if (jsonPath) {
|
||||
const snapshot = {
|
||||
tool: "claude plugin validate",
|
||||
version: versionNumber,
|
||||
measuredAt: new Date().toISOString().slice(0, 10),
|
||||
command: "node tools/measure-validate-overlap.mjs --json <path>",
|
||||
shapes: SHAPES.map(([l]) => l),
|
||||
cases: results.map((r) => ({
|
||||
id: r.id,
|
||||
rule: r.rule,
|
||||
what: r.what,
|
||||
flagged: flaggedAnywhere(r),
|
||||
per: r.per,
|
||||
})),
|
||||
};
|
||||
mkdirSync(dirname(jsonPath), { recursive: true });
|
||||
writeFileSync(jsonPath, `${JSON.stringify(snapshot, null, 2)}\n`);
|
||||
console.log(
|
||||
`[measure-validate-overlap] ${String(results.length)} case(s) → ${jsonPath} (claude ${versionNumber})`,
|
||||
);
|
||||
} else {
|
||||
console.log(`claude --version → ${version}\n`);
|
||||
for (const r of results) {
|
||||
const hits = SHAPES.filter(([l]) => r.per[l].length > 0).map(([l]) => l);
|
||||
console.log(
|
||||
`${flaggedAnywhere(r) ? "FLAGGED " : "passed "} ${r.id.padEnd(30)} ${
|
||||
hits.length ? `(${hits.join(", ")})` : ""
|
||||
}`,
|
||||
);
|
||||
}
|
||||
const n = results.filter(flaggedAnywhere).length;
|
||||
console.log(
|
||||
`\n${String(n)} of ${String(results.length)} planted defects flagged in ANY shape.`,
|
||||
);
|
||||
console.log(`
|
||||
DISCRIMINATOR — silence only counts once you have shown the tool speaks.
|
||||
Confirmed against 2.1.263 by separate probes, so the passes above are real
|
||||
passes and not a surface it never opened:
|
||||
Confirmed by separate probes, so the passes above are real passes and not a
|
||||
surface it never opened:
|
||||
· invalid manifest JSON → errors
|
||||
· hooks.json missing its root key → errors
|
||||
· agent with no description → warns
|
||||
· a manifest hook on a REAL event → passes (verified with \`Setup\`, which IS
|
||||
a documented Claude Code event; \`PreToolUze\` and \`Sesion\` both warn)
|
||||
`);
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user