zernie__vigiles/src/scaffold-test.ts

/**
 * The deterministic test-gen engine (B1 v0) — skill-internal (the `test-harness`
 * skill drives it; there is no standalone CLI verb).
 *
 * Free-form in, a RUNNABLE starter test out. Given an existing hand-written
 * skill / subagent / hook, emit a scaffolded `*.harness.mjs` / `*.eval.mjs` at the
 * surface's suggested test path — the deterministic counterpart to the
 * `test-harness` SKILL (which picks the tier with a model). The scaffold picks the
 * cheapest meaningful tier for the kind, wires the real public API + the surface's
 * own metadata (name, namespaced id, declared tools), and leaves TODOs only where a
 * human/model must supply judgement (the prompts, the event input, the assertion).
 *
 * Pure + model-free: hand it a `ScaffoldInput`, get back a `{ path, content }`. The
 * CLI resolves a surface path / plugin dir into inputs (reusing the scan + untested
 * detectors) and writes the files; this module owns only the templating.
 */
import { dirname } from "node:path";

export type SurfaceKind = "skill" | "agent" | "hook";

/** The cheapest meaningful tier for a surface kind (mirrors the test-harness skill). */
export type TestTier = "unit" | "harness" | "eval";

/** One field of a subagent's `result()` contract, parsed from its compiled `.md`. */
export interface ContractField {
  readonly name: string;
  /** The `OutputFieldType` literal: `"string" | "number" | "boolean" | "string[]"`. */
  readonly type: string;
}

/**
 * A subagent's typed `result(ok, err)` outcome contract — the typed-spec payoff
 * the generator turns into a real `assertAgentOk` test (no LLM judge). Parsed from
 * the `vigiles:ok` / `vigiles:err` blocks the compiler emits into the agent `.md`.
 */
export interface ResultContract {
  readonly ok: readonly ContractField[];
  readonly err: readonly ContractField[];
}

/** What the generator needs to know about a surface to scaffold its test. */
export interface ScaffoldInput {
  readonly kind: SurfaceKind;
  /** Skill dir name / agent name / hook-script basename. */
  readonly name: string;
  /** Extension to write, from `core/test-file-ext.ts`. Defaults to `mjs`, so a
   *  caller written before TypeScript support keeps its behaviour byte for byte. */
  readonly ext?: string;
  /** Repo-relative path to the surface (SKILL.md / agent .md / hook script). */
  readonly path: string;
  /** Plugin name for the namespaced skill id; a placeholder TODO when unknown. */
  readonly pluginName?: string;
  /** A user-invoked skill — trigger-rate is for model-invocable skills, so note it. */
  readonly userInvoked?: boolean;
  /** A subagent's declared tool contract (drives the assertion hint); null = inherits all. */
  readonly tools?: readonly string[] | null;
  /**
   * The side-effecting tools in the contract (`effectSurface(tools).sideEffecting`,
   * computed by the CLI with the resolved dialect — kept harness-agnostic here). A
   * non-empty list drives a generated SAFETY check: the agent's "hole" is mocked/denied
   * and asserted to stay in its lane — the typed `tools` contract writing its own test.
   */
  readonly sideEffectingTools?: readonly string[];
  /**
   * The subagent's `result()` contract parsed from its compiled `.md`. Present →
   * the generator emits a deterministic `assertAgentOk` outcome test (no LLM judge);
   * the single most-concrete "the typed spec wrote your test" payoff.
   */
  readonly resultContract?: ResultContract | null;
  /** How the CLI invokes the hook (e.g. `bash hooks/pre-edit.sh`); a TODO when unknown. */
  readonly hookCommand?: string;
}

export interface Scaffold {
  readonly path: string;
  readonly content: string;
  readonly kind: SurfaceKind;
  readonly tier: TestTier;
}

const PLUGIN_TODO = "<plugin>";

/**
 * The suggested test path for a surface — mirrors `suggestedTestPath` in
 * `test-coverage.ts` (a skill gets an `.eval.*`, agent/hook a `.harness.*`), so a
 * generated file is colocated where the untested-surface detector looks for it and
 * the surface stops being reported untested.
 *
 * The EXTENSION comes from `core/test-file-ext.ts`: a project that looks like
 * TypeScript gets a `.ts` test without being asked. Both places must agree — a
 * generator that writes `foo.harness.ts` while the finding tells the reader to add
 * `foo.harness.mjs` teaches them the tool contradicts itself.
 */
function suggestedPath(input: ScaffoldInput): string {
  const dir = dirname(input.path);
  const tier = input.kind === "skill" ? "eval" : "harness";
  return `${dir}/${input.name}.${tier}.${input.ext ?? "mjs"}`;
}

function header(title: string, run: string): string {
  return [
    "/**",
    ` * ${title}`,
    " *",
    " * Generated by vigiles (the test-harness skill) — a STARTER, not a finished test. Fill in",
    " * the TODOs (they're where a human/model must supply judgement), then run:",
    ` *   ${run}`,
    " */",
  ].join("\n");
}

/** A hook → the unit tier (`runHook`): free, no model, reaches every event. */
function hookScaffold(input: ScaffoldInput): string {
  const cmd = input.hookCommand ?? `bash ${input.path}`;
  return `${header(
    `Starter unit test for the \`${input.name}\` hook.`,
    `npx vigiles test ${suggestedPath(input)}`,
  )}
import {
  runHook,
  assertHookAllowed,
  verifyGuardrail,
  formatGuardrailReport,
  // assertBlocksDisasters, // uncomment to gate CI on the battery (see below)
} from "vigiles";

const cmd = ${JSON.stringify(cmd)};

// 1) A benign event should pass through.
// TODO: set the event + input your hook actually inspects (PreToolUse/Bash shown).
const benign = {
  hook_event_name: "PreToolUse",
  tool_name: "Bash",
  tool_input: { command: "echo hello" },
};
assertHookAllowed(runHook(cmd, benign));
console.log("✓ ${input.name}: allowed the benign event");

// 2) SAFETY: if this is a guard, PROVE it blocks the dangerous battery (the #1 hook
//    pain is a guard that silently doesn't — exit 1 instead of 2, wrong jq path, …).
//    This prints a coverage map; it does NOT fail by default (your hook may not be
//    meant to block all of these).
console.log(formatGuardrailReport(cmd, verifyGuardrail(cmd)));

// 3) To GATE CI: declare what this guard MUST block, then assert it. Uncomment +
//    pick the categories your hook is responsible for:
// assertBlocksDisasters(cmd, { categories: ["destructive-git"] });
`;
}

/** A skill → the eval tier (`measureTriggerRate`): does its description FIRE? */
function skillScaffold(input: ScaffoldInput): string {
  const id = `${input.pluginName ?? PLUGIN_TODO}:${input.name}`;
  const note = input.userInvoked
    ? "\n// NOTE: this skill is user-invoked (disableModelInvocation). Trigger-rate\n// measures MODEL-invocable skills; either make it model-invocable or test its\n// slash-command invocation with runHarnessTest instead.\n"
    : "";
  return `${header(
    `Starter trigger-rate eval for the \`${id}\` skill (recall + precision).`,
    `npx vigiles eval ${suggestedPath(input)}   # real model, on your subscription`,
  )}
import { paid_measureTriggerRate } from "vigiles/eval"; // paid_ = a real model runs
import { formatTriggerRateReport, assertTriggerRate, skillResolved } from "vigiles";
import { fileURLToPath } from "node:url";
${note}
// TODO: point at the plugin root (the dir holding .claude-plugin/ or skills/).
const pluginDir = fileURLToPath(new URL("../../", import.meta.url));
const skill = ${JSON.stringify(id)};

const report = await paid_measureTriggerRate({
  pluginDir,
  stubSkillBodies: true, // firing is a frontmatter property — stub bodies, pay less
  prompts: [
    // TODO: >=5 varied prompts that SHOULD fire ${input.name} (recall).
    "TODO: a realistic task that should trigger ${input.name}",
  ],
  irrelevantPrompts: [
    // TODO: >=5 unrelated prompts that should NOT fire it (precision).
    "TODO: an unrelated coding task",
  ],
  fired: (t) => skillResolved(t, skill),
  trials: Number(process.env.VIGILES_TRIALS || 1),
});

console.log(formatTriggerRateReport(report));
assertTriggerRate(report, { min: 0.8, maxFalsePositive: 0.3 });
`;
}

/** A JSON value placeholder for an `OutputFieldType`, for the `vigiles:ok` block. */
function placeholderFor(type: unknown): unknown {
  // An enum's placeholder must be a MEMBER, or the scaffolded test fails the moment it
  // is run — a generated test that cannot pass teaches the author to distrust the tool.
  if (Array.isArray(type) && type.length > 0) return type[0];
  switch (type) {
    case "number":
      return 1;
    case "boolean":
      return true;
    case "string[]":
      return ["example"];
    default:
      return "example";
  }
}

/** Render a `result(ok, err)` builder call reconstructed from the parsed contract. */
function renderContractBuilder(contract: ResultContract): string {
  const shape = (fields: readonly ContractField[]): string =>
    `{ ${fields.map((f) => `${f.name}: ${JSON.stringify(f.type)}`).join(", ")} }`;
  return `result(\n  ${shape(contract.ok)},\n  ${shape(contract.err)},\n)`;
}

/**
 * The OUTCOME test, GENERATED FROM the subagent's `result()` contract: reconstruct
 * the contract, build a matching `vigiles:ok` block, and `assertAgentOk` it —
 * deterministic, no LLM judge. This is the typed-spec payoff a markdown
 * `description:` cannot give you: a parseable, typed outcome a test reads directly.
 */
function outcomeSection(
  input: ScaffoldInput,
  contract: ResultContract,
): string {
  const okValue = Object.fromEntries(
    contract.ok.map((f) => [f.name, placeholderFor(f.type)]),
  );
  const firstField = contract.ok[0]?.name;
  const fieldAssertion = firstField
    ? `// TODO: assert the VALUES you expect (the shape is already validated above), e.g.:\n//   assert.ok(value.${firstField}, "expected a ${firstField}");`
    : "";
  return `import assert from "node:assert/strict";
import { experimental_agent } from "vigiles/spec";\nconst { result } = experimental_agent;
import { assertAgentOk } from "vigiles";

// Reconstructed from ${input.name}'s ## Output contract (its compiled .md) — the
// typed result() the spec wrote. assertAgentOk parses + validates the outcome
// with NO model judge; swap \`okOutput\` for a real \`runHarness\` turn (Part B in
// examples/harness/railway-result.harness.mjs) to assert REAL behaviour.
const contract = ${renderContractBuilder(contract)};

const okOutput = [
  "${input.name} finished its task.",
  "\`\`\`vigiles:ok",
  ${JSON.stringify(JSON.stringify(okValue))},
  "\`\`\`",
].join("\\n");

const value = assertAgentOk(okOutput, contract); // deterministic — no LLM judge
${fieldAssertion}
console.log("✓ ${input.name}: result() outcome parses + validates against its typed contract");
`;
}

/** The fallback when the subagent has no `result()` contract — assert a tool use. */
function fallbackSection(input: ScaffoldInput): string {
  const toolHint =
    input.tools && input.tools.length > 0
      ? `assertToolUsed(r, ${JSON.stringify(input.tools[0])}); // its declared contract: ${input.tools.join(", ")}`
      : `assertToolUsed(r, "Task"); // TODO: assert what the subagent should do`;
  return `import { runHarnessTest, assertToolUsed } from "vigiles";

// ${input.name} has no result() contract, so its outcome can't be asserted
// deterministically — add one (result() on its experimental_agent() spec) for a no-judge
// outcome test. For now, assert it reaches for the right tool.
const r = await runHarnessTest({
  plugin: ".", // TODO: the plugin dir holding this subagent
  // TODO: a prompt that dispatches ${input.name} (via the Task tool).
  prompt: "TODO: a task that should dispatch ${input.name}",
  transcript: true,
  model: [{ text: "on it" }],
});

${toolHint}
console.log("✓ ${input.name}: subagent test ran");
`;
}

/**
 * The SAFETY check, GENERATED FROM the subagent's side-effecting `tools`: assert it
 * stays inside its declared write surface and never reaches for a destructive op.
 * The `tools` allowlist + effectSurface identify the "hole"; the check asserts it —
 * a test the typed contract writes for you (markdown can declare the tools, not test them).
 */
function safetySection(
  input: ScaffoldInput,
  sideEffecting: readonly string[],
): string {
  const checks: string[] = [];
  if (sideEffecting.includes("Bash")) {
    checks.push(
      `  notTool("Bash", { command: /git push|rm -rf/ }), // never a destructive op`,
    );
  }
  if (sideEffecting.some((t) => t === "Write" || t === "Edit")) {
    checks.push(
      `  didNotWrite("secrets.env"), // TODO: the path(s) it must NOT write outside its surface`,
    );
  }
  if (checks.length === 0) {
    checks.push(
      `  // TODO: a notTool()/didNotWrite() per side-effecting tool: ${sideEffecting.join(", ")}`,
    );
  }
  return `
// --- Safety (deterministic) — generated from ${input.name}'s side-effecting tools: ${sideEffecting.join(", ")} ---
// In a real run, replace this constructed Trace with a real \`runHarness\` /
// \`measure\` turn (use interceptTools so a real model's attempt is DENIED, never
// executed — see research/eval-architecture.md). The checks below are derived from the
// declared tools contract — the agent's "hole" asserted to stay in its lane.
{
  const trace = {
    output: "done",
    turns: 1,
    hooks: [],
    toolCalls: [
      // TODO: the tool calls a benign run of ${input.name} makes.
      { name: "Bash", input: { command: "git status" } },
    ],
    file: () => null,
  };
  assertChecks(trace, [
${checks.join("\n")}
  ]);
  console.log("✓ ${input.name}: stayed inside its declared side-effect surface");
}
`;
}

/** A subagent → deterministic outcome + safety tests, generated from its typed contract. */
function agentScaffold(input: ScaffoldInput): string {
  const head = header(
    `Starter harness test for the \`${input.name}\` subagent.`,
    `npx vigiles test ${suggestedPath(input)}`,
  );
  const safetyImport =
    input.sideEffectingTools && input.sideEffectingTools.length > 0
      ? `import { notTool, didNotWrite, assertChecks } from "vigiles";\n`
      : "";
  const body = input.resultContract
    ? outcomeSection(input, input.resultContract)
    : fallbackSection(input);
  const safety =
    input.sideEffectingTools && input.sideEffectingTools.length > 0
      ? safetySection(input, input.sideEffectingTools)
      : "";
  return `${head}\n${safetyImport}${body}${safety}`;
}

const TIER: Record<SurfaceKind, TestTier> = {
  hook: "unit",
  skill: "eval",
  agent: "harness",
};

const BUILDER: Record<SurfaceKind, (i: ScaffoldInput) => string> = {
  hook: hookScaffold,
  skill: skillScaffold,
  agent: agentScaffold,
};

/** Scaffold a starter test for one surface. Pure: path + content, no I/O. */
export function scaffoldTest(input: ScaffoldInput): Scaffold {
  return {
    path: suggestedPath(input),
    content: BUILDER[input.kind](input),
    kind: input.kind,
    tier: TIER[input.kind],
  };
}

/** Render a set of scaffolds for the CLI (what was generated, where, which tier). */
export function formatScaffolds(scaffolds: readonly Scaffold[]): string {
  if (scaffolds.length === 0) {
    return "Nothing to scaffold — every surface already has a test, or none was found.";
  }
  const lines = [`Scaffolded ${String(scaffolds.length)} starter test(s):`, ""];
  for (const s of scaffolds) {
    lines.push(`  ${s.path}  [${s.kind} → ${s.tier} tier]`);
  }
  lines.push(
    "",
    "These are STARTERS — fill in the TODOs (prompts / event / assertions), then run",
    "them with `npx vigiles test` (deterministic) or `npx vigiles eval` (real model).",
  );
  return lines.join("\n");
}