refactor(benchmarks): use native canary runners and upgrade to agent-eval v2 (#2785)

Signed-off-by: Colton Padden <colton.padden@vercel.com>

.gitignore

@@ -20,6 +20,7 @@ apps/benchmarks/experiments/
 apps/benchmarks/results/
 apps/benchmarks/evals/*/PROMPT.md
 apps/benchmarks/evals/*/package.json
+apps/benchmarks/evals/*/.eve-authoring-bootstrap.json
 .extension-contracts-cache/
 packages/eve/.workflow-vitest/
 packages/eve/.generated/

apps/benchmarks/README.md

@@ -6,84 +6,46 @@ run in CI or as part of `pnpm test`.
 
 ## Run
 
-The default subject is the current working tree, including uncommitted and untracked files that
+The default subject is the current `main` canary. The runner resolves that moving alias once to its
-Git does not ignore:
+immutable commit URL, then every model, treatment, and repetition uses that same artifact:
 
 ```sh
-pnpm benchmark author-000-imessage
+pnpm benchmark author-001-weather-tool
 pnpm benchmark
-pnpm benchmark author-000-imessage --runs 3
+pnpm benchmark author-001-weather-tool --runs 3
-pnpm benchmark author-000-imessage --model kimi-k3
+pnpm benchmark author-001-weather-tool --model kimi-k3
-pnpm benchmark author-000-imessage --treatment baseline
+pnpm benchmark author-001-weather-tool --treatment baseline
-pnpm benchmark author-000-imessage --dry
+pnpm benchmark author-001-weather-tool --dry
-pnpm benchmark author-000-imessage --verbose
+pnpm benchmark author-001-weather-tool --verbose
-pnpm benchmark author-000-imessage --keep-failures
+pnpm benchmark author-001-weather-tool --keep-failures
+pnpm benchmark author-001-weather-tool --canary main
 ```
 
 `--keep-failures` keeps a run the runner judged an infrastructure failure — a stalled turn, a
 sandbox error — as the final result instead of discarding it. Use it while iterating on the
 harness, when the failure itself is what you want to read.
 
-Set `EVE_BENCHMARK_TRACE_PARTS=1` to print every stream part the harness receives with the gap
+`--canary <ref>` selects another published canary ref. The runner rejects refs without a package
-since the previous one. The harness decides a turn is over by reading those parts, so this is what
+artifact before it starts an eval. Local working trees, unpublished commits, and revision comparisons
-to reach for when a turn ends too early or hangs past its closing message.
+are not supported by the native runner.
 
-Use `--base` to compare a local Git revision with the working tree:
+The runner uses agent-eval's native Gateway harnesses: OpenCode for other providers, Claude Code
-
+for Anthropic models, and Codex for OpenAI models. Each attempt starts an isolated Vercel Sandbox,
-```sh
+then scaffolds the selected immutable canary with `npx` before the coding agent starts.
-pnpm benchmark author-000-imessage --base origin/main --runs 3
-```
-
-Pass `--head` to compare two local revisions instead:
-
-```sh
-pnpm benchmark author-000-imessage \
-  --base origin/main \
-  --head feature-branch \
-  --runs 3
-```
-
-The runner archives each subject locally and uploads it to the sandbox. Revisions and local-only
-commits do not need to be pushed. It maintains two persistent snapshot layers: a dependency
-snapshot keyed by package-manager inputs, and a subject snapshot keyed by the source tree,
-starting point, setup IDs, and bootstrap version. Source-only changes reuse the dependency snapshot
-but create a new subject snapshot. For one eval and one run, `--verbose` streams setup phases,
-assistant text, tool calls, grading, and build progress.
 
 Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated
 by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.
 
-Results are written under `apps/benchmarks/results/`. Each run includes the transcript,
+Results are written under `apps/benchmarks/results/`. Each run includes the native transcript,
-grader output, summary, copied project files, and `project/benchmark/timings.json`. The timing
+grader output, summary, copied project files, and validation output. Vercel Sandbox and AI Gateway
-artifact records the snapshot-cache outcome, source installation and build phases, workspace setup,
+credentials are required.
-each user turn with token and tool-call counts, and grading and validation durations. Use it to
-separate sandbox setup time from agent time when comparing runs. Print a compact local report with:
-
-```sh
-pnpm benchmark:timings results/current/<timestamp>/<case>/run-1
-```
-
-Pass `--json` to print the original timing artifact. Vercel Sandbox and AI Gateway credentials are
-required.
-
-To read a whole results directory at once — pass/fail, agent time against setup time, turn and tool
-counts, tokens, and any stalled turns:
-
-```sh
-node scripts/analyze.mjs results/current/<timestamp>
-```
-
-Pass `--docs` to also list, per run, which docs page the agent entered at and every page it went on
-to read. That is the fastest way to see whether a documentation change moved agents toward the page
-that answers the task or sent them spidering.
 
 ## Publish canonical results
 
-Canonical publication compares the `baseline` and `guided` treatments with the same eve revision,
+Canonical publication compares the `baseline` and `guided` treatments with the same immutable eve
-model, harness, cases, and graders. The matrix holds the harness constant at OpenCode and varies
+canary, model, harness, cases, and graders. The configured harness reflects the provider: OpenCode
-only the model, so rows are comparable. A model ID is selected independently from the coding-agent
+for other providers, Claude Code for Anthropic, and Codex for OpenAI. Publication requires a clean
-harness; adding Claude Code, Codex, or Gemini CLI belongs to a separate harness comparison. Publication
+working tree and defaults to `origin/main`:
-requires a clean working tree and defaults to `origin/main`:
 
 ```sh
 pnpm benchmark:publish --dry
@@ -128,6 +90,6 @@ export default defineAuthoringCase({
 });
 ```
 
-Use `simpleProject` for the selected subject's `eve init` output and `emptyProject` for an empty
+Use `simpleProject` for the selected canary's `eve init` output and `emptyProject` for a project
-directory with the subject CLI installed. Put reusable setup under `lib/setups/`. Prefer source
+the coding agent creates. Put reusable setup under `lib/setups/`. Native runs support one-turn
-and event assertions over an LLM judge.
+cases; the iMessage case remains local-only. Prefer source assertions over an LLM judge.

apps/benchmarks/evals/author-002-new-project/EVAL.ts

@@ -2,13 +2,15 @@ import { existsSync, readFileSync } from "node:fs";
 
 import { expect, test } from "vitest";
 
-import { subjectDefaultAgentModel } from "./grader.js";
+const projectRoot = "wayfinder";
+
+const defaultAgentModel = "openai/gpt-5.6-luna-fast";
 
 test("creates a complete eve project in place", () => {
-  expect(existsSync("agent/channels/eve.ts")).toBe(true);
+  expect(existsSync(`${projectRoot}/agent/channels/eve.ts`)).toBe(true);
-  expect(existsSync("agent/instructions.md")).toBe(true);
+  expect(existsSync(`${projectRoot}/agent/instructions.md`)).toBe(true);
 
-  const packageJson = JSON.parse(readFileSync("package.json", "utf8")) as {
+  const packageJson = JSON.parse(readFileSync(`${projectRoot}/package.json`, "utf8")) as {
     dependencies?: Record<string, string>;
     scripts?: Record<string, string>;
   };
@@ -17,16 +19,16 @@ test("creates a complete eve project in place", () => {
 });
 
 test("authors the requested identity without pinning a different model", () => {
-  const instructions = readFileSync("agent/instructions.md", "utf8");
+  const instructions = readFileSync(`${projectRoot}/agent/instructions.md`, "utf8");
   expect(instructions).toMatch(/Wayfinder/i);
   expect(instructions).toMatch(/travel/i);
 
   // `agent/agent.ts` is optional, and omitting it selects the same default the
   // scaffold pins explicitly. Both shapes satisfy "use the default model"; a
   // different model id does not.
-  if (existsSync("agent/agent.ts")) {
+  if (existsSync(`${projectRoot}/agent/agent.ts`)) {
-    expect(readFileSync("agent/agent.ts", "utf8")).toContain(
+    expect(readFileSync(`${projectRoot}/agent/agent.ts`, "utf8")).toContain(
-      `model: "${subjectDefaultAgentModel()}"`,
+      `model: "${defaultAgentModel}"`,
     );
   }
 });

apps/benchmarks/evals/author-007-digest-schedule/CASE.ts

@@ -4,7 +4,7 @@ export default defineAuthoringCase({
   startingPoint: simpleProject,
   async interact({ send }) {
     await send(
-      "Put the agent on a cron: every weekday at 9am UTC, run it on a short prompt asking for a status digest. Nothing needs to be delivered anywhere — the run log is fine.",
+      "In this eve project, add an agent schedule under `agent/schedules/`: every weekday at 9am UTC, run the agent on a short prompt asking for a status digest. Nothing needs to be delivered anywhere — the eve run log is fine.",
     );
   },
 });

apps/benchmarks/lib/authoring-case.ts

@@ -1,13 +1,6 @@
-import type { HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";
-import type { HarnessAgentSession } from "@ai-sdk/harness/agent";
-
-import { AUTHORING_EVAL_DIRECTORY } from "./paths.js";
-import type { AuthoringTranscriptEntry } from "./protocol.js";
-
 export interface AuthoringSetupContext {
-  readonly sandbox: HarnessV1NetworkSandboxSession;
   readonly workspace: string;
-  readonly artifactsRoot: typeof AUTHORING_EVAL_DIRECTORY;
+  readonly artifactsRoot: string;
   run(command: string, workingDirectory?: string): Promise<void>;
   write(path: string, content: string): Promise<void>;
 }
@@ -34,8 +27,6 @@ export interface AuthoringTurn {
 }
 
 export interface AuthoringInteractionContext {
-  readonly session: HarnessAgentSession;
-  readonly transcript: ReadonlyArray<AuthoringTranscriptEntry>;
   send(prompt: string): Promise<AuthoringTurn>;
 }
 

apps/benchmarks/lib/benchmark-config.test.mjs

@@ -4,6 +4,7 @@ import { test } from "node:test";
 import {
   findBenchmarkModel,
   findPublishedBenchmarkModel,
+  harnessId,
   publishedBenchmark,
   publishedBenchmarkModels,
 } from "./benchmark-config.ts";
@@ -39,6 +40,9 @@ test("publishes only compatibility-validated models", () => {
   assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code");
   assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code");
   assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode");
+  assert.equal(findPublishedBenchmarkModel("gpt-5-6-sol").harness, "Codex");
+  assert.equal(findPublishedBenchmarkModel("gpt-5-6-terra").harness, "Codex");
+  assert.equal(harnessId("Codex"), "codex");
 });
 
 test("allows candidate probes and rejects unknown models", () => {

apps/benchmarks/lib/benchmark-config.ts

@@ -8,7 +8,7 @@ export interface AuthoringBenchmarkModel {
   readonly id: string;
   readonly model: string;
   readonly displayName: string;
-  readonly harness: "OpenCode" | "Claude Code";
+  readonly harness: "OpenCode" | "Claude Code" | "Codex";
   readonly support: AuthoringBenchmarkSupport;
 }
 
@@ -45,14 +45,14 @@ export const benchmarkModels = [
     id: "gpt-5-6-sol",
     model: "openai/gpt-5.6-sol",
     displayName: "GPT-5.6 Sol",
-    harness: "OpenCode",
+    harness: "Codex",
     support: "supported",
   },
   {
     id: "gpt-5-6-terra",
     model: "openai/gpt-5.6-terra",
     displayName: "GPT-5.6 Terra",
-    harness: "OpenCode",
+    harness: "Codex",
     support: "supported",
   },
   {
@@ -110,7 +110,9 @@ export function publishedExperimentId(
 }
 
 export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
-  return harness === "Claude Code" ? "claude-code" : "opencode";
+  if (harness === "Claude Code") return "claude-code";
+  if (harness === "Codex") return "codex";
+  return "opencode";
 }
 
 export function parseAuthoringTreatment(value: string): AuthoringTreatment {

apps/benchmarks/lib/dependency-sandbox.ts

@@ -1,211 +0,0 @@
-import type { HarnessV1SandboxProvider } from "@ai-sdk/harness";
-import { createVercelSandbox } from "@ai-sdk/sandbox-vercel";
-import { Sandbox } from "@vercel/sandbox";
-
-import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";
-import type { BenchmarkTimings } from "./timing.js";
-
-const dependencySnapshots = new Map<string, Promise<string>>();
-const subjectSnapshots = new Map<string, Promise<string>>();
-
-export function createDependencyCachedSandbox(options: {
-  readonly archive: Uint8Array;
-  readonly dependencyArchive: Uint8Array;
-  readonly dependencyDigest: string;
-  readonly ports: ReadonlyArray<number>;
-  readonly env: Readonly<Record<string, string>>;
-  readonly log: (message: string) => void;
-  readonly timings: BenchmarkTimings;
-}): HarnessV1SandboxProvider {
-  const sessionProvider = (snapshotId: string) =>
-    createVercelSandbox({
-      source: { type: "snapshot", snapshotId },
-      ports: [...options.ports],
-      timeout: 15 * 60_000,
-      env: { ...options.env },
-      networkPolicy: "allow-all",
-    });
-
-  return {
-    specificationVersion: "harness-sandbox-v1",
-    providerId: "eve-benchmark-vercel",
-    async createSession(request = {}) {
-      options.log("[setup] preparing dependency cache");
-      const dependencies = await dependencySnapshot(
-        options.dependencyArchive,
-        options.dependencyDigest,
-        options.log,
-        options.timings,
-      );
-      if (request.identity === undefined || request.onFirstCreate === undefined) {
-        return sessionProvider(dependencies).createSession(request);
-      }
-      const subject = await subjectSnapshot(
-        dependencies,
-        options.archive,
-        request.identity,
-        options.env,
-        request.onFirstCreate,
-        request.abortSignal,
-        options.timings,
-      );
-      return sessionProvider(subject).createSession({
-        sessionId: request.sessionId,
-        abortSignal: request.abortSignal,
-      });
-    },
-    async resumeSession(request) {
-      const dependencies = await dependencySnapshot(
-        options.dependencyArchive,
-        options.dependencyDigest,
-        options.log,
-        options.timings,
-      );
-      const provider = sessionProvider(dependencies);
-      if (provider.resumeSession === undefined) {
-        throw new Error("Vercel Sandbox does not support session resume.");
-      }
-      return provider.resumeSession(request);
-    },
-  };
-}
-
-function dependencySnapshot(
-  archive: Uint8Array,
-  digest: string,
-  log: (message: string) => void,
-  timings: BenchmarkTimings,
-): Promise<string> {
-  const name = `eve-benchmark-dependencies-v5-${digest.slice(0, 24)}`;
-  let snapshot = dependencySnapshots.get(name);
-  if (snapshot !== undefined) {
-    timings.record("dependency-snapshot.memory-cache", 0, "success", { name });
-    return snapshot;
-  }
-  snapshot = createDependencySnapshot(name, archive, log, timings);
-  dependencySnapshots.set(name, snapshot);
-  return snapshot;
-}
-
-async function createDependencySnapshot(
-  name: string,
-  archive: Uint8Array,
-  log: (message: string) => void,
-  timings: BenchmarkTimings,
-): Promise<string> {
-  let created = false;
-  const sandbox = await timings.measure("dependency-snapshot.get-or-create", () =>
-    Sandbox.getOrCreate({
-      name,
-      runtime: "node24",
-      timeout: 15 * 60_000,
-      persistent: true,
-      snapshotExpiration: 0,
-      networkPolicy: "allow-all",
-      async onCreate(current) {
-        created = true;
-        log("[setup] fetching workspace dependencies");
-        await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
-        const command = await current.runCommand("bash", [
-          "-lc",
-          `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global pnpm@11.15.0 vitest@4.1.10 && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
-        ]);
-        if (command.exitCode !== 0) {
-          throw new Error(
-            `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,
-          );
-        }
-      },
-    }),
-  );
-  if (!created && sandbox.currentSnapshotId !== undefined) {
-    timings.record("dependency-snapshot.reused", 0, "success", { name });
-    return sandbox.currentSnapshotId;
-  }
-  return timings.measure("dependency-snapshot.publish", () =>
-    stopWithSnapshot(sandbox, "Dependency"),
-  );
-}
-
-// Publish bootstrap mutations explicitly; a layered Vercel template can expose
-// its inherited snapshot ID before those mutations receive a new snapshot.
-function subjectSnapshot(
-  dependencySnapshotId: string,
-  archive: Uint8Array,
-  identity: string,
-  env: Readonly<Record<string, string>>,
-  bootstrap: NonNullable<
-    NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
-  >,
-  abortSignal: AbortSignal | undefined,
-  timings: BenchmarkTimings,
-): Promise<string> {
-  const name = `eve-benchmark-subject-${identity}`;
-  let snapshot = subjectSnapshots.get(name);
-  if (snapshot !== undefined) {
-    timings.record("subject-snapshot.memory-cache", 0, "success", { name });
-    return snapshot;
-  }
-  snapshot = createSubjectSnapshot(
-    name,
-    dependencySnapshotId,
-    archive,
-    env,
-    bootstrap,
-    abortSignal,
-    timings,
-  );
-  subjectSnapshots.set(name, snapshot);
-  return snapshot;
-}
-
-async function createSubjectSnapshot(
-  name: string,
-  dependencySnapshotId: string,
-  archive: Uint8Array,
-  env: Readonly<Record<string, string>>,
-  bootstrap: NonNullable<
-    NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
-  >,
-  abortSignal: AbortSignal | undefined,
-  timings: BenchmarkTimings,
-): Promise<string> {
-  let created = false;
-  const sandbox = await timings.measure("subject-snapshot.get-or-create", () =>
-    Sandbox.getOrCreate({
-      name,
-      source: { type: "snapshot", snapshotId: dependencySnapshotId },
-      timeout: 15 * 60_000,
-      env: { ...env },
-      persistent: true,
-      snapshotExpiration: 0,
-      networkPolicy: "allow-all",
-      signal: abortSignal,
-      async onCreate(current) {
-        created = true;
-        await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
-        const provider = createVercelSandbox({ sandbox: current });
-        const session = await provider.createSession({ abortSignal });
-        await bootstrap(session.restricted(), { abortSignal });
-      },
-    }),
-  );
-  if (!created && sandbox.currentSnapshotId !== undefined) {
-    timings.record("subject-snapshot.reused", 0, "success", { name });
-    return sandbox.currentSnapshotId;
-  }
-  return timings.measure("subject-snapshot.publish", () =>
-    stopWithSnapshot(sandbox, "Subject", abortSignal),
-  );
-}
-
-async function stopWithSnapshot(
-  sandbox: Sandbox,
-  label: string,
-  signal?: AbortSignal,
-): Promise<string> {
-  const stopped = await sandbox.stop(signal === undefined ? undefined : { signal });
-  const snapshotId = stopped.snapshot?.id ?? sandbox.currentSnapshotId;
-  if (snapshotId === undefined) throw new Error(`${label} snapshot was not published.`);
-  return snapshotId;
-}

apps/benchmarks/lib/experiment-files.mjs

@@ -1,6 +1,8 @@
 import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
+import { createJiti } from "jiti";
+
 export function fixtureNames(evalsRoot) {
   return readdirSync(evalsRoot, { withFileTypes: true })
     .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
@@ -8,10 +10,25 @@ export function fixtureNames(evalsRoot) {
     .sort();
 }
 
-export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) {
+export async function prepareFixtures(evalsRoot, subject, names = fixtureNames(evalsRoot)) {
   for (const name of names) {
     const fixtureRoot = join(evalsRoot, name);
-    writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
+    const authoringCase = await loadCase(fixtureRoot);
+    writeFileSync(
+      join(fixtureRoot, "PROMPT.md"),
+      `${await promptForCase(authoringCase, fixtureRoot)}\n`,
+    );
+    writeFileSync(
+      join(fixtureRoot, ".eve-authoring-bootstrap.json"),
+      `${JSON.stringify({
+        startingPoint: authoringCase.startingPoint.workspace,
+        projectDirectory: authoringCase.projectDirectory,
+        revision: subject.revision,
+        setupIds: [authoringCase.startingPoint.setup, authoringCase.setup]
+          .filter(Boolean)
+          .map((setup) => setup.id),
+      })}\n`,
+    );
     writeFileSync(
       join(fixtureRoot, "package.json"),
       `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
@@ -19,29 +36,54 @@ export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) {
   }
 }
 
+async function loadCase(fixtureRoot) {
+  const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+  let authoringCase = await jiti.import(`${fixtureRoot}/CASE.ts`);
+  while (!isAuthoringCase(authoringCase) && hasDefaultExport(authoringCase)) {
+    authoringCase = authoringCase.default;
+  }
+  if (!isAuthoringCase(authoringCase)) {
+    throw new Error(`${fixtureRoot}/CASE.ts must export an authoring case as default.`);
+  }
+  return authoringCase;
+}
+
+async function promptForCase(authoringCase, fixtureRoot) {
+  const prompts = [];
+  await authoringCase.interact({
+    send: async (prompt) => {
+      prompts.push(prompt);
+      return { text: "", toolCalls: [] };
+    },
+  });
+  if (prompts.length !== 1) {
+    throw new Error(
+      `${fixtureRoot}/CASE.ts is multi-turn and cannot run on a native agent-eval runner.`,
+    );
+  }
+  return prompts[0];
+}
+
+function isAuthoringCase(value) {
+  return typeof value === "object" && value !== null && typeof value.interact === "function";
+}
+
+function hasDefaultExport(value) {
+  return typeof value === "object" && value !== null && "default" in value;
+}
+
 export function resetExperiments(experimentsRoot) {
   rmSync(experimentsRoot, { recursive: true, force: true });
   mkdirSync(experimentsRoot, { recursive: true });
 }
 
-export function writeSubjectArchives(experimentsRoot, subject, name) {
-  const archiveName = `${name}.source.tar.gz`;
-  const dependencyArchiveName = `${name}.dependencies.tar.gz`;
-  writeFileSync(join(experimentsRoot, archiveName), subject.archive);
-  writeFileSync(join(experimentsRoot, dependencyArchiveName), subject.dependencyArchive);
-  return { archiveName, dependencyArchiveName };
-}
-
 export function writeExperiment(experimentsRoot, name, options) {
   writeFileSync(
     join(experimentsRoot, `${name}.ts`),
-    `import { readFileSync } from "node:fs";\n` +
+    `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
-      `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
       `export default authoringExperiment({\n` +
-      `  archive: readFileSync(new URL(${JSON.stringify(`./${options.archiveName}`)}, import.meta.url)),\n` +
+      `  revision: ${JSON.stringify(options.revision)},\n` +
-      `  dependencyArchive: readFileSync(new URL(${JSON.stringify(`./${options.dependencyArchiveName}`)}, import.meta.url)),\n` +
+      `  packageSpec: ${JSON.stringify(options.packageSpec)},\n` +
-      `  digest: ${JSON.stringify(options.digest)},\n` +
-      `  dependencyDigest: ${JSON.stringify(options.dependencyDigest)},\n` +
       `  runs: ${options.runs},\n` +
       (options.evals === undefined ? "" : `  evals: ${JSON.stringify(options.evals)},\n`) +
       `  benchmark: ${JSON.stringify(options.benchmark)},\n` +

apps/benchmarks/lib/experiment-files.test.mjs

@@ -9,14 +9,11 @@ import {
   prepareFixtures,
   resetExperiments,
   writeExperiment,
-  writeSubjectArchives,
 } from "./experiment-files.mjs";
 
 const subject = {
-  archive: Buffer.from("source"),
+  revision: "1234567890abcdef1234567890abcdef12345678",
-  dependencyArchive: Buffer.from("dependencies"),
+  packageSpec: "https://pkg.eve.dev/1234567890abcdef1234567890abcdef12345678/eve.tgz",
-  digest: "source-digest",
-  dependencyDigest: "dependency-digest",
 };
 const benchmark = {
   id: "test",
@@ -26,7 +23,7 @@ const benchmark = {
   support: "supported",
 };
 
-test("materializes fixtures and complete experiment inputs", () => {
+test("materializes fixtures and complete experiment inputs", async () => {
   const root = mkdtempSync(join(tmpdir(), "eve-benchmark-experiments-"));
   const evals = join(root, "evals");
   const experiments = join(root, "experiments");
@@ -36,8 +33,12 @@ test("materializes fixtures and complete experiment inputs", () => {
     writeFileSync(join(evals, "not-a-case"), "ignored");
     assert.deepEqual(fixtureNames(evals), ["author-001-first", "author-002-second"]);
 
-    prepareFixtures(evals);
+    await prepareFixtures(evals, subject);
-    assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "");
+    assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "Build it.\n");
+    assert.deepEqual(
+      JSON.parse(readFileSync(join(evals, "author-001-first", ".eve-authoring-bootstrap.json"))),
+      { startingPoint: "scaffolded", revision: subject.revision, setupIds: [] },
+    );
     assert.deepEqual(JSON.parse(readFileSync(join(evals, "author-001-first", "package.json"))), {
       name: "eve-authoring-author-001-first",
       private: true,
@@ -45,25 +46,18 @@ test("materializes fixtures and complete experiment inputs", () => {
     });
 
     resetExperiments(experiments);
-    const archives = writeSubjectArchives(experiments, subject, "published-deadbeef");
     writeExperiment(experiments, "test-opencode--guided", {
-      ...archives,
+      revision: subject.revision,
-      digest: subject.digest,
+      packageSpec: subject.packageSpec,
-      dependencyDigest: subject.dependencyDigest,
       runs: 3,
       evals: ["author-001-first"],
       benchmark,
       treatment: "guided",
     });
 
-    assert.equal(readFileSync(join(experiments, archives.archiveName), "utf8"), "source");
-    assert.equal(
-      readFileSync(join(experiments, archives.dependencyArchiveName), "utf8"),
-      "dependencies",
-    );
     const experiment = readFileSync(join(experiments, "test-opencode--guided.ts"), "utf8");
-    assert.match(experiment, /dependencyArchive: readFileSync/u);
+    assert.match(experiment, /revision: "1234567890abcdef1234567890abcdef12345678"/u);
-    assert.match(experiment, /published-deadbeef\.dependencies\.tar\.gz/u);
+    assert.match(experiment, /pkg\.eve\.dev\/1234567890abcdef1234567890abcdef12345678\/eve\.tgz/u);
   } finally {
     rmSync(root, { recursive: true, force: true });
   }
@@ -72,6 +66,9 @@ test("materializes fixtures and complete experiment inputs", () => {
 function writeCase(evals, name) {
   const root = join(evals, name);
   mkdirSync(root, { recursive: true });
-  writeFileSync(join(root, "CASE.ts"), "export default {};\n");
+  writeFileSync(
+    join(root, "CASE.ts"),
+    'export default { startingPoint: { workspace: "scaffolded" }, async interact({ send }) { await send("Build it."); } };\n',
+  );
   assert.ok(existsSync(root));
 }

apps/benchmarks/lib/experiment.test.mjs

@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { createJiti } from "jiti";
+
+const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+const { authoringExperiment } = await jiti.import(
+  new URL("./experiment.ts", import.meta.url).pathname,
+);
+
+const common = {
+  revision: "1234567890abcdef1234567890abcdef12345678",
+  packageSpec: "https://pkg.eve.dev/1234567890abcdef1234567890abcdef12345678/eve.tgz",
+  treatment: "guided",
+};
+
+test("uses native Gateway agents and canonical cases", () => {
+  const config = authoringExperiment({
+    ...common,
+    benchmark: {
+      id: "gpt-5-6-sol",
+      model: "openai/gpt-5.6-sol",
+      displayName: "GPT-5.6 Sol",
+      harness: "Codex",
+      support: "supported",
+    },
+  });
+
+  assert.equal(config.agent, "vercel-ai-gateway/codex");
+  assert.equal(config.model, "openai/gpt-5.6-sol");
+  assert.deepEqual(config.evals, [
+    "author-001-weather-tool",
+    "author-002-new-project",
+    "author-003-openapi-connection",
+    "author-004-packaged-skill",
+    "author-005-conditional-approval",
+    "author-006-custom-channel",
+    "author-007-digest-schedule",
+  ]);
+  assert.equal(typeof config.setup, "function");
+});
+
+test("strips the Gateway provider prefix for native Claude Code", () => {
+  const config = authoringExperiment({
+    ...common,
+    benchmark: {
+      id: "claude-sonnet-5",
+      model: "anthropic/claude-sonnet-5",
+      displayName: "Claude Sonnet 5",
+      harness: "Claude Code",
+      support: "supported",
+    },
+  });
+
+  assert.equal(config.agent, "vercel-ai-gateway/claude-code");
+  assert.equal(config.model, "claude-sonnet-5");
+});

apps/benchmarks/lib/experiment.ts

@@ -1,33 +1,28 @@
 import type { ExperimentConfig } from "@vercel/agent-eval";
-import { registerAgent } from "@vercel/agent-eval";
 
-import type { AuthoringBenchmarkModel, AuthoringTreatment } from "./benchmark-config.js";
+import {
-import { createAuthoringAgent } from "./harness-agent.js";
+  harnessId,
+  type AuthoringBenchmarkModel,
+  type AuthoringTreatment,
+  publishedBenchmark,
+} from "./benchmark-config.js";
+import { createNativeAuthoringSetup } from "./native-authoring-setup.js";
 
 export function authoringExperiment(options: {
-  readonly archive: Uint8Array;
+  readonly revision: string;
-  readonly dependencyArchive: Uint8Array;
+  readonly packageSpec: string;
-  readonly digest: string;
-  readonly dependencyDigest: string;
   readonly runs?: number;
   readonly evals?: readonly string[];
   readonly benchmark: AuthoringBenchmarkModel;
   readonly treatment: AuthoringTreatment;
   readonly verbose?: boolean;
 }): ExperimentConfig {
-  const agent = createAuthoringAgent({
-    harness: options.benchmark.harness,
-    model: options.benchmark.model,
-    archive: options.archive,
-    dependencyArchive: options.dependencyArchive,
-    digest: options.digest,
-    dependencyDigest: options.dependencyDigest,
-  });
-  registerAgent(agent);
   return {
-    agent: agent.name,
+    agent: `vercel-ai-gateway/${harnessId(options.benchmark.harness)}`,
-    model: options.benchmark.model,
+    model: nativeModel(options.benchmark),
-    evals: process.env.EVE_BENCHMARK_EVAL ?? (options.evals ? [...options.evals] : "*"),
+    evals:
+      process.env.EVE_BENCHMARK_EVAL ??
+      (options.evals ? [...options.evals] : [...publishedBenchmark.caseIds]),
     scripts: ["typecheck", "build"],
     runs: options.runs ?? 1,
     earlyExit: false,
@@ -36,9 +31,16 @@ export function authoringExperiment(options: {
     timeout: Number(process.env.EVE_BENCHMARK_TIMEOUT ?? 900),
     sandbox: "vercel",
     copyFiles: "changed",
-    agentOptions: {
+    setup: createNativeAuthoringSetup({
-      agentsMd: options.treatment === "guided",
+      packageSpec: options.packageSpec,
-      verbose: options.verbose ?? false,
+      revision: options.revision,
-    },
+      treatment: options.treatment,
+    }),
+    agentOptions: { verbose: options.verbose ?? false },
   };
 }
+
+function nativeModel(benchmark: AuthoringBenchmarkModel): string {
+  if (benchmark.harness !== "Claude Code") return benchmark.model;
+  return benchmark.model.replace(/^anthropic\//u, "");
+}

apps/benchmarks/lib/harness-agent.test.mjs

@@ -1,56 +0,0 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-import { createJiti } from "jiti";
-
-const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
-const { addUsage } = await jiti.import(new URL("./harness-agent.ts", import.meta.url).pathname);
-
-test("normalizes nested Claude Code token usage", () => {
-  const usage = {
-    inputTokens: 0,
-    outputTokens: 0,
-    reasoningTokens: 0,
-    cachedInputTokens: 0,
-    cacheWriteTokens: 0,
-  };
-
-  addUsage(usage, {
-    inputTokens: { total: 32_177, noCache: 1, cacheRead: 32_176, cacheWrite: 10_944 },
-    outputTokens: { total: 327, text: 327 },
-  });
-
-  assert.deepEqual(usage, {
-    inputTokens: 1,
-    outputTokens: 327,
-    reasoningTokens: 0,
-    cachedInputTokens: 32_176,
-    cacheWriteTokens: 10_944,
-  });
-});
-
-test("retains flat OpenCode token usage", () => {
-  const usage = {
-    inputTokens: 0,
-    outputTokens: 0,
-    reasoningTokens: 0,
-    cachedInputTokens: 0,
-    cacheWriteTokens: 0,
-  };
-
-  addUsage(usage, {
-    inputTokens: 100,
-    outputTokens: 20,
-    reasoningTokens: 5,
-    cachedInputTokens: 50,
-    cacheWriteTokens: 10,
-  });
-
-  assert.deepEqual(usage, {
-    inputTokens: 100,
-    outputTokens: 20,
-    reasoningTokens: 5,
-    cachedInputTokens: 50,
-    cacheWriteTokens: 10,
-  });
-});

apps/benchmarks/lib/harness-agent.ts

@@ -1,766 +0,0 @@
-import { readFileSync } from "node:fs";
-import type { HarnessV1, HarnessV1NetworkSandboxSession } from "@ai-sdk/harness";
-import type { HarnessAgentSession } from "@ai-sdk/harness/agent";
-import { HarnessAgent } from "@ai-sdk/harness/agent";
-import { createClaudeCode } from "@ai-sdk/harness-claude-code";
-import { createOpenCode } from "@ai-sdk/harness-opencode";
-import type { Agent, AgentRunResult } from "@vercel/agent-eval";
-import { z } from "zod";
-
-import type {
-  AuthoringCase,
-  AuthoringSetup,
-  AuthoringSetupContext,
-  AuthoringTurn,
-} from "./authoring-case.js";
-import type { AuthoringBenchmarkModel } from "./benchmark-config.js";
-import { createDependencyCachedSandbox } from "./dependency-sandbox.js";
-import { loadAuthoringCase } from "./load-authoring-case.js";
-import {
-  AGENT_EVAL_DIRECTORY,
-  AUTHORING_EVAL_DIRECTORY,
-  AUTHORING_EVAL_DIRECTORY_ENV,
-  EVE_PACKAGE_PATH,
-  SOURCE_ARCHIVE_PATH,
-  SOURCE_ROOT,
-  WORKSPACE,
-} from "./paths.js";
-import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.js";
-import { BenchmarkTimings } from "./timing.js";
-
-const HARNESS_BRIDGE_PORT = 4172;
-const POST_RUN_GRADER_DIRECTORY = ".eve-grader";
-const BOOTSTRAP_VERSION = "v9";
-// A turn that stops producing output should end the turn, not the eval: the
-// remaining turns still run and the graders still see what the agent did.
-const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 300);
-// A turn that has produced no chunk for this long is waiting on the runtime, not
-// working. The bound is one slow tool call: the bridge reports a call and its
-// result together once the call returns, so an install that takes minutes looks
-// from here like silence rather than work in flight.
-const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 120);
-// The runtime frequently never closes a turn whose last act was the model
-// talking, so silence after a closing message has to end the turn on its own.
-// Long enough that a pause before the next tool call is not mistaken for one.
-const CLOSING_IDLE_MILLIS = 45_000;
-// How long to let an aborted turn settle before starting the next one.
-const TURN_SETTLE_MILLIS = 15_000;
-
-// The coding agent's own question tool is the one place it can ask the user
-// mid-turn. Left unanswered it waits for a human and the runtime stops driving
-// the turn, so the harness answers it immediately and points the agent at the
-// channel this benchmark can answer on: its reply, which the case's next `send`
-// responds to.
-type AuthoringHarnessAgent = HarnessAgent<HarnessV1, typeof INTERACTIVE_QUESTION_TOOL>;
-
-const INTERACTIVE_QUESTION_TOOL = {
-  question: {
-    description: "Ask the user a question and wait for their answer.",
-    inputSchema: z.looseObject({}),
-    execute: async () =>
-      "The user cannot answer an interactive prompt in this environment. Ask the question in your reply instead and end your turn; the user answers in their next message.",
-  },
-};
-
-export function createAuthoringAgent(subject: {
-  readonly harness: AuthoringBenchmarkModel["harness"];
-  readonly model: string;
-  readonly archive: Uint8Array;
-  readonly dependencyArchive: Uint8Array;
-  readonly digest: string;
-  readonly dependencyDigest: string;
-}): Agent {
-  return {
-    name: harnessId(subject.harness),
-    displayName: `${subject.harness} eve authoring harness`,
-    getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",
-    getDefaultModel: () => subject.model,
-    definition: {
-      name: harnessId(subject.harness),
-      displayName: `${subject.harness} eve authoring harness`,
-      defaultModel: subject.model,
-      o11yAgentName: harnessId(subject.harness),
-      runnerPath: "",
-      getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY",
-      install: () => [],
-      configFiles: () => [],
-      authEnv: () => ({}),
-    },
-    async run(fixturePath, options): Promise<AgentRunResult> {
-      const verbose = options.agentOptions?.verbose === true;
-      const log = (message: string) => {
-        if (verbose) console.log(message);
-      };
-      const authoringCase = await loadAuthoringCase(fixturePath);
-      const setups = [authoringCase.startingPoint.setup, authoringCase.setup].filter(
-        (setup): setup is AuthoringSetup => setup !== undefined,
-      );
-      const timings = new BenchmarkTimings();
-      timings.record("run.context", 0, "success", {
-        sourceArchiveBytes: subject.archive.length,
-        dependencyArchiveBytes: subject.dependencyArchive.length,
-        sourceDigest: subject.digest,
-        dependencyDigest: subject.dependencyDigest,
-        startingPoint: authoringCase.startingPoint.id,
-        setupCount: setups.length,
-      });
-      const sandbox = createDependencyCachedSandbox({
-        archive: subject.archive,
-        dependencyArchive: subject.dependencyArchive,
-        dependencyDigest: subject.dependencyDigest,
-        ports: [HARNESS_BRIDGE_PORT, ...new Set(setups.flatMap((setup) => setup.ports ?? []))],
-        env: {
-          EVE_INIT_PACKAGE_SPEC: EVE_PACKAGE_PATH,
-          [AUTHORING_EVAL_DIRECTORY_ENV]: AUTHORING_EVAL_DIRECTORY,
-          ...Object.assign({}, ...setups.map((setup) => setup.environment ?? {})),
-        },
-        log,
-        timings,
-      });
-      const startedAt = Date.now();
-      const commands: string[] = [];
-      const transcript: AuthoringTranscriptEntry[] = [];
-      let session: HarnessAgentSession | undefined;
-      let activeSandbox: HarnessV1NetworkSandboxSession | undefined;
-      let workspace: string | undefined;
-
-      const agent = new HarnessAgent({
-        id: "eve-authoring-eval",
-        harness: createHarness(subject.harness, options.model ?? subject.model),
-        sandbox,
-        tools: INTERACTIVE_QUESTION_TOOL,
-        sandboxConfig: {
-          workDir: WORKSPACE,
-          bootstrapHash: bootstrapHash(authoringCase, subject),
-          onBootstrap: async ({ session: bootstrap, workDir }) => {
-            const bootstrapSandbox = bootstrap as HarnessV1NetworkSandboxSession;
-            const context = setupContext(bootstrapSandbox, workDir);
-            log("[setup] building the selected eve source");
-            await bootstrapSubject(
-              bootstrapSandbox,
-              workDir,
-              authoringCase.startingPoint.workspace,
-              subject.archive,
-              timings,
-            );
-            // Case setup runs against the finished starting point. A setup that
-            // installs a fixture dependency would otherwise create the project's
-            // `package.json` itself, and `eve init` would then treat the
-            // workspace as an existing package and skip its own scaffold.
-            await timings.measure("subject.case-bootstrap", async () => {
-              for (const setup of setups) await setup.onBootstrap?.(context);
-            });
-          },
-          onSession: async ({ session: current, sessionWorkDir }) => {
-            activeSandbox = current as HarnessV1NetworkSandboxSession;
-            workspace = sessionWorkDir;
-            const context = setupContext(activeSandbox, workspace);
-            await ensureWorkspace(
-              context,
-              authoringCase.startingPoint.workspace,
-              subject.archive,
-              timings,
-            );
-            await timings.measure("session.setup", async () => {
-              for (const setup of setups) await setup.onSession?.(context);
-              if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);
-            });
-            if (options.agentOptions?.agentsMd !== true) {
-              await context.run("rm -f AGENTS.md CLAUDE.md GEMINI.md");
-            }
-          },
-        },
-        permissionMode: "allow-all",
-      });
-
-      try {
-        session = await timings.measure("session.create", () =>
-          agent.createSession({ abortSignal: options.signal }),
-        );
-        if (activeSandbox === undefined || workspace === undefined) {
-          throw new Error("HarnessAgent did not initialize its sandbox session.");
-        }
-
-        await authoringCase.interact({
-          session,
-          transcript,
-          send: async (prompt) => {
-            const turn = transcript.filter((entry) => entry.role === "user").length + 1;
-            transcript.push({ role: "user", content: prompt });
-            if (verbose) console.log(`[user] ${prompt}`);
-            const result = await timings.measure(`agent.turn.${turn}`, () =>
-              runTurn({
-                agent,
-                session: session!,
-                prompt,
-                timeout: options.timeout,
-                verbose,
-                abortSignal: options.signal,
-              }),
-            );
-            const toolCalls = result.toolCalls;
-            const usage = result.usage;
-            transcript.push({ role: "assistant", content: result.text, toolCalls, usage });
-            const details: Record<string, string | number | boolean> = {
-              promptCharacters: prompt.length,
-              responseCharacters: result.text.length,
-              toolCalls: toolCalls.length,
-              inputTokens: usage.inputTokens,
-              outputTokens: usage.outputTokens,
-              reasoningTokens: usage.reasoningTokens,
-            };
-            if (result.stall !== undefined) details.stall = result.stall;
-            timings.record(
-              `agent.turn.${turn}.summary`,
-              0,
-              result.stall === undefined ? "success" : "failure",
-              details,
-            );
-            if (result.stall !== undefined) log(`[turn ${turn}] ${result.stall}`);
-            commands.push(...shellCommands(result.toolCalls));
-            return { text: result.text, toolCalls };
-          },
-        });
-
-        const projectWorkspace =
-          authoringCase.projectDirectory === undefined
-            ? workspace
-            : `${workspace}/${authoringCase.projectDirectory}`;
-        const context = setupContext(activeSandbox, workspace);
-        const graderContext = setupContext(activeSandbox, projectWorkspace);
-        await prepareGraderDirectory(context);
-        await context.write(
-          `${AGENT_EVAL_DIRECTORY}/results.json`,
-          JSON.stringify({ o11y: { shellCommands: commands.map((command) => ({ command })) } }),
-        );
-        await context.write(
-          `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,
-          JSON.stringify(transcript),
-        );
-        await Promise.all([
-          graderContext.write(
-            `${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
-            readFileSync(`${fixturePath}/EVAL.ts`, "utf8"),
-          ),
-          graderContext.write(
-            `${POST_RUN_GRADER_DIRECTORY}/grader.ts`,
-            readFileSync(new URL("./grader.ts", import.meta.url), "utf8"),
-          ),
-          graderContext.write(
-            `${POST_RUN_GRADER_DIRECTORY}/paths.ts`,
-            readFileSync(new URL("./paths.ts", import.meta.url), "utf8"),
-          ),
-          graderContext.write(
-            `${POST_RUN_GRADER_DIRECTORY}/protocol.ts`,
-            readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"),
-          ),
-        ]);
-
-        log("[grade] running deterministic assertions");
-        const test = await timings.measure("validation.grader", () =>
-          resultOf(
-            activeSandbox!,
-            `vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
-            projectWorkspace,
-          ),
-        );
-        const scriptsResults = Object.fromEntries(
-          await Promise.all(
-            (options.scripts ?? []).map(async (script) => {
-              log(`[${script}] running`);
-              const result = await timings.measure(`validation.${script}`, () =>
-                resultOf(activeSandbox!, `npm run ${shellQuote(script)}`, projectWorkspace),
-              );
-              return [
-                script,
-                { success: result.exitCode === 0, output: `${result.stdout}${result.stderr}` },
-              ] as const;
-            }),
-          ),
-        );
-        const scriptsPassed = Object.values(scriptsResults).every((result) => result.success);
-        timings.record("run.total", Date.now() - startedAt);
-        await context.write(
-          `${AGENT_EVAL_DIRECTORY}/timings.json`,
-          JSON.stringify(timings.entries),
-        );
-        logTimingSummary(log, timings);
-        log(`[result] ${test.exitCode === 0 && scriptsPassed ? "passed" : "failed"}`);
-        return {
-          success: test.exitCode === 0 && scriptsPassed,
-          output: transcript.at(-1)?.content ?? "",
-          error:
-            test.exitCode === 0 && scriptsPassed ? undefined : `${test.stdout}\n${test.stderr}`,
-          duration: Date.now() - startedAt,
-          testResult: { success: test.exitCode === 0, output: `${test.stdout}${test.stderr}` },
-          transcript: harnessTranscript(transcript),
-          scriptsResults,
-          sandboxId: activeSandbox.id,
-          generatedFiles: timingArtifact(timings),
-        };
-      } catch (error) {
-        timings.record("run.total", Date.now() - startedAt, "failure");
-        if (activeSandbox !== undefined && workspace !== undefined) {
-          await prepareGraderDirectory(setupContext(activeSandbox, workspace)).catch(
-            () => undefined,
-          );
-          await Promise.all([
-            setupContext(activeSandbox, workspace).write(
-              `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,
-              JSON.stringify(transcript),
-            ),
-            setupContext(activeSandbox, workspace).write(
-              `${AGENT_EVAL_DIRECTORY}/timings.json`,
-              JSON.stringify(timings.entries),
-            ),
-          ]).catch(() => undefined);
-        }
-        const result: AgentRunResult = {
-          success: false,
-          output: transcript.at(-1)?.content ?? "",
-          error: error instanceof Error ? error.message : String(error),
-          duration: Date.now() - startedAt,
-          transcript: harnessTranscript(transcript),
-          scriptsResults: {},
-          generatedFiles: timingArtifact(timings),
-        };
-        if (activeSandbox !== undefined) result.sandboxId = activeSandbox.id;
-        return result;
-      } finally {
-        await session?.destroy();
-      }
-    },
-  };
-}
-
-interface AuthoringTurnResult extends AuthoringTurn {
-  readonly usage: AuthoringTokenUsage;
-  /** Set when the turn did not finish on its own and was cut short. */
-  readonly stall?: string;
-}
-
-// One streaming path for every turn, so a turn that never finishes still yields
-// the text and tool calls it produced. The underlying runtime can leave a turn
-// open indefinitely after the model's last message, and a turn that consumed the
-// whole eval budget used to discard the transcript along with it.
-async function runTurn(input: {
-  readonly agent: AuthoringHarnessAgent;
-  readonly session: HarnessAgentSession;
-  readonly prompt: string;
-  readonly timeout: number;
-  readonly verbose: boolean;
-  readonly abortSignal?: AbortSignal;
-}): Promise<AuthoringTurnResult> {
-  const { agent, session, prompt, timeout, verbose } = input;
-  const steps: string[] = [];
-  const toolCalls: Array<{ name: string; input: unknown }> = [];
-  const usage = {
-    inputTokens: 0,
-    outputTokens: 0,
-    reasoningTokens: 0,
-    cachedInputTokens: 0,
-    cacheWriteTokens: 0,
-  };
-  let current = "";
-  let lineOpen = false;
-  const turnTimeout = Math.min(timeout, TURN_TIMEOUT_SECONDS);
-  let stall: string | undefined;
-
-  // The runtime routinely leaves a turn open after the model's last message and
-  // honors neither the total nor the per-chunk budget the SDK passes down, so
-  // the harness has to decide when a turn is over. Text the model never follows
-  // with a tool call is its closing message, so silence after it ends the turn;
-  // silence anywhere else can still be work in flight.
-  const controller = new AbortController();
-  const abort = () => controller.abort();
-  input.abortSignal?.addEventListener("abort", abort, { once: true });
-  const deadline = Date.now() + turnTimeout * 1000;
-
-  try {
-    const result = await agent.stream({
-      session,
-      prompt,
-      timeout: { totalMs: turnTimeout * 1000, chunkMs: TURN_STALL_SECONDS * 1000 },
-      abortSignal: controller.signal,
-    });
-    const stream = result.fullStream[Symbol.asyncIterator]();
-    const tracing = process.env.EVE_BENCHMARK_TRACE_PARTS === "1";
-    let lastPartAt = Date.now();
-    let pendingTools = 0;
-    let stepCalledTool = false;
-    let turnLooksDone = false;
-    for (;;) {
-      const idle = pendingTools > 0 ? Number.POSITIVE_INFINITY : idleBudget(turnLooksDone);
-      const budget = Math.min(idle, deadline - Date.now());
-      const next = await withDeadline(stream.next(), budget);
-      if (next === "expired") {
-        if (Date.now() >= deadline) stall = `turn exceeded its ${turnTimeout}s budget`;
-        else if (!turnLooksDone) stall = `turn produced no output for ${TURN_STALL_SECONDS}s`;
-        await settleStalledTurn(stream, controller, stall ?? "turn ended on its closing message");
-        break;
-      }
-      if (next.done === true) break;
-      const part = next.value;
-      if (tracing) {
-        const now = Date.now();
-        console.log(`[part] +${((now - lastPartAt) / 1000).toFixed(1)}s ${part.type}`);
-        lastPartAt = now;
-      }
-      if (part.type === "text-delta") {
-        current += part.text;
-        // Text the model is not following with a tool call is it signing off.
-        // The runtime often emits nothing after that closing message, not even
-        // the step boundary, so the text itself has to be the signal.
-        turnLooksDone = true;
-        if (verbose) {
-          if (!lineOpen) process.stdout.write("[assistant] ");
-          lineOpen = true;
-          process.stdout.write(part.text);
-        }
-        continue;
-      }
-      if (verbose && lineOpen) process.stdout.write("\n");
-      lineOpen = false;
-      if (part.type === "tool-call") {
-        // Only a tool call reopens the turn. The parts that bracket a message
-        // (`text-start`, `text-end`) are not work, and treating them as work is
-        // what used to hide a closing message behind the full stall budget.
-        turnLooksDone = false;
-        pendingTools += 1;
-        stepCalledTool = true;
-        toolCalls.push({ name: part.toolName, input: part.input });
-        if (verbose) console.log(`[tool] ${formatToolCall(part.toolName, part.input)}`);
-      } else if (part.type === "tool-result" || part.type === "tool-error") {
-        pendingTools = Math.max(0, pendingTools - 1);
-      } else if (part.type === "finish-step") {
-        if (current.trim().length > 0) steps.push(current.trim());
-        current = "";
-        addUsage(usage, (part as { usage?: unknown }).usage);
-        turnLooksDone = !stepCalledTool;
-        stepCalledTool = false;
-      } else if (part.type === "finish") {
-        replaceUsage(usage, (part as { totalUsage?: unknown }).totalUsage);
-        turnLooksDone = true;
-      }
-    }
-  } catch (error) {
-    stall = `turn did not finish: ${error instanceof Error ? error.message : String(error)}`;
-  } finally {
-    input.abortSignal?.removeEventListener("abort", abort);
-    if (verbose && lineOpen) process.stdout.write("\n");
-  }
-
-  if (current.trim().length > 0) steps.push(current.trim());
-  const turn: AuthoringTurnResult = { text: steps.join("\n\n"), toolCalls, usage };
-  return stall === undefined ? turn : { ...turn, stall };
-}
-
-function idleBudget(turnLooksDone: boolean): number {
-  return turnLooksDone ? CLOSING_IDLE_MILLIS : TURN_STALL_SECONDS * 1000;
-}
-
-// Aborting is what marks the turn finished. Walking away from the stream is not
-// enough: the session keeps the turn open and rejects the next prompt as one
-// already in progress, which loses every remaining turn of the case. Draining
-// afterwards gives that settlement time to land before the next prompt.
-async function settleStalledTurn(
-  stream: AsyncIterator<unknown>,
-  controller: AbortController,
-  reason: string,
-): Promise<void> {
-  controller.abort(new Error(reason));
-  await withDeadline(
-    (async () => {
-      try {
-        while ((await stream.next()).done !== true);
-      } catch {
-        // The abort surfaces here as a rejection, which is the settlement.
-      }
-    })(),
-    TURN_SETTLE_MILLIS,
-  );
-}
-
-async function withDeadline<T>(promise: Promise<T>, millis: number): Promise<T | "expired"> {
-  if (millis <= 0) return "expired";
-  let timer: NodeJS.Timeout | undefined;
-  try {
-    return await Promise.race([
-      promise,
-      new Promise<"expired">((resolve) => {
-        timer = setTimeout(() => resolve("expired"), millis);
-      }),
-    ]);
-  } finally {
-    if (timer !== undefined) clearTimeout(timer);
-  }
-}
-
-type MutableAuthoringTokenUsage = {
-  -readonly [Key in keyof AuthoringTokenUsage]: AuthoringTokenUsage[Key];
-};
-
-export function addUsage(total: MutableAuthoringTokenUsage, value: unknown): void {
-  if (typeof value !== "object" || value === null) return;
-  const usage = value as Record<string, unknown>;
-  const input = usage.inputTokens;
-  const output = usage.outputTokens;
-  if (typeof input === "object" && input !== null) {
-    const tokens = input as Record<string, unknown>;
-    total.inputTokens += usageNumber(tokens.noCache ?? tokens.total);
-    total.cachedInputTokens += usageNumber(tokens.cacheRead);
-    total.cacheWriteTokens += usageNumber(tokens.cacheWrite);
-  } else {
-    total.inputTokens += usageNumber(input);
-  }
-  if (typeof output === "object" && output !== null) {
-    total.outputTokens += usageNumber((output as Record<string, unknown>).total);
-  } else {
-    total.outputTokens += usageNumber(output);
-  }
-  total.reasoningTokens += usageNumber(usage.reasoningTokens);
-  total.cachedInputTokens += usageNumber(usage.cachedInputTokens);
-  total.cacheWriteTokens += usageNumber(usage.cacheWriteTokens);
-}
-
-function replaceUsage(total: MutableAuthoringTokenUsage, value: unknown): void {
-  if (typeof value !== "object" || value === null) return;
-  total.inputTokens = 0;
-  total.outputTokens = 0;
-  total.reasoningTokens = 0;
-  total.cachedInputTokens = 0;
-  total.cacheWriteTokens = 0;
-  addUsage(total, value);
-}
-
-function usageNumber(value: unknown): number {
-  return typeof value === "number" && Number.isFinite(value) ? value : 0;
-}
-
-function createHarness(harness: AuthoringBenchmarkModel["harness"], model: string): HarnessV1 {
-  if (harness === "Claude Code") {
-    return createClaudeCode({ auth: { gateway: {} }, model });
-  }
-  return createOpenCode({
-    auth: "ai-gateway",
-    model: openCodeModel(model),
-    port: HARNESS_BRIDGE_PORT,
-  });
-}
-
-function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
-  return harness === "Claude Code" ? "claude-code" : "opencode";
-}
-
-function openCodeModel(model: string): string {
-  const gatewayModel = model.includes("/") ? model : `anthropic/${model}`;
-  // OpenCode's Moonshot provider supplies the OpenAI-compatible transport; the
-  // canonical model ID still makes AI Gateway select the requested provider.
-  return `moonshotai/${gatewayModel}`;
-}
-
-function formatToolCall(name: string, input: unknown): string {
-  if (typeof input === "object" && input !== null) {
-    const command = (input as { command?: unknown }).command;
-    if (typeof command === "string") return command;
-  }
-  return `${name} ${JSON.stringify(input)}`;
-}
-
-async function bootstrapSubject(
-  sandbox: HarnessV1NetworkSandboxSession,
-  workspace: string,
-  workspaceKind: "scaffolded" | "empty",
-  archive: Uint8Array,
-  timings: BenchmarkTimings,
-): Promise<void> {
-  await timings.measure("subject.source-upload", async () => {
-    await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });
-  });
-  await timings.measure("subject.source-install", () =>
-    run(
-      sandbox,
-      `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,
-    ),
-  );
-  await timings.measure("subject.eve-build", () =>
-    run(sandbox, `pnpm --dir ${SOURCE_ROOT} --filter eve build`),
-  );
-  await timings.measure("subject.eve-pack-and-cli", () =>
-    run(
-      sandbox,
-      `mkdir -p /tmp/eve-package && pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package && mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH} && ln -sf ${SOURCE_ROOT}/packages/eve/bin/eve.js /usr/local/bin/eve && command -v eve`,
-    ),
-  );
-  const workspaceCommands: string[] = [];
-  if (workspaceKind === "scaffolded") {
-    workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);
-  }
-  // Grader files are created after the agent finishes so an empty starting point remains empty.
-  workspaceCommands.push("command -v vitest >/dev/null");
-  await timings.measure("subject.workspace-bootstrap", () =>
-    run(sandbox, workspaceCommands.join(" && ")),
-  );
-}
-
-function setupContext(
-  sandbox: HarnessV1NetworkSandboxSession,
-  workspace: string,
-): AuthoringSetupContext {
-  return {
-    sandbox,
-    workspace,
-    artifactsRoot: AUTHORING_EVAL_DIRECTORY,
-    run: async (command, workingDirectory = workspace) => {
-      await run(sandbox, command, workingDirectory);
-    },
-    write: async (path, content) => {
-      await sandbox.writeTextFile({
-        path: path.startsWith("/") ? path : `${workspace}/${path}`,
-        content,
-      });
-    },
-  };
-}
-
-// Created only after the agent's turns finish: an `empty` starting point has to
-// look empty to `eve init .`, which refuses to scaffold into a directory that
-// already holds entries it does not recognize.
-async function ensureWorkspace(
-  context: AuthoringSetupContext,
-  workspaceKind: "scaffolded" | "empty",
-  archive: Uint8Array,
-  timings: BenchmarkTimings,
-): Promise<void> {
-  if (workspaceKind === "empty") {
-    await verifyWorkspace(context, workspaceKind);
-    return;
-  }
-  if (await hasPreparedWorkspace(context)) return;
-
-  await timings.measure("session.workspace-recovery", () =>
-    bootstrapSubject(context.sandbox, context.workspace, workspaceKind, archive, timings),
-  );
-  await verifyWorkspace(context, workspaceKind);
-}
-
-async function hasPreparedWorkspace(context: AuthoringSetupContext): Promise<boolean> {
-  const result = await resultOf(
-    context.sandbox,
-    `test -f package.json && test -f agent/instructions.md && test -f AGENTS.md && test -f ${EVE_PACKAGE_PATH}`,
-    context.workspace,
-  );
-  return result.exitCode === 0;
-}
-
-async function verifyWorkspace(
-  context: AuthoringSetupContext,
-  workspaceKind: "scaffolded" | "empty",
-): Promise<void> {
-  const command =
-    workspaceKind === "empty"
-      ? "test ! -e package.json && test ! -e AGENTS.md && test ! -e agent && test ! -e .eve-grader"
-      : `for path in package.json agent/instructions.md AGENTS.md ${EVE_PACKAGE_PATH}; do test -f "$path" || { echo "Missing prepared workspace file: $path" >&2; exit 1; }; done`;
-  await context.run(command);
-}
-
-async function prepareGraderDirectory(context: AuthoringSetupContext): Promise<void> {
-  await context.run(
-    `mkdir -p ${AGENT_EVAL_DIRECTORY} && printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,
-  );
-}
-
-async function installBaselineEveWrapper(context: AuthoringSetupContext): Promise<void> {
-  await context.run(`
-cli_path="/usr/local/bin/eve"
-test -x "$cli_path"
-real_cli="$cli_path.guided"
-if [ ! -e "$real_cli" ]; then mv "$cli_path" "$real_cli"; fi
-cat >"$cli_path" <<'EOF'
-#!/bin/sh
-"$0.guided" "$@"
-status=$?
-rm -f AGENTS.md CLAUDE.md GEMINI.md
-exit "$status"
-EOF
-chmod +x "$cli_path"
-`);
-}
-
-function timingArtifact(timings: BenchmarkTimings): Record<string, string> {
-  return { "benchmark/timings.json": `${JSON.stringify(timings.entries, null, 2)}\n` };
-}
-
-function logTimingSummary(log: (message: string) => void, timings: BenchmarkTimings): void {
-  for (const timing of timings.entries) {
-    log(`[timing] ${timing.phase}: ${timing.durationMs}ms (${timing.outcome})`);
-  }
-}
-
-function harnessTranscript(transcript: ReadonlyArray<AuthoringTranscriptEntry>): string {
-  return transcript
-    .map((entry) => {
-      const message: {
-        role: AuthoringTranscriptEntry["role"];
-        content: unknown;
-        usage?: AuthoringTokenUsage;
-      } = {
-        role: entry.role,
-        content:
-          entry.role === "assistant"
-            ? [
-                ...(entry.content ? [{ type: "text", text: entry.content }] : []),
-                ...(entry.toolCalls ?? []).map((call) => ({
-                  type: "tool_use",
-                  name: call.name,
-                  input: call.input,
-                })),
-              ]
-            : entry.content,
-      };
-      if (entry.usage !== undefined) message.usage = entry.usage;
-      return JSON.stringify({ type: entry.role, message });
-    })
-    .join("\n");
-}
-
-function shellCommands(toolCalls: ReadonlyArray<{ input: unknown }>): string[] {
-  return toolCalls.flatMap((call) => {
-    if (typeof call.input !== "object" || call.input === null) return [];
-    const command = (call.input as { command?: unknown }).command;
-    return typeof command === "string" ? [command] : [];
-  });
-}
-
-function bootstrapHash(authoringCase: AuthoringCase, subject: { readonly digest: string }): string {
-  const setupIds = [authoringCase.startingPoint.setup, authoringCase.setup]
-    .filter((setup): setup is AuthoringSetup => setup !== undefined)
-    .map((setup) => setup.id)
-    .join("-");
-  return `eve-authoring-${BOOTSTRAP_VERSION}-${subject.digest}-${authoringCase.startingPoint.id}-${setupIds}`;
-}
-
-async function resultOf(
-  sandbox: HarnessV1NetworkSandboxSession,
-  command: string,
-  workingDirectory?: string,
-) {
-  const options: { command: string; workingDirectory?: string } = { command };
-  if (workingDirectory !== undefined) options.workingDirectory = workingDirectory;
-  return sandbox.run(options);
-}
-
-async function run(
-  sandbox: HarnessV1NetworkSandboxSession,
-  command: string,
-  workingDirectory?: string,
-): Promise<void> {
-  const result = await resultOf(sandbox, command, workingDirectory);
-  if (result.exitCode !== 0) {
-    throw new Error(`${command} failed (${result.exitCode}):\n${result.stdout}\n${result.stderr}`);
-  }
-}
-
-function shellQuote(value: string): string {
-  return `'${value.replaceAll("'", `'\\''`)}'`;
-}

apps/benchmarks/lib/native-authoring-setup.test.mjs

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { createJiti } from "jiti";
+
+const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+const { createNativeAuthoringSetup } = await jiti.import(
+  new URL("./native-authoring-setup.ts", import.meta.url).pathname,
+);
+
+test("bootstraps the selected source before a native agent starts", async () => {
+  const commands = [];
+  const writes = [];
+  const setup = createNativeAuthoringSetup({
+    packageSpec: "https://pkg.eve.dev/commit/eve.tgz",
+    revision: "commit",
+    treatment: "baseline",
+  });
+  await setup({
+    async readFile(path) {
+      assert.equal(path, ".eve-authoring-bootstrap.json");
+      return JSON.stringify({
+        startingPoint: "scaffolded",
+        revision: "commit",
+        setupIds: ["inventory-openapi-v1"],
+      });
+    },
+    async writeFiles(files) {
+      writes.push(files);
+    },
+    async runCommand(command, args, options) {
+      commands.push({ command, args, options });
+      return { stdout: "", stderr: "", exitCode: 0 };
+    },
+    getWorkingDirectory: () => "/workspace",
+    setWorkingDirectory() {},
+  });
+
+  assert.match(commands[0].args[1], /rm -f \.eve-authoring-bootstrap\.json/u);
+  assert.deepEqual(Object.keys(writes[0]), ["/usr/local/bin/eve"]);
+  assert.match(writes[0]["/usr/local/bin/eve"], /pkg\.eve\.dev\/commit\/eve\.tgz/u);
+  assert.match(commands[1].args[1], /chmod \+x \/usr\/local\/bin\/eve/u);
+  assert.match(
+    commands[2].args[1],
+    /AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=.* eve init \. --model openai\/gpt-5\.5/u,
+  );
+  assert.match(commands[3].args[1], /mkdir -p agent\/lib/u);
+  assert.match(commands[4].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u);
+  assert.match(commands[5].args[1], /git add \. && git commit --amend --no-edit --quiet/u);
+  assert.deepEqual(Object.keys(writes[1]), ["/workspace/agent/lib/inventory-openapi.ts"]);
+  assert.match(writes[1]["/workspace/agent/lib/inventory-openapi.ts"], /getStock/u);
+});

apps/benchmarks/lib/native-authoring-setup.ts

@@ -0,0 +1,106 @@
+import type { Sandbox } from "@vercel/agent-eval";
+
+import type { AuthoringSetup, AuthoringSetupContext } from "./authoring-case.js";
+import { inventoryOpenApiSetup } from "./setups/inventory-openapi.js";
+
+interface FixtureBootstrap {
+  readonly startingPoint: "scaffolded" | "empty";
+  readonly projectDirectory?: string;
+  readonly revision: string;
+  readonly setupIds: readonly string[];
+}
+
+export function createNativeAuthoringSetup(options: {
+  readonly packageSpec: string;
+  readonly revision: string;
+  readonly treatment: "baseline" | "guided";
+}) {
+  return async (sandbox: Sandbox): Promise<void> => {
+    const bootstrap = JSON.parse(
+      await sandbox.readFile(".eve-authoring-bootstrap.json"),
+    ) as FixtureBootstrap;
+    if (bootstrap.revision !== options.revision) {
+      throw new Error("Native fixture revision does not match the selected subject.");
+    }
+    await run(
+      sandbox,
+      "rm -f .eve-authoring-bootstrap.json CASE.ts package.json package-lock.json PROMPT.md",
+      "fixture cleanup",
+    );
+    await sandbox.writeFiles({
+      "/usr/local/bin/eve": `#!/bin/sh\nexec npx --yes --allow-remote=all --package=${shellQuote(options.packageSpec)} eve "$@"\n`,
+    });
+    await run(sandbox, "chmod +x /usr/local/bin/eve", "eve canary wrapper");
+
+    if (bootstrap.startingPoint === "scaffolded") {
+      await run(
+        sandbox,
+        `AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=${shellQuote(options.packageSpec)} eve init . --model openai/gpt-5.5`,
+        "workspace bootstrap",
+      );
+    }
+
+    if (bootstrap.startingPoint === "empty") {
+      if (bootstrap.projectDirectory === undefined) {
+        throw new Error("An empty native fixture must declare its project directory.");
+      }
+      await sandbox.writeFiles({
+        "package.json": `${JSON.stringify({
+          private: true,
+          scripts: {
+            typecheck: `npm --prefix ${bootstrap.projectDirectory} run typecheck`,
+            build: `npm --prefix ${bootstrap.projectDirectory} run build`,
+          },
+        })}\n`,
+      });
+    }
+
+    const context = setupContext(sandbox, bootstrap.projectDirectory);
+    for (const setup of setupsFor(bootstrap.setupIds)) await setup.onSession?.(context);
+    if (options.treatment === "baseline") await run(sandbox, "rm -f AGENTS.md CLAUDE.md GEMINI.md");
+    await run(sandbox, "git add . && git commit --amend --no-edit --quiet");
+  };
+}
+
+function setupsFor(ids: readonly string[]): readonly AuthoringSetup[] {
+  return ids.flatMap((id) => (id === inventoryOpenApiSetup.id ? [inventoryOpenApiSetup] : []));
+}
+
+function setupContext(sandbox: Sandbox, projectDirectory?: string): AuthoringSetupContext {
+  const workspace =
+    projectDirectory === undefined
+      ? sandbox.getWorkingDirectory()
+      : `${sandbox.getWorkingDirectory()}/${projectDirectory}`;
+  return {
+    workspace,
+    artifactsRoot: "/tmp/photon",
+    run: async (command, workingDirectory = workspace) =>
+      run(sandbox, command, command, undefined, workingDirectory),
+    write: async (path, content) =>
+      sandbox.writeFiles({ [path.startsWith("/") ? path : `${workspace}/${path}`]: content }),
+  };
+}
+
+async function run(
+  sandbox: Sandbox,
+  command: string,
+  label = command,
+  env?: Record<string, string>,
+  cwd?: string,
+): Promise<void> {
+  const result = await sandbox.runCommand(
+    "bash",
+    ["-lc", `timeout 240 bash -lc ${shellQuote(command)}`],
+    {
+      env,
+      cwd,
+    },
+  );
+  if (result.exitCode !== 0) {
+    throw new Error(`${label} failed (${result.exitCode}):\n${result.stdout}\n${result.stderr}`);
+  }
+}
+
+function shellQuote(value: string): string {
+  return `'${value.replaceAll("'", `'\\''`)}'`;
+}

apps/benchmarks/lib/source.mjs

@@ -1,108 +1,48 @@
 import { execFileSync } from "node:child_process";
-import { createHash } from "node:crypto";
-import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
 
-export function workingTreeSubject(repositoryRoot) {
+const PACKAGE_HOST = "https://pkg.eve.dev";
-  return archiveSubject(repositoryRoot, "working tree", "current", (archivePath, environment) => {
+const IMMUTABLE_PACKAGE_PATH = /^\/([0-9a-f]{40})\/eve\.tgz$/u;
-    git(repositoryRoot, ["read-tree", "HEAD"], environment);
-    git(repositoryRoot, ["add", "-A"], environment);
-    const tree = git(repositoryRoot, ["write-tree"], environment).trim();
-    git(repositoryRoot, ["archive", "--format=tar.gz", `--output=${archivePath}`, tree]);
-    return tree;
-  });
-}
 
-export function revisionSubject(repositoryRoot, requestedRevision, label) {
+/** Resolves a mutable canary ref to the immutable artifact all runs must share. */
-  const revision = git(repositoryRoot, [
+export function canarySubject(ref, label, resolve = resolveCanaryPackageSpec) {
-    "rev-parse",
+  const packageSpec = resolve(ref);
-    "--verify",
+  const revision = packageRevision(packageSpec);
-    `${requestedRevision}^{commit}`,
+  return {
-  ]).trim();
-  return archiveSubject(
-    repositoryRoot,
-    revision.slice(0, 12),
     label,
-    (archivePath) => {
+    revision,
-      const tree = git(repositoryRoot, ["rev-parse", `${revision}^{tree}`]).trim();
+    description: revision.slice(0, 12),
-      git(repositoryRoot, ["archive", "--format=tar.gz", `--output=${archivePath}`, revision]);
+    packageSpec,
-      return tree;
+  };
-    },
-    { revision },
-  );
 }
 
-function archiveSubject(repositoryRoot, description, label, createArchive, details = {}) {
+export function resolveCanaryPackageSpec(ref) {
-  const temporaryDirectory = mkdtempSync(join(tmpdir(), "eve-authoring-"));
+  const requested = `${PACKAGE_HOST}/${encodeURIComponent(ref)}/eve.tgz`;
-  const archivePath = join(temporaryDirectory, "source.tar.gz");
+  let resolved;
-  const indexPath = join(temporaryDirectory, "index");
   try {
-    const digest = createArchive(archivePath, { ...process.env, GIT_INDEX_FILE: indexPath });
+    resolved = execFileSync(
-    const archive = readFileSync(archivePath);
+      "curl",
-    return {
+      ["-fsSL", "-o", "/dev/null", "-w", "%{url_effective}", requested],
-      label,
+      {
-      description,
+        encoding: "utf8",
-      archive,
+      },
-      digest,
+    ).trim();
-      dependencyArchive: dependencyArchive(archive),
+  } catch {
-      dependencyDigest: dependencyDigest(archive),
+    throw new Error(
-      ...details,
+      `No eve canary artifact is available for ${JSON.stringify(ref)}. Publish that revision or use a published canary ref such as "main".`,
-    };
+    );
-  } finally {
-    rmSync(temporaryDirectory, { recursive: true, force: true });
   }
+  packageRevision(resolved);
+  return resolved;
 }
 
-function dependencyDigest(archive) {
+export function packageRevision(packageSpec) {
-  const hash = createHash("sha256");
+  const url = new URL(packageSpec);
-  for (const path of dependencyPaths(archive)) {
+  if (url.origin !== PACKAGE_HOST) {
-    const content = execFileSync("tar", ["-xOzf", "-", path], {
+    throw new Error(`Eve canary resolved outside ${PACKAGE_HOST}: ${packageSpec}`);
-      input: archive,
-      maxBuffer: 10 * 1024 * 1024,
-    });
-    hash.update(path).update("\0").update(content).update("\0");
   }
-  return hash.digest("hex");
+  const revision = url.pathname.match(IMMUTABLE_PACKAGE_PATH)?.[1];
-}
+  if (revision === undefined) {
-
+    throw new Error(`Eve canary did not resolve to an immutable revision: ${packageSpec}`);
-function dependencyArchive(archive) {
-  const directory = mkdtempSync(join(tmpdir(), "eve-authoring-dependencies-"));
-  const archivePath = join(directory, "dependencies.tar.gz");
-  try {
-    for (const path of dependencyPaths(archive)) {
-      const destination = join(directory, "input", path);
-      mkdirSync(join(destination, ".."), { recursive: true });
-      writeFileSync(
-        destination,
-        execFileSync("tar", ["-xOzf", "-", path], {
-          input: archive,
-          maxBuffer: 10 * 1024 * 1024,
-        }),
-      );
-    }
-    execFileSync("tar", ["-czf", archivePath, "-C", join(directory, "input"), "."]);
-    return readFileSync(archivePath);
-  } finally {
-    rmSync(directory, { recursive: true, force: true });
   }
-}
+  return revision;
-
-function dependencyPaths(archive) {
-  const paths = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8" })
-    .split("\n")
-    .filter(Boolean);
-  return paths.filter(
-    (path) =>
-      path === ".npmrc" ||
-      path === "package.json" ||
-      path === "pnpm-lock.yaml" ||
-      path === "pnpm-workspace.yaml" ||
-      /(?:^|\/)package\.json$/u.test(path) ||
-      path.startsWith("patches/"),
-  );
-}
-
-function git(cwd, args, env = process.env) {
-  return execFileSync("git", args, { cwd, env, encoding: "utf8" });
 }

apps/benchmarks/lib/source.test.mjs

@@ -1,96 +1,37 @@
 import assert from "node:assert/strict";
-import { execFileSync } from "node:child_process";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
 import { test } from "node:test";
 
-import { revisionSubject, workingTreeSubject } from "./source.mjs";
+import { canarySubject, packageRevision } from "./source.mjs";
 
-test("archives the working tree without changing the index", () => {
+const revision = "1234567890abcdef1234567890abcdef12345678";
-  const repository = repositoryFixture();
+const packageSpec = `https://pkg.eve.dev/${revision}/eve.tgz`;
-  try {
-    writeFileSync(join(repository, "tracked.txt"), "changed\n");
-    writeFileSync(join(repository, "new.txt"), "new\n");
-    writeFileSync(join(repository, "ignored.txt"), "ignored\n");
-    rmSync(join(repository, "deleted.txt"));
-    const statusBefore = git(repository, ["status", "--porcelain"]);
 
-    const subject = workingTreeSubject(repository);
+test("resolves a canary alias once to an immutable subject", () => {
-    withExtracted(subject.archive, (extracted) => {
+  const calls = [];
-      assert.equal(readFileSync(join(extracted, "tracked.txt"), "utf8"), "changed\n");
+  const subject = canarySubject("main", "current", (ref) => {
-      assert.equal(readFileSync(join(extracted, "new.txt"), "utf8"), "new\n");
+    calls.push(ref);
-      assert.equal(existsSync(join(extracted, "ignored.txt")), false);
+    return packageSpec;
-      assert.equal(existsSync(join(extracted, "deleted.txt")), false);
+  });
-    });
+
-    withExtracted(subject.dependencyArchive, (extracted) => {
+  assert.deepEqual(calls, ["main"]);
-      assert.equal(readFileSync(join(extracted, "package.json"), "utf8"), '{"private":true}\n');
+  assert.deepEqual(subject, {
-      assert.equal(
+    label: "current",
-        readFileSync(join(extracted, "pnpm-lock.yaml"), "utf8"),
+    revision,
-        "lockfileVersion: '9.0'\n",
+    description: revision.slice(0, 12),
-      );
+    packageSpec,
-      assert.equal(existsSync(join(extracted, "tracked.txt")), false);
+  });
-    });
-    assert.equal(git(repository, ["status", "--porcelain"]), statusBefore);
-    writeFileSync(join(repository, "tracked.txt"), "another source change\n");
-    const sourceChange = workingTreeSubject(repository);
-    assert.notEqual(sourceChange.digest, subject.digest);
-    assert.equal(sourceChange.dependencyDigest, subject.dependencyDigest);
-    assert.deepEqual(sourceChange.dependencyArchive, subject.dependencyArchive);
-    writeFileSync(join(repository, "packages.json"), "not a manifest\n");
-    assert.equal(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
-    writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n");
-    assert.notEqual(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
-  } finally {
-    rmSync(repository, { recursive: true, force: true });
-  }
 });
 
-test("archives a local revision", () => {
+test("rejects a non-immutable canary URL", () => {
-  const repository = repositoryFixture();
+  assert.throws(
-  try {
+    () => packageRevision("https://pkg.eve.dev/main/eve.tgz"),
-    writeFileSync(join(repository, "tracked.txt"), "working tree\n");
+    /did not resolve to an immutable revision/u,
-    const subject = revisionSubject(repository, "HEAD", "base");
+  );
-    assert.equal(subject.label, "base");
-    withExtracted(subject.archive, (extracted) => {
-      assert.equal(readFileSync(join(extracted, "tracked.txt"), "utf8"), "committed\n");
-    });
-  } finally {
-    rmSync(repository, { recursive: true, force: true });
-  }
 });
 
-function repositoryFixture() {
+test("rejects a canary URL from another origin", () => {
-  const repository = mkdtempSync(join(tmpdir(), "eve-authoring-source-test-"));
+  assert.throws(
-  git(repository, ["init", "--quiet"]);
+    () => packageRevision(`https://example.com/${revision}/eve.tgz`),
-  git(repository, ["config", "user.name", "Test"]);
+    /resolved outside/u,
-  git(repository, ["config", "user.email", "test@example.com"]);
+  );
-  writeFileSync(join(repository, ".gitignore"), "ignored.txt\n");
+});
-  writeFileSync(join(repository, ".npmrc"), "link-workspace-packages=true\n");
-  writeFileSync(join(repository, "package.json"), '{"private":true}\n');
-  writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n");
-  writeFileSync(join(repository, "pnpm-workspace.yaml"), "packages: []\n");
-  writeFileSync(join(repository, "tracked.txt"), "committed\n");
-  writeFileSync(join(repository, "deleted.txt"), "delete me\n");
-  git(repository, ["add", "."]);
-  git(repository, ["commit", "--quiet", "-m", "fixture"]);
-  return repository;
-}
-
-function withExtracted(archive, assertion) {
-  const directory = mkdtempSync(join(tmpdir(), "eve-authoring-archive-test-"));
-  const archivePath = join(directory, "source.tar.gz");
-  const extracted = join(directory, "extracted");
-  try {
-    writeFileSync(archivePath, archive);
-    execFileSync("mkdir", [extracted]);
-    execFileSync("tar", ["-xzf", archivePath, "-C", extracted]);
-    assertion(extracted);
-  } finally {
-    rmSync(directory, { recursive: true, force: true });
-  }
-}
-
-function git(cwd, args) {
-  return execFileSync("git", args, { cwd, encoding: "utf8" });
-}

apps/benchmarks/lib/timing.test.mjs

@@ -1,20 +0,0 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-import { BenchmarkTimings } from "./timing.ts";
-
-test("records successful and failed phases", async () => {
-  const timings = new BenchmarkTimings();
-  await timings.measure("success", async () => undefined, { turns: 1 });
-  await assert.rejects(timings.measure("failure", async () => Promise.reject(new Error("nope"))));
-
-  assert.equal(timings.entries.length, 2);
-  assert.deepEqual(
-    timings.entries.map(({ phase, outcome, details }) => ({ phase, outcome, details })),
-    [
-      { phase: "success", outcome: "success", details: { turns: 1 } },
-      { phase: "failure", outcome: "failure", details: undefined },
-    ],
-  );
-  assert.ok(timings.entries.every((entry) => entry.durationMs >= 0));
-});

apps/benchmarks/lib/timing.ts

@@ -1,55 +0,0 @@
-export interface BenchmarkTiming {
-  readonly phase: string;
-  readonly startedAt: string;
-  readonly durationMs: number;
-  readonly outcome: "success" | "failure";
-  readonly details?: Readonly<Record<string, string | number | boolean>>;
-}
-
-type TimingDetails = BenchmarkTiming["details"];
-
-export class BenchmarkTimings {
-  readonly entries: BenchmarkTiming[] = [];
-
-  async measure<T>(
-    phase: string,
-    operation: () => Promise<T>,
-    details?: TimingDetails,
-  ): Promise<T> {
-    const startedAt = new Date().toISOString();
-    const started = performance.now();
-    try {
-      const result = await operation();
-      this.add(phase, startedAt, performance.now() - started, "success", details);
-      return result;
-    } catch (error) {
-      this.add(phase, startedAt, performance.now() - started, "failure", details);
-      throw error;
-    }
-  }
-
-  record(
-    phase: string,
-    durationMs: number,
-    outcome: BenchmarkTiming["outcome"] = "success",
-    details?: TimingDetails,
-  ): void {
-    this.add(phase, new Date(Date.now() - durationMs).toISOString(), durationMs, outcome, details);
-  }
-
-  private add(
-    phase: string,
-    startedAt: string,
-    durationMs: number,
-    outcome: BenchmarkTiming["outcome"],
-    details: TimingDetails,
-  ): void {
-    const entry = {
-      phase,
-      startedAt,
-      durationMs: Math.round(durationMs),
-      outcome,
-    };
-    this.entries.push(details === undefined ? entry : { ...entry, details });
-  }
-}

apps/benchmarks/package.json

@@ -12,12 +12,7 @@
     "benchmark:timings": "node scripts/timings.mjs"
   },
   "dependencies": {
-    "@ai-sdk/harness": "1.0.87",
+    "@vercel/agent-eval": "2.2.1",
-    "@ai-sdk/harness-claude-code": "1.0.90",
-    "@ai-sdk/harness-opencode": "1.0.88",
-    "@ai-sdk/sandbox-vercel": "1.0.87",
-    "@vercel/agent-eval": "1.5.0",
-    "@vercel/sandbox": "3.2.0",
     "jiti": "2.7.0",
     "zod": "catalog:"
   },

apps/benchmarks/publish.mjs

@@ -12,13 +12,8 @@ import {
   publishedBenchmarkModels,
   publishedExperimentId,
 } from "./lib/benchmark-config.ts";
-import {
+import { prepareFixtures, resetExperiments, writeExperiment } from "./lib/experiment-files.mjs";
-  prepareFixtures,
+import { canarySubject } from "./lib/source.mjs";
-  resetExperiments,
-  writeExperiment,
-  writeSubjectArchives,
-} from "./lib/experiment-files.mjs";
-import { revisionSubject } from "./lib/source.mjs";
 
 const appRoot = dirname(fileURLToPath(import.meta.url));
 const repositoryRoot = resolve(appRoot, "../..");
@@ -51,14 +46,15 @@ if (values["allow-dirty"]) {
   );
 }
 
-const revision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim();
+const requestedRevision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim();
-const subject = revisionSubject(repositoryRoot, revision, "published");
+const subject = canarySubject(requestedRevision, "published");
+const revision = subject.revision;
 const benchmarks = selectedBenchmarks(values.models);
 const experimentNames = benchmarks.flatMap((benchmark) =>
   authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),
 );
 
-prepareFixtures(evalsRoot);
+await prepareFixtures(evalsRoot, subject, publishedBenchmark.caseIds);
 writeExperiments(subject, revision, benchmarks);
 
 console.log(`> eve revision: ${revision}`);
@@ -158,19 +154,11 @@ To inspect pending benchmark cells without publishing:
 
 function writeExperiments(subject, revision, benchmarks) {
   resetExperiments(experimentsRoot);
-  const { archiveName, dependencyArchiveName } = writeSubjectArchives(
-    experimentsRoot,
-    subject,
-    `published-${revision.slice(0, 12)}`,
-  );
-
   for (const benchmark of benchmarks) {
     for (const treatment of authoringTreatments) {
       writeExperiment(experimentsRoot, publishedExperimentId(benchmark, treatment), {
-        archiveName,
+        revision: subject.revision,
-        dependencyArchiveName,
+        packageSpec: subject.packageSpec,
-        digest: subject.digest,
-        dependencyDigest: subject.dependencyDigest,
         runs: publishedBenchmark.runs,
         evals: publishedBenchmark.caseIds,
         benchmark,

apps/benchmarks/run.mjs

@@ -6,14 +6,13 @@ import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 
-import { findBenchmarkModel, parseAuthoringTreatment } from "./lib/benchmark-config.ts";
 import {
-  prepareFixtures,
+  findBenchmarkModel,
-  resetExperiments,
+  parseAuthoringTreatment,
-  writeExperiment,
+  publishedBenchmark,
-  writeSubjectArchives,
+} from "./lib/benchmark-config.ts";
-} from "./lib/experiment-files.mjs";
+import { prepareFixtures, resetExperiments, writeExperiment } from "./lib/experiment-files.mjs";
-import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";
+import { canarySubject } from "./lib/source.mjs";
 
 const appRoot = dirname(fileURLToPath(import.meta.url));
 const repositoryRoot = resolve(appRoot, "../..");
@@ -23,8 +22,7 @@ const { values, positionals } = parseArgs({
   args: process.argv.slice(2),
   allowPositionals: true,
   options: {
-    base: { type: "string" },
+    canary: { type: "string", default: "main" },
-    head: { type: "string" },
     dry: { type: "boolean" },
     runs: { type: "string" },
     model: { type: "string", default: "claude-sonnet-5" },
@@ -44,32 +42,23 @@ if (values.help) {
   process.exit(0);
 }
 if (positionals.length > 1) throw new Error("Expected at most one <eval-name>.");
-if (values.head !== undefined && values.base === undefined) {
-  throw new Error("--head requires --base.");
-}
 const runs = parseRuns(values.runs);
 const selectedEval = positionals[0];
 const treatment = parseAuthoringTreatment(values.treatment);
 const benchmark = findBenchmarkModel(values.model);
-if (values.verbose && (selectedEval === undefined || runs !== 1 || values.base !== undefined)) {
+if (values.verbose && (selectedEval === undefined || runs !== 1)) {
-  throw new Error("--verbose requires one eval, one run, and no revision comparison.");
+  throw new Error("--verbose requires one eval and one run.");
 }
 if (selectedEval !== undefined && !existsSync(join(evalsRoot, selectedEval, "CASE.ts"))) {
   throw new Error(`Unknown eval ${JSON.stringify(selectedEval)}.`);
 }
 
-const workingTree = () => workingTreeSubject(repositoryRoot);
+const subjects = [canarySubject(values.canary, "current")];
-const subjects =
+await prepareFixtures(
-  values.base === undefined
+  evalsRoot,
-    ? [workingTree()]
+  subjects[0],
-    : [
+  selectedEval === undefined ? publishedBenchmark.caseIds : [selectedEval],
-        revisionSubject(repositoryRoot, values.base, "base"),
+);
-        values.head === undefined
-          ? { ...workingTree(), label: "head" }
-          : revisionSubject(repositoryRoot, values.head, "head"),
-      ];
-
-prepareFixtures(evalsRoot, selectedEval === undefined ? undefined : [selectedEval]);
 mkdirSync(join(appRoot, "results"), { recursive: true });
 writeExperiments(subjects, runs, benchmark, treatment, values.verbose ?? false);
 const executable = join(appRoot, "node_modules/.bin/agent-eval");
@@ -82,7 +71,8 @@ for (const subject of subjects) console.log(`> ${subject.label}: ${subject.descr
 const result = spawnSync(executable, args, {
   cwd: appRoot,
   stdio: "inherit",
-  env: { ...process.env, EVE_BENCHMARK_EVAL: selectedEval ?? "*" },
+  env:
+    selectedEval === undefined ? process.env : { ...process.env, EVE_BENCHMARK_EVAL: selectedEval },
 });
 if (result.error) throw result.error;
 process.exit(result.status ?? 1);
@@ -90,16 +80,9 @@ process.exit(result.status ?? 1);
 function writeExperiments(subjects, runs, benchmark, treatment, verbose) {
   resetExperiments(experimentsRoot);
   for (const subject of subjects) {
-    const { archiveName, dependencyArchiveName } = writeSubjectArchives(
-      experimentsRoot,
-      subject,
-      subject.label,
-    );
     writeExperiment(experimentsRoot, subject.label, {
-      archiveName,
+      revision: subject.revision,
-      dependencyArchiveName,
+      packageSpec: subject.packageSpec,
-      digest: subject.digest,
-      dependencyDigest: subject.dependencyDigest,
       runs,
       benchmark,
       treatment,
@@ -118,6 +101,5 @@ function parseRuns(value) {
 
 function usage() {
   console.log(`Usage:
-  pnpm benchmark [eval-name] [--model <id>] [--runs N] [--treatment baseline|guided] [--dry] [--verbose] [--keep-failures]
+  pnpm benchmark [eval-name] [--canary main] [--model <id>] [--runs N] [--treatment baseline|guided] [--dry] [--verbose] [--keep-failures]`);
-  pnpm benchmark [eval-name] --base <revision> [--head <revision>] [--model <id>] [--runs N] [--treatment baseline|guided] [--dry]`);
 }

apps/benchmarks/scripts/cost.mjs

@@ -86,20 +86,19 @@ export const modelPricing = {
 const number = (value) => (typeof value === "number" && Number.isFinite(value) ? value : 0);
 const object = (value) => (value !== null && typeof value === "object" ? value : {});
 
-export function extractRunUsage(raw) {
+export function extractRunUsage(raw, harness) {
   const usage = { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 };
   let found = false;
   for (const line of raw.split("\n")) {
     try {
       const event = JSON.parse(line);
-      if (event.type !== "assistant") continue;
+      const value = usageForEvent(event, harness);
-      const value = object(object(event.message).usage);
+      if (value === undefined) continue;
-      if (Object.keys(value).length === 0) continue;
+      usage.input += value.input;
-      usage.input += number(value.inputTokens);
+      usage.output += value.output;
-      usage.output += number(value.outputTokens);
+      usage.reasoning += value.reasoning;
-      usage.reasoning += number(value.reasoningTokens);
+      usage.cacheRead += value.cacheRead;
-      usage.cacheRead += number(value.cachedInputTokens);
+      usage.cacheWrite += value.cacheWrite;
-      usage.cacheWrite += number(value.cacheWriteTokens);
       found = true;
     } catch {
       // Ignore malformed transcript lines.
@@ -108,6 +107,59 @@ export function extractRunUsage(raw) {
   return found && tokenConsumption(usage) > 0 ? usage : null;
 }
 
+function usageForEvent(event, harness) {
+  const legacy = object(object(event.message).usage);
+  if (event.type === "assistant" && "inputTokens" in legacy) {
+    return {
+      input: number(legacy.inputTokens),
+      output: number(legacy.outputTokens),
+      reasoning: number(legacy.reasoningTokens),
+      cacheRead: number(legacy.cachedInputTokens),
+      cacheWrite: number(legacy.cacheWriteTokens),
+    };
+  }
+
+  if (harness === "OpenCode" && event.type === "step_finish") {
+    const tokens = object(object(event.part).tokens);
+    const cache = object(tokens.cache);
+    return {
+      input: number(tokens.input),
+      output: number(tokens.output),
+      reasoning: number(tokens.reasoning),
+      cacheRead: number(cache.read),
+      cacheWrite: number(cache.write),
+    };
+  }
+
+  if (harness === "Claude Code" && event.type === "assistant") {
+    const usage = object(object(event.message).usage);
+    if (Object.keys(usage).length === 0) return undefined;
+    return {
+      input: number(usage.input_tokens),
+      output: number(usage.output_tokens),
+      reasoning: number(usage.reasoning_tokens),
+      cacheRead: number(usage.cache_read_input_tokens),
+      cacheWrite: number(usage.cache_creation_input_tokens),
+    };
+  }
+
+  if (harness === "Codex" && event.type === "turn.completed") {
+    const usage = object(event.usage);
+    const cacheRead = number(usage.cached_input_tokens);
+    return {
+      // Codex reports total input tokens. Cache reads are a subset, unlike the
+      // other runners which report non-cached input separately.
+      input: Math.max(0, number(usage.input_tokens) - cacheRead),
+      output: number(usage.output_tokens),
+      reasoning: number(usage.reasoning_output_tokens ?? usage.reasoning_tokens),
+      cacheRead,
+      cacheWrite: number(usage.cache_write_input_tokens ?? usage.cache_creation_input_tokens),
+    };
+  }
+
+  return undefined;
+}
+
 export function tokenConsumption(usage) {
   return usage.input + usage.output + usage.reasoning;
 }
@@ -117,10 +169,18 @@ export function countToolInvocations(raw) {
   for (const line of raw.split("\n")) {
     try {
       const event = JSON.parse(line);
+      if (event.type === "tool_use") {
+        count++;
+        continue;
+      }
+      if (event.type === "item.started" && object(event.item).type === "command_execution") {
+        count++;
+        continue;
+      }
       if (event.type !== "assistant") continue;
       const content = object(event.message).content;
-      if (!Array.isArray(content)) continue;
+      if (Array.isArray(content))
-      count += content.filter((part) => object(part).type === "tool_use").length;
+        count += content.filter((part) => object(part).type === "tool_use").length;
     } catch {
       // Ignore malformed transcript lines.
     }

apps/benchmarks/scripts/cost.test.mjs

@@ -32,6 +32,57 @@ test("extracts usage from authoring harness transcripts", () => {
   });
 });
 
+test("extracts OpenCode step usage", () => {
+  const usage = extractRunUsage(
+    JSON.stringify({
+      type: "step_finish",
+      part: {
+        tokens: { input: 100, output: 20, reasoning: 5, cache: { read: 50, write: 10 } },
+      },
+    }),
+    "OpenCode",
+  );
+
+  assert.deepEqual(usage, { input: 100, output: 20, reasoning: 5, cacheRead: 50, cacheWrite: 10 });
+});
+
+test("extracts Claude Code message usage", () => {
+  const usage = extractRunUsage(
+    JSON.stringify({
+      type: "assistant",
+      message: {
+        usage: {
+          input_tokens: 100,
+          output_tokens: 20,
+          cache_read_input_tokens: 50,
+          cache_creation_input_tokens: 10,
+        },
+      },
+    }),
+    "Claude Code",
+  );
+
+  assert.deepEqual(usage, { input: 100, output: 20, reasoning: 0, cacheRead: 50, cacheWrite: 10 });
+});
+
+test("extracts Codex completion usage without double-counting cached input", () => {
+  const usage = extractRunUsage(
+    JSON.stringify({
+      type: "turn.completed",
+      usage: {
+        input_tokens: 150,
+        cached_input_tokens: 50,
+        cache_write_input_tokens: 10,
+        output_tokens: 20,
+        reasoning_output_tokens: 5,
+      },
+    }),
+    "Codex",
+  );
+
+  assert.deepEqual(usage, { input: 100, output: 20, reasoning: 5, cacheRead: 50, cacheWrite: 10 });
+});
+
 test("calculates token consumption without double-counting cache details", () => {
   assert.equal(
     tokenConsumption({
@@ -45,6 +96,15 @@ test("calculates token consumption without double-counting cache details", () =>
   );
 });
 
+test("counts native OpenCode and Codex tool invocations", () => {
+  const raw = [
+    JSON.stringify({ type: "tool_use", part: { tool: "bash" } }),
+    JSON.stringify({ type: "item.started", item: { type: "command_execution" } }),
+  ].join("\n");
+
+  assert.equal(countToolInvocations(raw), 2);
+});
+
 test("counts tool invocations in authoring harness transcripts", () => {
   const raw = [
     JSON.stringify({ type: "user", message: { role: "user", content: "Hi" } }),

apps/benchmarks/scripts/export-results.mjs

@@ -81,7 +81,7 @@ for (const benchmark of benchmarks) {
         caseId,
         status: status ?? "current",
         ...result,
-        ...meanRunMetrics(summaryPath, benchmark.model),
+        ...meanRunMetrics(summaryPath, benchmark.model, benchmark.harness),
       });
     }
   }
@@ -227,8 +227,8 @@ function latestValidResult(experimentId, caseId) {
   return undefined;
 }
 
-function meanRunMetrics(summaryPath, model) {
+function meanRunMetrics(summaryPath, model, harness) {
-  const performanceRuns = runMetrics(summaryPath);
+  const performanceRuns = runMetrics(summaryPath, harness);
   const performanceUsage = performanceRuns.flatMap((run) =>
     run.usage === null ? [] : [run.usage],
   );
@@ -248,14 +248,14 @@ function meanRunMetrics(summaryPath, model) {
   return result;
 }
 
-function runMetrics(summaryPath) {
+function runMetrics(summaryPath, harness) {
   return readdirSync(dirname(summaryPath), { withFileTypes: true })
     .filter((entry) => entry.isDirectory() && /^run-\d+$/u.test(entry.name))
     .flatMap((entry) => {
       const transcriptPath = join(dirname(summaryPath), entry.name, "transcript-raw.jsonl");
       if (!existsSync(transcriptPath)) return [];
       const raw = readFileSync(transcriptPath, "utf8");
-      return [{ usage: extractRunUsage(raw), toolInvocations: countToolInvocations(raw) }];
+      return [{ usage: extractRunUsage(raw, harness), toolInvocations: countToolInvocations(raw) }];
     });
 }
 

pnpm-lock.yaml

@@ -138,24 +138,9 @@ importers:
 
   apps/benchmarks:
     dependencies:
-      '@ai-sdk/harness':
-        specifier: 1.0.87
-        version: 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)
-      '@ai-sdk/harness-claude-code':
-        specifier: 1.0.90
-        version: 1.0.90(bufferutil@4.1.0)(zod@4.5.4)
-      '@ai-sdk/harness-opencode':
-        specifier: 1.0.88
-        version: 1.0.88(bufferutil@4.1.0)(zod@4.5.4)
-      '@ai-sdk/sandbox-vercel':
-        specifier: 1.0.87
-        version: 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)
       '@vercel/agent-eval':
-        specifier: 1.5.0
+        specifier: 2.2.1
-        version: 1.5.0(supports-color@10.2.2)
+        version: 2.2.1(supports-color@10.2.2)
-      '@vercel/sandbox':
-        specifier: 3.2.0
-        version: 3.2.0
       jiti:
         specifier: 2.7.0
         version: 2.7.0
@@ -1670,28 +1655,6 @@ packages:
     peerDependencies:
       zod: ^3.25.76 || ^4.1.8
 
-  '@ai-sdk/harness-claude-code@1.0.90':
-    resolution: {integrity: sha512-SL77y5l3596n6cxuLN4OB19iTkvwg6zJq2KxIV9MAPwAMYEEXHz7YGRXxrAd7GOixIVqcsXEw1HOLN1zcG1L3A==}
-    engines: {node: '>=22'}
-    peerDependencies:
-      zod: ^3.25.76 || ^4.1.8
-
-  '@ai-sdk/harness-opencode@1.0.88':
-    resolution: {integrity: sha512-bA22/rBaTbsO/o/x91P78RvLpvgwnCoW6Ie7ATsC7Z6BxGYZpFrS6BGWD1DQL2pN3jy61KWnDLDwy0egg7ypZg==}
-    engines: {node: '>=22'}
-    peerDependencies:
-      zod: ^3.25.76 || ^4.1.8
-
-  '@ai-sdk/harness@1.0.87':
-    resolution: {integrity: sha512-OY+M1B58lLv+fRHB0dh/FJ6lCzmg9HQmTe1z+XS1WTPF8PnTMEr2E/6vNTUoORN7iZMkTyVkszA4DOhod1eFQw==}
-    engines: {node: '>=22'}
-    peerDependencies:
-      ws: ^8.21.0
-      zod: ^3.25.76 || ^4.1.8
-    peerDependenciesMeta:
-      ws:
-        optional: true
-
   '@ai-sdk/mcp@2.0.29':
     resolution: {integrity: sha512-cK8mYpjKGp7/gmsTYtq/dkdmNlBeWQrwcfsah/W9hN6Vr1yNzunLnAZAqL5tMzumLDBueSz6y+vrb5z8DJ8TMg==}
     engines: {node: '>=22'}
@@ -1794,10 +1757,6 @@ packages:
     peerDependencies:
       react: ^18 || ~19.0.1 || ~19.1.2 || ^19.2.1
 
-  '@ai-sdk/sandbox-vercel@1.0.87':
-    resolution: {integrity: sha512-rW8mtwvH5lcoXWPn8hEdh2Kns/2qAQokS9xG/UeqZOgp79ifKJQp8dvWpUSzRNZKgPtqsXNmVFac6J/UnlmO+Q==}
-    engines: {node: '>=22'}
-
   '@ai-sdk/togetherai@1.0.49':
     resolution: {integrity: sha512-g4BpEatN7flh3GZ0CN9KvAUX6uLPmIqGSrKKFvAmC3HZdnF940zl+ChXs3atdbtpr6+cwirxM5RACbUzr0uYhA==}
     engines: {node: '>=18'}
@@ -7846,8 +7805,8 @@ packages:
     peerDependencies:
       chat: ^4.0.0
 
-  '@vercel/agent-eval@1.5.0':
+  '@vercel/agent-eval@2.2.1':
-    resolution: {integrity: sha512-evbgSkCv6CoQxgIr6dfjh6/3K54NxXoykZN5stXS97kTVm4NuU2y7su1WR71Y55tR4l0xgiUTsVp+BlLVh6zZw==}
+    resolution: {integrity: sha512-TVowf/Q60kw8anu4oAIhb1fc4y8k4jhwjCPwdfNoy9m9W1LHBHZYaIi81QZ+7w2S77hvBi4CAoOnohovjA2Mow==}
     engines: {node: '>=18.0.0'}
 
   '@vercel/agent-readability@0.6.0':
@@ -15764,18 +15723,6 @@ packages:
       utf-8-validate:
         optional: true
 
-  ws@8.21.0:
-    resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==}
-    engines: {node: '>=10.0.0'}
-    peerDependencies:
-      bufferutil: ^4.0.1
-      utf-8-validate: '>=5.0.2'
-    peerDependenciesMeta:
-      bufferutil:
-        optional: true
-      utf-8-validate:
-        optional: true
-
   ws@8.21.3:
     resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==}
     engines: {node: '>=10.0.0'}
@@ -16098,44 +16045,6 @@ snapshots:
       zod: 4.4.3
     optional: true
 
-  '@ai-sdk/harness-claude-code@1.0.90(bufferutil@4.1.0)(zod@4.5.4)':
-    dependencies:
-      '@ai-sdk/harness': 1.0.87(ws@8.21.3(bufferutil@4.1.0))(zod@4.5.4)
-      '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4)
-      ws: 8.21.3(bufferutil@4.1.0)
-      zod: 4.5.4
-    transitivePeerDependencies:
-      - bufferutil
-      - utf-8-validate
-
-  '@ai-sdk/harness-opencode@1.0.88(bufferutil@4.1.0)(zod@4.5.4)':
-    dependencies:
-      '@ai-sdk/harness': 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)
-      '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4)
-      ws: 8.21.0(bufferutil@4.1.0)
-      zod: 4.5.4
-    transitivePeerDependencies:
-      - bufferutil
-      - utf-8-validate
-
-  '@ai-sdk/harness@1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)':
-    dependencies:
-      '@ai-sdk/provider': 4.0.8
-      '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4)
-      ai: 7.0.79(zod@4.5.4)
-      zod: 4.5.4
-    optionalDependencies:
-      ws: 8.21.0(bufferutil@4.1.0)
-
-  '@ai-sdk/harness@1.0.87(ws@8.21.3(bufferutil@4.1.0))(zod@4.5.4)':
-    dependencies:
-      '@ai-sdk/provider': 4.0.8
-      '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4)
-      ai: 7.0.79(zod@4.5.4)
-      zod: 4.5.4
-    optionalDependencies:
-      ws: 8.21.3(bufferutil@4.1.0)
-
   '@ai-sdk/mcp@2.0.29(zod@4.5.4)':
     dependencies:
       '@ai-sdk/provider': 4.0.7
@@ -16270,17 +16179,6 @@ snapshots:
     transitivePeerDependencies:
       - zod
 
-  '@ai-sdk/sandbox-vercel@1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)':
-    dependencies:
-      '@ai-sdk/harness': 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)
-      '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4)
-      '@vercel/sandbox': 3.2.0
-    transitivePeerDependencies:
-      - bare-abort-controller
-      - react-native-b4a
-      - ws
-      - zod
-
   '@ai-sdk/togetherai@1.0.49(zod@4.4.3)':
     dependencies:
       '@ai-sdk/openai-compatible': 1.0.46(zod@4.4.3)
@@ -22322,7 +22220,7 @@ snapshots:
       - supports-color
       - zod
 
-  '@vercel/agent-eval@1.5.0(supports-color@10.2.2)':
+  '@vercel/agent-eval@2.2.1(supports-color@10.2.2)':
     dependencies:
       '@ai-sdk/anthropic': 1.2.12(zod@3.25.76)
       '@vercel/sandbox': 1.10.2
@@ -33064,10 +32962,6 @@ snapshots:
     optionalDependencies:
       bufferutil: 4.1.0
 
-  ws@8.21.0(bufferutil@4.1.0):
-    optionalDependencies:
-      bufferutil: 4.1.0
-
   ws@8.21.3(bufferutil@4.1.0):
     optionalDependencies:
       bufferutil: 4.1.0