From 32d298467691e3703e0314611c53aa8a44652334 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 31 Aug 2026 17:01:28 -0400 Subject: [PATCH] refactor(benchmarks): use native canary runners and upgrade to agent-eval v2 (#2785) Signed-off-by: Colton Padden --- .gitignore | 1 + apps/benchmarks/README.md | 90 +- .../evals/author-002-new-project/EVAL.ts | 18 +- .../evals/author-007-digest-schedule/CASE.ts | 2 +- apps/benchmarks/lib/authoring-case.ts | 11 +- apps/benchmarks/lib/benchmark-config.test.mjs | 4 + apps/benchmarks/lib/benchmark-config.ts | 10 +- apps/benchmarks/lib/dependency-sandbox.ts | 211 ----- apps/benchmarks/lib/experiment-files.mjs | 74 +- apps/benchmarks/lib/experiment-files.test.mjs | 37 +- apps/benchmarks/lib/experiment.test.mjs | 57 ++ apps/benchmarks/lib/experiment.ts | 48 +- apps/benchmarks/lib/harness-agent.test.mjs | 56 -- apps/benchmarks/lib/harness-agent.ts | 766 ------------------ .../lib/native-authoring-setup.test.mjs | 52 ++ apps/benchmarks/lib/native-authoring-setup.ts | 106 +++ apps/benchmarks/lib/source.mjs | 130 +-- apps/benchmarks/lib/source.test.mjs | 115 +-- apps/benchmarks/lib/timing.test.mjs | 20 - apps/benchmarks/lib/timing.ts | 55 -- apps/benchmarks/package.json | 7 +- apps/benchmarks/publish.mjs | 28 +- apps/benchmarks/run.mjs | 58 +- apps/benchmarks/scripts/cost.mjs | 82 +- apps/benchmarks/scripts/cost.test.mjs | 60 ++ apps/benchmarks/scripts/export-results.mjs | 10 +- pnpm-lock.yaml | 116 +-- 27 files changed, 597 insertions(+), 1627 deletions(-) delete mode 100644 apps/benchmarks/lib/dependency-sandbox.ts create mode 100644 apps/benchmarks/lib/experiment.test.mjs delete mode 100644 apps/benchmarks/lib/harness-agent.test.mjs delete mode 100644 apps/benchmarks/lib/harness-agent.ts create mode 100644 apps/benchmarks/lib/native-authoring-setup.test.mjs create mode 100644 apps/benchmarks/lib/native-authoring-setup.ts delete mode 100644 apps/benchmarks/lib/timing.test.mjs delete mode 100644 apps/benchmarks/lib/timing.ts diff --git a/.gitignore b/.gitignore index 305d74c08..0670f7507 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ apps/benchmarks/experiments/ apps/benchmarks/results/ apps/benchmarks/evals/*/PROMPT.md apps/benchmarks/evals/*/package.json +apps/benchmarks/evals/*/.eve-authoring-bootstrap.json .extension-contracts-cache/ packages/eve/.workflow-vitest/ packages/eve/.generated/ diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md index cbd12a8cc..2c2d6c89f 100644 --- a/apps/benchmarks/README.md +++ b/apps/benchmarks/README.md @@ -6,84 +6,46 @@ run in CI or as part of `pnpm test`. ## Run -The default subject is the current working tree, including uncommitted and untracked files that -Git does not ignore: +The default subject is the current `main` canary. The runner resolves that moving alias once to its +immutable commit URL, then every model, treatment, and repetition uses that same artifact: ```sh -pnpm benchmark author-000-imessage +pnpm benchmark author-001-weather-tool pnpm benchmark -pnpm benchmark author-000-imessage --runs 3 -pnpm benchmark author-000-imessage --model kimi-k3 -pnpm benchmark author-000-imessage --treatment baseline -pnpm benchmark author-000-imessage --dry -pnpm benchmark author-000-imessage --verbose -pnpm benchmark author-000-imessage --keep-failures +pnpm benchmark author-001-weather-tool --runs 3 +pnpm benchmark author-001-weather-tool --model kimi-k3 +pnpm benchmark author-001-weather-tool --treatment baseline +pnpm benchmark author-001-weather-tool --dry +pnpm benchmark author-001-weather-tool --verbose +pnpm benchmark author-001-weather-tool --keep-failures +pnpm benchmark author-001-weather-tool --canary main ``` `--keep-failures` keeps a run the runner judged an infrastructure failure — a stalled turn, a sandbox error — as the final result instead of discarding it. Use it while iterating on the harness, when the failure itself is what you want to read. -Set `EVE_BENCHMARK_TRACE_PARTS=1` to print every stream part the harness receives with the gap -since the previous one. The harness decides a turn is over by reading those parts, so this is what -to reach for when a turn ends too early or hangs past its closing message. +`--canary ` selects another published canary ref. The runner rejects refs without a package +artifact before it starts an eval. Local working trees, unpublished commits, and revision comparisons +are not supported by the native runner. -Use `--base` to compare a local Git revision with the working tree: - -```sh -pnpm benchmark author-000-imessage --base origin/main --runs 3 -``` - -Pass `--head` to compare two local revisions instead: - -```sh -pnpm benchmark author-000-imessage \ - --base origin/main \ - --head feature-branch \ - --runs 3 -``` - -The runner archives each subject locally and uploads it to the sandbox. Revisions and local-only -commits do not need to be pushed. It maintains two persistent snapshot layers: a dependency -snapshot keyed by package-manager inputs, and a subject snapshot keyed by the source tree, -starting point, setup IDs, and bootstrap version. Source-only changes reuse the dependency snapshot -but create a new subject snapshot. For one eval and one run, `--verbose` streams setup phases, -assistant text, tool calls, grading, and build progress. +The runner uses agent-eval's native Gateway harnesses: OpenCode for other providers, Claude Code +for Anthropic models, and Codex for OpenAI models. Each attempt starts an isolated Vercel Sandbox, +then scaffolds the selected immutable canary with `npx` before the coding agent starts. Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts. -Results are written under `apps/benchmarks/results/`. Each run includes the transcript, -grader output, summary, copied project files, and `project/benchmark/timings.json`. The timing -artifact records the snapshot-cache outcome, source installation and build phases, workspace setup, -each user turn with token and tool-call counts, and grading and validation durations. Use it to -separate sandbox setup time from agent time when comparing runs. Print a compact local report with: - -```sh -pnpm benchmark:timings results/current///run-1 -``` - -Pass `--json` to print the original timing artifact. Vercel Sandbox and AI Gateway credentials are -required. - -To read a whole results directory at once — pass/fail, agent time against setup time, turn and tool -counts, tokens, and any stalled turns: - -```sh -node scripts/analyze.mjs results/current/ -``` - -Pass `--docs` to also list, per run, which docs page the agent entered at and every page it went on -to read. That is the fastest way to see whether a documentation change moved agents toward the page -that answers the task or sent them spidering. +Results are written under `apps/benchmarks/results/`. Each run includes the native transcript, +grader output, summary, copied project files, and validation output. Vercel Sandbox and AI Gateway +credentials are required. ## Publish canonical results -Canonical publication compares the `baseline` and `guided` treatments with the same eve revision, -model, harness, cases, and graders. The matrix holds the harness constant at OpenCode and varies -only the model, so rows are comparable. A model ID is selected independently from the coding-agent -harness; adding Claude Code, Codex, or Gemini CLI belongs to a separate harness comparison. Publication -requires a clean working tree and defaults to `origin/main`: +Canonical publication compares the `baseline` and `guided` treatments with the same immutable eve +canary, model, harness, cases, and graders. The configured harness reflects the provider: OpenCode +for other providers, Claude Code for Anthropic, and Codex for OpenAI. Publication requires a clean +working tree and defaults to `origin/main`: ```sh pnpm benchmark:publish --dry @@ -128,6 +90,6 @@ export default defineAuthoringCase({ }); ``` -Use `simpleProject` for the selected subject's `eve init` output and `emptyProject` for an empty -directory with the subject CLI installed. Put reusable setup under `lib/setups/`. Prefer source -and event assertions over an LLM judge. +Use `simpleProject` for the selected canary's `eve init` output and `emptyProject` for a project +the coding agent creates. Put reusable setup under `lib/setups/`. Native runs support one-turn +cases; the iMessage case remains local-only. Prefer source assertions over an LLM judge. diff --git a/apps/benchmarks/evals/author-002-new-project/EVAL.ts b/apps/benchmarks/evals/author-002-new-project/EVAL.ts index 33d0d0168..86ec7e5fb 100644 --- a/apps/benchmarks/evals/author-002-new-project/EVAL.ts +++ b/apps/benchmarks/evals/author-002-new-project/EVAL.ts @@ -2,13 +2,15 @@ import { existsSync, readFileSync } from "node:fs"; import { expect, test } from "vitest"; -import { subjectDefaultAgentModel } from "./grader.js"; +const projectRoot = "wayfinder"; + +const defaultAgentModel = "openai/gpt-5.6-luna-fast"; test("creates a complete eve project in place", () => { - expect(existsSync("agent/channels/eve.ts")).toBe(true); - expect(existsSync("agent/instructions.md")).toBe(true); + expect(existsSync(`${projectRoot}/agent/channels/eve.ts`)).toBe(true); + expect(existsSync(`${projectRoot}/agent/instructions.md`)).toBe(true); - const packageJson = JSON.parse(readFileSync("package.json", "utf8")) as { + const packageJson = JSON.parse(readFileSync(`${projectRoot}/package.json`, "utf8")) as { dependencies?: Record; scripts?: Record; }; @@ -17,16 +19,16 @@ test("creates a complete eve project in place", () => { }); test("authors the requested identity without pinning a different model", () => { - const instructions = readFileSync("agent/instructions.md", "utf8"); + const instructions = readFileSync(`${projectRoot}/agent/instructions.md`, "utf8"); expect(instructions).toMatch(/Wayfinder/i); expect(instructions).toMatch(/travel/i); // `agent/agent.ts` is optional, and omitting it selects the same default the // scaffold pins explicitly. Both shapes satisfy "use the default model"; a // different model id does not. - if (existsSync("agent/agent.ts")) { - expect(readFileSync("agent/agent.ts", "utf8")).toContain( - `model: "${subjectDefaultAgentModel()}"`, + if (existsSync(`${projectRoot}/agent/agent.ts`)) { + expect(readFileSync(`${projectRoot}/agent/agent.ts`, "utf8")).toContain( + `model: "${defaultAgentModel}"`, ); } }); diff --git a/apps/benchmarks/evals/author-007-digest-schedule/CASE.ts b/apps/benchmarks/evals/author-007-digest-schedule/CASE.ts index 85a00f83d..aa13acbe0 100644 --- a/apps/benchmarks/evals/author-007-digest-schedule/CASE.ts +++ b/apps/benchmarks/evals/author-007-digest-schedule/CASE.ts @@ -4,7 +4,7 @@ export default defineAuthoringCase({ startingPoint: simpleProject, async interact({ send }) { await send( - "Put the agent on a cron: every weekday at 9am UTC, run it on a short prompt asking for a status digest. Nothing needs to be delivered anywhere — the run log is fine.", + "In this eve project, add an agent schedule under `agent/schedules/`: every weekday at 9am UTC, run the agent on a short prompt asking for a status digest. Nothing needs to be delivered anywhere — the eve run log is fine.", ); }, }); diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.ts index d708b24f3..44e29bbc3 100644 --- a/apps/benchmarks/lib/authoring-case.ts +++ b/apps/benchmarks/lib/authoring-case.ts @@ -1,13 +1,6 @@ -import type { HarnessV1NetworkSandboxSession } from "@ai-sdk/harness"; -import type { HarnessAgentSession } from "@ai-sdk/harness/agent"; - -import { AUTHORING_EVAL_DIRECTORY } from "./paths.js"; -import type { AuthoringTranscriptEntry } from "./protocol.js"; - export interface AuthoringSetupContext { - readonly sandbox: HarnessV1NetworkSandboxSession; readonly workspace: string; - readonly artifactsRoot: typeof AUTHORING_EVAL_DIRECTORY; + readonly artifactsRoot: string; run(command: string, workingDirectory?: string): Promise; write(path: string, content: string): Promise; } @@ -34,8 +27,6 @@ export interface AuthoringTurn { } export interface AuthoringInteractionContext { - readonly session: HarnessAgentSession; - readonly transcript: ReadonlyArray; send(prompt: string): Promise; } diff --git a/apps/benchmarks/lib/benchmark-config.test.mjs b/apps/benchmarks/lib/benchmark-config.test.mjs index dc550aafa..7d9ff1b0a 100644 --- a/apps/benchmarks/lib/benchmark-config.test.mjs +++ b/apps/benchmarks/lib/benchmark-config.test.mjs @@ -4,6 +4,7 @@ import { test } from "node:test"; import { findBenchmarkModel, findPublishedBenchmarkModel, + harnessId, publishedBenchmark, publishedBenchmarkModels, } from "./benchmark-config.ts"; @@ -39,6 +40,9 @@ test("publishes only compatibility-validated models", () => { assert.equal(findPublishedBenchmarkModel("claude-sonnet-5").harness, "Claude Code"); assert.equal(findPublishedBenchmarkModel("claude-opus-5").harness, "Claude Code"); assert.equal(findPublishedBenchmarkModel("kimi-k3").harness, "OpenCode"); + assert.equal(findPublishedBenchmarkModel("gpt-5-6-sol").harness, "Codex"); + assert.equal(findPublishedBenchmarkModel("gpt-5-6-terra").harness, "Codex"); + assert.equal(harnessId("Codex"), "codex"); }); test("allows candidate probes and rejects unknown models", () => { diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts index eb2185b75..54c083f06 100644 --- a/apps/benchmarks/lib/benchmark-config.ts +++ b/apps/benchmarks/lib/benchmark-config.ts @@ -8,7 +8,7 @@ export interface AuthoringBenchmarkModel { readonly id: string; readonly model: string; readonly displayName: string; - readonly harness: "OpenCode" | "Claude Code"; + readonly harness: "OpenCode" | "Claude Code" | "Codex"; readonly support: AuthoringBenchmarkSupport; } @@ -45,14 +45,14 @@ export const benchmarkModels = [ id: "gpt-5-6-sol", model: "openai/gpt-5.6-sol", displayName: "GPT-5.6 Sol", - harness: "OpenCode", + harness: "Codex", support: "supported", }, { id: "gpt-5-6-terra", model: "openai/gpt-5.6-terra", displayName: "GPT-5.6 Terra", - harness: "OpenCode", + harness: "Codex", support: "supported", }, { @@ -110,7 +110,9 @@ export function publishedExperimentId( } export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string { - return harness === "Claude Code" ? "claude-code" : "opencode"; + if (harness === "Claude Code") return "claude-code"; + if (harness === "Codex") return "codex"; + return "opencode"; } export function parseAuthoringTreatment(value: string): AuthoringTreatment { diff --git a/apps/benchmarks/lib/dependency-sandbox.ts b/apps/benchmarks/lib/dependency-sandbox.ts deleted file mode 100644 index cb16cf905..000000000 --- a/apps/benchmarks/lib/dependency-sandbox.ts +++ /dev/null @@ -1,211 +0,0 @@ -import type { HarnessV1SandboxProvider } from "@ai-sdk/harness"; -import { createVercelSandbox } from "@ai-sdk/sandbox-vercel"; -import { Sandbox } from "@vercel/sandbox"; - -import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js"; -import type { BenchmarkTimings } from "./timing.js"; - -const dependencySnapshots = new Map>(); -const subjectSnapshots = new Map>(); - -export function createDependencyCachedSandbox(options: { - readonly archive: Uint8Array; - readonly dependencyArchive: Uint8Array; - readonly dependencyDigest: string; - readonly ports: ReadonlyArray; - readonly env: Readonly>; - readonly log: (message: string) => void; - readonly timings: BenchmarkTimings; -}): HarnessV1SandboxProvider { - const sessionProvider = (snapshotId: string) => - createVercelSandbox({ - source: { type: "snapshot", snapshotId }, - ports: [...options.ports], - timeout: 15 * 60_000, - env: { ...options.env }, - networkPolicy: "allow-all", - }); - - return { - specificationVersion: "harness-sandbox-v1", - providerId: "eve-benchmark-vercel", - async createSession(request = {}) { - options.log("[setup] preparing dependency cache"); - const dependencies = await dependencySnapshot( - options.dependencyArchive, - options.dependencyDigest, - options.log, - options.timings, - ); - if (request.identity === undefined || request.onFirstCreate === undefined) { - return sessionProvider(dependencies).createSession(request); - } - const subject = await subjectSnapshot( - dependencies, - options.archive, - request.identity, - options.env, - request.onFirstCreate, - request.abortSignal, - options.timings, - ); - return sessionProvider(subject).createSession({ - sessionId: request.sessionId, - abortSignal: request.abortSignal, - }); - }, - async resumeSession(request) { - const dependencies = await dependencySnapshot( - options.dependencyArchive, - options.dependencyDigest, - options.log, - options.timings, - ); - const provider = sessionProvider(dependencies); - if (provider.resumeSession === undefined) { - throw new Error("Vercel Sandbox does not support session resume."); - } - return provider.resumeSession(request); - }, - }; -} - -function dependencySnapshot( - archive: Uint8Array, - digest: string, - log: (message: string) => void, - timings: BenchmarkTimings, -): Promise { - const name = `eve-benchmark-dependencies-v5-${digest.slice(0, 24)}`; - let snapshot = dependencySnapshots.get(name); - if (snapshot !== undefined) { - timings.record("dependency-snapshot.memory-cache", 0, "success", { name }); - return snapshot; - } - snapshot = createDependencySnapshot(name, archive, log, timings); - dependencySnapshots.set(name, snapshot); - return snapshot; -} - -async function createDependencySnapshot( - name: string, - archive: Uint8Array, - log: (message: string) => void, - timings: BenchmarkTimings, -): Promise { - let created = false; - const sandbox = await timings.measure("dependency-snapshot.get-or-create", () => - Sandbox.getOrCreate({ - name, - runtime: "node24", - timeout: 15 * 60_000, - persistent: true, - snapshotExpiration: 0, - networkPolicy: "allow-all", - async onCreate(current) { - created = true; - log("[setup] fetching workspace dependencies"); - await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]); - const command = await current.runCommand("bash", [ - "-lc", - `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global pnpm@11.15.0 vitest@4.1.10 && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`, - ]); - if (command.exitCode !== 0) { - throw new Error( - `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`, - ); - } - }, - }), - ); - if (!created && sandbox.currentSnapshotId !== undefined) { - timings.record("dependency-snapshot.reused", 0, "success", { name }); - return sandbox.currentSnapshotId; - } - return timings.measure("dependency-snapshot.publish", () => - stopWithSnapshot(sandbox, "Dependency"), - ); -} - -// Publish bootstrap mutations explicitly; a layered Vercel template can expose -// its inherited snapshot ID before those mutations receive a new snapshot. -function subjectSnapshot( - dependencySnapshotId: string, - archive: Uint8Array, - identity: string, - env: Readonly>, - bootstrap: NonNullable< - NonNullable[0]>["onFirstCreate"] - >, - abortSignal: AbortSignal | undefined, - timings: BenchmarkTimings, -): Promise { - const name = `eve-benchmark-subject-${identity}`; - let snapshot = subjectSnapshots.get(name); - if (snapshot !== undefined) { - timings.record("subject-snapshot.memory-cache", 0, "success", { name }); - return snapshot; - } - snapshot = createSubjectSnapshot( - name, - dependencySnapshotId, - archive, - env, - bootstrap, - abortSignal, - timings, - ); - subjectSnapshots.set(name, snapshot); - return snapshot; -} - -async function createSubjectSnapshot( - name: string, - dependencySnapshotId: string, - archive: Uint8Array, - env: Readonly>, - bootstrap: NonNullable< - NonNullable[0]>["onFirstCreate"] - >, - abortSignal: AbortSignal | undefined, - timings: BenchmarkTimings, -): Promise { - let created = false; - const sandbox = await timings.measure("subject-snapshot.get-or-create", () => - Sandbox.getOrCreate({ - name, - source: { type: "snapshot", snapshotId: dependencySnapshotId }, - timeout: 15 * 60_000, - env: { ...env }, - persistent: true, - snapshotExpiration: 0, - networkPolicy: "allow-all", - signal: abortSignal, - async onCreate(current) { - created = true; - await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]); - const provider = createVercelSandbox({ sandbox: current }); - const session = await provider.createSession({ abortSignal }); - await bootstrap(session.restricted(), { abortSignal }); - }, - }), - ); - if (!created && sandbox.currentSnapshotId !== undefined) { - timings.record("subject-snapshot.reused", 0, "success", { name }); - return sandbox.currentSnapshotId; - } - return timings.measure("subject-snapshot.publish", () => - stopWithSnapshot(sandbox, "Subject", abortSignal), - ); -} - -async function stopWithSnapshot( - sandbox: Sandbox, - label: string, - signal?: AbortSignal, -): Promise { - const stopped = await sandbox.stop(signal === undefined ? undefined : { signal }); - const snapshotId = stopped.snapshot?.id ?? sandbox.currentSnapshotId; - if (snapshotId === undefined) throw new Error(`${label} snapshot was not published.`); - return snapshotId; -} diff --git a/apps/benchmarks/lib/experiment-files.mjs b/apps/benchmarks/lib/experiment-files.mjs index 27c9d9a65..c84f54936 100644 --- a/apps/benchmarks/lib/experiment-files.mjs +++ b/apps/benchmarks/lib/experiment-files.mjs @@ -1,6 +1,8 @@ import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { createJiti } from "jiti"; + export function fixtureNames(evalsRoot) { return readdirSync(evalsRoot, { withFileTypes: true }) .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts"))) @@ -8,10 +10,25 @@ export function fixtureNames(evalsRoot) { .sort(); } -export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) { +export async function prepareFixtures(evalsRoot, subject, names = fixtureNames(evalsRoot)) { for (const name of names) { const fixtureRoot = join(evalsRoot, name); - writeFileSync(join(fixtureRoot, "PROMPT.md"), ""); + const authoringCase = await loadCase(fixtureRoot); + writeFileSync( + join(fixtureRoot, "PROMPT.md"), + `${await promptForCase(authoringCase, fixtureRoot)}\n`, + ); + writeFileSync( + join(fixtureRoot, ".eve-authoring-bootstrap.json"), + `${JSON.stringify({ + startingPoint: authoringCase.startingPoint.workspace, + projectDirectory: authoringCase.projectDirectory, + revision: subject.revision, + setupIds: [authoringCase.startingPoint.setup, authoringCase.setup] + .filter(Boolean) + .map((setup) => setup.id), + })}\n`, + ); writeFileSync( join(fixtureRoot, "package.json"), `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`, @@ -19,29 +36,54 @@ export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) { } } +async function loadCase(fixtureRoot) { + const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false }); + let authoringCase = await jiti.import(`${fixtureRoot}/CASE.ts`); + while (!isAuthoringCase(authoringCase) && hasDefaultExport(authoringCase)) { + authoringCase = authoringCase.default; + } + if (!isAuthoringCase(authoringCase)) { + throw new Error(`${fixtureRoot}/CASE.ts must export an authoring case as default.`); + } + return authoringCase; +} + +async function promptForCase(authoringCase, fixtureRoot) { + const prompts = []; + await authoringCase.interact({ + send: async (prompt) => { + prompts.push(prompt); + return { text: "", toolCalls: [] }; + }, + }); + if (prompts.length !== 1) { + throw new Error( + `${fixtureRoot}/CASE.ts is multi-turn and cannot run on a native agent-eval runner.`, + ); + } + return prompts[0]; +} + +function isAuthoringCase(value) { + return typeof value === "object" && value !== null && typeof value.interact === "function"; +} + +function hasDefaultExport(value) { + return typeof value === "object" && value !== null && "default" in value; +} + export function resetExperiments(experimentsRoot) { rmSync(experimentsRoot, { recursive: true, force: true }); mkdirSync(experimentsRoot, { recursive: true }); } -export function writeSubjectArchives(experimentsRoot, subject, name) { - const archiveName = `${name}.source.tar.gz`; - const dependencyArchiveName = `${name}.dependencies.tar.gz`; - writeFileSync(join(experimentsRoot, archiveName), subject.archive); - writeFileSync(join(experimentsRoot, dependencyArchiveName), subject.dependencyArchive); - return { archiveName, dependencyArchiveName }; -} - export function writeExperiment(experimentsRoot, name, options) { writeFileSync( join(experimentsRoot, `${name}.ts`), - `import { readFileSync } from "node:fs";\n` + - `import { authoringExperiment } from "../lib/experiment.js";\n\n` + + `import { authoringExperiment } from "../lib/experiment.js";\n\n` + `export default authoringExperiment({\n` + - ` archive: readFileSync(new URL(${JSON.stringify(`./${options.archiveName}`)}, import.meta.url)),\n` + - ` dependencyArchive: readFileSync(new URL(${JSON.stringify(`./${options.dependencyArchiveName}`)}, import.meta.url)),\n` + - ` digest: ${JSON.stringify(options.digest)},\n` + - ` dependencyDigest: ${JSON.stringify(options.dependencyDigest)},\n` + + ` revision: ${JSON.stringify(options.revision)},\n` + + ` packageSpec: ${JSON.stringify(options.packageSpec)},\n` + ` runs: ${options.runs},\n` + (options.evals === undefined ? "" : ` evals: ${JSON.stringify(options.evals)},\n`) + ` benchmark: ${JSON.stringify(options.benchmark)},\n` + diff --git a/apps/benchmarks/lib/experiment-files.test.mjs b/apps/benchmarks/lib/experiment-files.test.mjs index b122d3733..7942ab104 100644 --- a/apps/benchmarks/lib/experiment-files.test.mjs +++ b/apps/benchmarks/lib/experiment-files.test.mjs @@ -9,14 +9,11 @@ import { prepareFixtures, resetExperiments, writeExperiment, - writeSubjectArchives, } from "./experiment-files.mjs"; const subject = { - archive: Buffer.from("source"), - dependencyArchive: Buffer.from("dependencies"), - digest: "source-digest", - dependencyDigest: "dependency-digest", + revision: "1234567890abcdef1234567890abcdef12345678", + packageSpec: "https://pkg.eve.dev/1234567890abcdef1234567890abcdef12345678/eve.tgz", }; const benchmark = { id: "test", @@ -26,7 +23,7 @@ const benchmark = { support: "supported", }; -test("materializes fixtures and complete experiment inputs", () => { +test("materializes fixtures and complete experiment inputs", async () => { const root = mkdtempSync(join(tmpdir(), "eve-benchmark-experiments-")); const evals = join(root, "evals"); const experiments = join(root, "experiments"); @@ -36,8 +33,12 @@ test("materializes fixtures and complete experiment inputs", () => { writeFileSync(join(evals, "not-a-case"), "ignored"); assert.deepEqual(fixtureNames(evals), ["author-001-first", "author-002-second"]); - prepareFixtures(evals); - assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), ""); + await prepareFixtures(evals, subject); + assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "Build it.\n"); + assert.deepEqual( + JSON.parse(readFileSync(join(evals, "author-001-first", ".eve-authoring-bootstrap.json"))), + { startingPoint: "scaffolded", revision: subject.revision, setupIds: [] }, + ); assert.deepEqual(JSON.parse(readFileSync(join(evals, "author-001-first", "package.json"))), { name: "eve-authoring-author-001-first", private: true, @@ -45,25 +46,18 @@ test("materializes fixtures and complete experiment inputs", () => { }); resetExperiments(experiments); - const archives = writeSubjectArchives(experiments, subject, "published-deadbeef"); writeExperiment(experiments, "test-opencode--guided", { - ...archives, - digest: subject.digest, - dependencyDigest: subject.dependencyDigest, + revision: subject.revision, + packageSpec: subject.packageSpec, runs: 3, evals: ["author-001-first"], benchmark, treatment: "guided", }); - assert.equal(readFileSync(join(experiments, archives.archiveName), "utf8"), "source"); - assert.equal( - readFileSync(join(experiments, archives.dependencyArchiveName), "utf8"), - "dependencies", - ); const experiment = readFileSync(join(experiments, "test-opencode--guided.ts"), "utf8"); - assert.match(experiment, /dependencyArchive: readFileSync/u); - assert.match(experiment, /published-deadbeef\.dependencies\.tar\.gz/u); + assert.match(experiment, /revision: "1234567890abcdef1234567890abcdef12345678"/u); + assert.match(experiment, /pkg\.eve\.dev\/1234567890abcdef1234567890abcdef12345678\/eve\.tgz/u); } finally { rmSync(root, { recursive: true, force: true }); } @@ -72,6 +66,9 @@ test("materializes fixtures and complete experiment inputs", () => { function writeCase(evals, name) { const root = join(evals, name); mkdirSync(root, { recursive: true }); - writeFileSync(join(root, "CASE.ts"), "export default {};\n"); + writeFileSync( + join(root, "CASE.ts"), + 'export default { startingPoint: { workspace: "scaffolded" }, async interact({ send }) { await send("Build it."); } };\n', + ); assert.ok(existsSync(root)); } diff --git a/apps/benchmarks/lib/experiment.test.mjs b/apps/benchmarks/lib/experiment.test.mjs new file mode 100644 index 000000000..bccabcc5c --- /dev/null +++ b/apps/benchmarks/lib/experiment.test.mjs @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createJiti } from "jiti"; + +const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false }); +const { authoringExperiment } = await jiti.import( + new URL("./experiment.ts", import.meta.url).pathname, +); + +const common = { + revision: "1234567890abcdef1234567890abcdef12345678", + packageSpec: "https://pkg.eve.dev/1234567890abcdef1234567890abcdef12345678/eve.tgz", + treatment: "guided", +}; + +test("uses native Gateway agents and canonical cases", () => { + const config = authoringExperiment({ + ...common, + benchmark: { + id: "gpt-5-6-sol", + model: "openai/gpt-5.6-sol", + displayName: "GPT-5.6 Sol", + harness: "Codex", + support: "supported", + }, + }); + + assert.equal(config.agent, "vercel-ai-gateway/codex"); + assert.equal(config.model, "openai/gpt-5.6-sol"); + assert.deepEqual(config.evals, [ + "author-001-weather-tool", + "author-002-new-project", + "author-003-openapi-connection", + "author-004-packaged-skill", + "author-005-conditional-approval", + "author-006-custom-channel", + "author-007-digest-schedule", + ]); + assert.equal(typeof config.setup, "function"); +}); + +test("strips the Gateway provider prefix for native Claude Code", () => { + const config = authoringExperiment({ + ...common, + benchmark: { + id: "claude-sonnet-5", + model: "anthropic/claude-sonnet-5", + displayName: "Claude Sonnet 5", + harness: "Claude Code", + support: "supported", + }, + }); + + assert.equal(config.agent, "vercel-ai-gateway/claude-code"); + assert.equal(config.model, "claude-sonnet-5"); +}); diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.ts index 71e6f8bdb..00d021e7f 100644 --- a/apps/benchmarks/lib/experiment.ts +++ b/apps/benchmarks/lib/experiment.ts @@ -1,33 +1,28 @@ import type { ExperimentConfig } from "@vercel/agent-eval"; -import { registerAgent } from "@vercel/agent-eval"; -import type { AuthoringBenchmarkModel, AuthoringTreatment } from "./benchmark-config.js"; -import { createAuthoringAgent } from "./harness-agent.js"; +import { + harnessId, + type AuthoringBenchmarkModel, + type AuthoringTreatment, + publishedBenchmark, +} from "./benchmark-config.js"; +import { createNativeAuthoringSetup } from "./native-authoring-setup.js"; export function authoringExperiment(options: { - readonly archive: Uint8Array; - readonly dependencyArchive: Uint8Array; - readonly digest: string; - readonly dependencyDigest: string; + readonly revision: string; + readonly packageSpec: string; readonly runs?: number; readonly evals?: readonly string[]; readonly benchmark: AuthoringBenchmarkModel; readonly treatment: AuthoringTreatment; readonly verbose?: boolean; }): ExperimentConfig { - const agent = createAuthoringAgent({ - harness: options.benchmark.harness, - model: options.benchmark.model, - archive: options.archive, - dependencyArchive: options.dependencyArchive, - digest: options.digest, - dependencyDigest: options.dependencyDigest, - }); - registerAgent(agent); return { - agent: agent.name, - model: options.benchmark.model, - evals: process.env.EVE_BENCHMARK_EVAL ?? (options.evals ? [...options.evals] : "*"), + agent: `vercel-ai-gateway/${harnessId(options.benchmark.harness)}`, + model: nativeModel(options.benchmark), + evals: + process.env.EVE_BENCHMARK_EVAL ?? + (options.evals ? [...options.evals] : [...publishedBenchmark.caseIds]), scripts: ["typecheck", "build"], runs: options.runs ?? 1, earlyExit: false, @@ -36,9 +31,16 @@ export function authoringExperiment(options: { timeout: Number(process.env.EVE_BENCHMARK_TIMEOUT ?? 900), sandbox: "vercel", copyFiles: "changed", - agentOptions: { - agentsMd: options.treatment === "guided", - verbose: options.verbose ?? false, - }, + setup: createNativeAuthoringSetup({ + packageSpec: options.packageSpec, + revision: options.revision, + treatment: options.treatment, + }), + agentOptions: { verbose: options.verbose ?? false }, }; } + +function nativeModel(benchmark: AuthoringBenchmarkModel): string { + if (benchmark.harness !== "Claude Code") return benchmark.model; + return benchmark.model.replace(/^anthropic\//u, ""); +} diff --git a/apps/benchmarks/lib/harness-agent.test.mjs b/apps/benchmarks/lib/harness-agent.test.mjs deleted file mode 100644 index c49a19937..000000000 --- a/apps/benchmarks/lib/harness-agent.test.mjs +++ /dev/null @@ -1,56 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; - -import { createJiti } from "jiti"; - -const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false }); -const { addUsage } = await jiti.import(new URL("./harness-agent.ts", import.meta.url).pathname); - -test("normalizes nested Claude Code token usage", () => { - const usage = { - inputTokens: 0, - outputTokens: 0, - reasoningTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - }; - - addUsage(usage, { - inputTokens: { total: 32_177, noCache: 1, cacheRead: 32_176, cacheWrite: 10_944 }, - outputTokens: { total: 327, text: 327 }, - }); - - assert.deepEqual(usage, { - inputTokens: 1, - outputTokens: 327, - reasoningTokens: 0, - cachedInputTokens: 32_176, - cacheWriteTokens: 10_944, - }); -}); - -test("retains flat OpenCode token usage", () => { - const usage = { - inputTokens: 0, - outputTokens: 0, - reasoningTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - }; - - addUsage(usage, { - inputTokens: 100, - outputTokens: 20, - reasoningTokens: 5, - cachedInputTokens: 50, - cacheWriteTokens: 10, - }); - - assert.deepEqual(usage, { - inputTokens: 100, - outputTokens: 20, - reasoningTokens: 5, - cachedInputTokens: 50, - cacheWriteTokens: 10, - }); -}); diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts deleted file mode 100644 index eacbb47f0..000000000 --- a/apps/benchmarks/lib/harness-agent.ts +++ /dev/null @@ -1,766 +0,0 @@ -import { readFileSync } from "node:fs"; -import type { HarnessV1, HarnessV1NetworkSandboxSession } from "@ai-sdk/harness"; -import type { HarnessAgentSession } from "@ai-sdk/harness/agent"; -import { HarnessAgent } from "@ai-sdk/harness/agent"; -import { createClaudeCode } from "@ai-sdk/harness-claude-code"; -import { createOpenCode } from "@ai-sdk/harness-opencode"; -import type { Agent, AgentRunResult } from "@vercel/agent-eval"; -import { z } from "zod"; - -import type { - AuthoringCase, - AuthoringSetup, - AuthoringSetupContext, - AuthoringTurn, -} from "./authoring-case.js"; -import type { AuthoringBenchmarkModel } from "./benchmark-config.js"; -import { createDependencyCachedSandbox } from "./dependency-sandbox.js"; -import { loadAuthoringCase } from "./load-authoring-case.js"; -import { - AGENT_EVAL_DIRECTORY, - AUTHORING_EVAL_DIRECTORY, - AUTHORING_EVAL_DIRECTORY_ENV, - EVE_PACKAGE_PATH, - SOURCE_ARCHIVE_PATH, - SOURCE_ROOT, - WORKSPACE, -} from "./paths.js"; -import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.js"; -import { BenchmarkTimings } from "./timing.js"; - -const HARNESS_BRIDGE_PORT = 4172; -const POST_RUN_GRADER_DIRECTORY = ".eve-grader"; -const BOOTSTRAP_VERSION = "v9"; -// A turn that stops producing output should end the turn, not the eval: the -// remaining turns still run and the graders still see what the agent did. -const TURN_TIMEOUT_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_TIMEOUT ?? 300); -// A turn that has produced no chunk for this long is waiting on the runtime, not -// working. The bound is one slow tool call: the bridge reports a call and its -// result together once the call returns, so an install that takes minutes looks -// from here like silence rather than work in flight. -const TURN_STALL_SECONDS = Number(process.env.EVE_BENCHMARK_TURN_STALL ?? 120); -// The runtime frequently never closes a turn whose last act was the model -// talking, so silence after a closing message has to end the turn on its own. -// Long enough that a pause before the next tool call is not mistaken for one. -const CLOSING_IDLE_MILLIS = 45_000; -// How long to let an aborted turn settle before starting the next one. -const TURN_SETTLE_MILLIS = 15_000; - -// The coding agent's own question tool is the one place it can ask the user -// mid-turn. Left unanswered it waits for a human and the runtime stops driving -// the turn, so the harness answers it immediately and points the agent at the -// channel this benchmark can answer on: its reply, which the case's next `send` -// responds to. -type AuthoringHarnessAgent = HarnessAgent; - -const INTERACTIVE_QUESTION_TOOL = { - question: { - description: "Ask the user a question and wait for their answer.", - inputSchema: z.looseObject({}), - execute: async () => - "The user cannot answer an interactive prompt in this environment. Ask the question in your reply instead and end your turn; the user answers in their next message.", - }, -}; - -export function createAuthoringAgent(subject: { - readonly harness: AuthoringBenchmarkModel["harness"]; - readonly model: string; - readonly archive: Uint8Array; - readonly dependencyArchive: Uint8Array; - readonly digest: string; - readonly dependencyDigest: string; -}): Agent { - return { - name: harnessId(subject.harness), - displayName: `${subject.harness} eve authoring harness`, - getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY", - getDefaultModel: () => subject.model, - definition: { - name: harnessId(subject.harness), - displayName: `${subject.harness} eve authoring harness`, - defaultModel: subject.model, - o11yAgentName: harnessId(subject.harness), - runnerPath: "", - getApiKeyEnvVar: () => "AI_GATEWAY_API_KEY", - install: () => [], - configFiles: () => [], - authEnv: () => ({}), - }, - async run(fixturePath, options): Promise { - const verbose = options.agentOptions?.verbose === true; - const log = (message: string) => { - if (verbose) console.log(message); - }; - const authoringCase = await loadAuthoringCase(fixturePath); - const setups = [authoringCase.startingPoint.setup, authoringCase.setup].filter( - (setup): setup is AuthoringSetup => setup !== undefined, - ); - const timings = new BenchmarkTimings(); - timings.record("run.context", 0, "success", { - sourceArchiveBytes: subject.archive.length, - dependencyArchiveBytes: subject.dependencyArchive.length, - sourceDigest: subject.digest, - dependencyDigest: subject.dependencyDigest, - startingPoint: authoringCase.startingPoint.id, - setupCount: setups.length, - }); - const sandbox = createDependencyCachedSandbox({ - archive: subject.archive, - dependencyArchive: subject.dependencyArchive, - dependencyDigest: subject.dependencyDigest, - ports: [HARNESS_BRIDGE_PORT, ...new Set(setups.flatMap((setup) => setup.ports ?? []))], - env: { - EVE_INIT_PACKAGE_SPEC: EVE_PACKAGE_PATH, - [AUTHORING_EVAL_DIRECTORY_ENV]: AUTHORING_EVAL_DIRECTORY, - ...Object.assign({}, ...setups.map((setup) => setup.environment ?? {})), - }, - log, - timings, - }); - const startedAt = Date.now(); - const commands: string[] = []; - const transcript: AuthoringTranscriptEntry[] = []; - let session: HarnessAgentSession | undefined; - let activeSandbox: HarnessV1NetworkSandboxSession | undefined; - let workspace: string | undefined; - - const agent = new HarnessAgent({ - id: "eve-authoring-eval", - harness: createHarness(subject.harness, options.model ?? subject.model), - sandbox, - tools: INTERACTIVE_QUESTION_TOOL, - sandboxConfig: { - workDir: WORKSPACE, - bootstrapHash: bootstrapHash(authoringCase, subject), - onBootstrap: async ({ session: bootstrap, workDir }) => { - const bootstrapSandbox = bootstrap as HarnessV1NetworkSandboxSession; - const context = setupContext(bootstrapSandbox, workDir); - log("[setup] building the selected eve source"); - await bootstrapSubject( - bootstrapSandbox, - workDir, - authoringCase.startingPoint.workspace, - subject.archive, - timings, - ); - // Case setup runs against the finished starting point. A setup that - // installs a fixture dependency would otherwise create the project's - // `package.json` itself, and `eve init` would then treat the - // workspace as an existing package and skip its own scaffold. - await timings.measure("subject.case-bootstrap", async () => { - for (const setup of setups) await setup.onBootstrap?.(context); - }); - }, - onSession: async ({ session: current, sessionWorkDir }) => { - activeSandbox = current as HarnessV1NetworkSandboxSession; - workspace = sessionWorkDir; - const context = setupContext(activeSandbox, workspace); - await ensureWorkspace( - context, - authoringCase.startingPoint.workspace, - subject.archive, - timings, - ); - await timings.measure("session.setup", async () => { - for (const setup of setups) await setup.onSession?.(context); - if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context); - }); - if (options.agentOptions?.agentsMd !== true) { - await context.run("rm -f AGENTS.md CLAUDE.md GEMINI.md"); - } - }, - }, - permissionMode: "allow-all", - }); - - try { - session = await timings.measure("session.create", () => - agent.createSession({ abortSignal: options.signal }), - ); - if (activeSandbox === undefined || workspace === undefined) { - throw new Error("HarnessAgent did not initialize its sandbox session."); - } - - await authoringCase.interact({ - session, - transcript, - send: async (prompt) => { - const turn = transcript.filter((entry) => entry.role === "user").length + 1; - transcript.push({ role: "user", content: prompt }); - if (verbose) console.log(`[user] ${prompt}`); - const result = await timings.measure(`agent.turn.${turn}`, () => - runTurn({ - agent, - session: session!, - prompt, - timeout: options.timeout, - verbose, - abortSignal: options.signal, - }), - ); - const toolCalls = result.toolCalls; - const usage = result.usage; - transcript.push({ role: "assistant", content: result.text, toolCalls, usage }); - const details: Record = { - promptCharacters: prompt.length, - responseCharacters: result.text.length, - toolCalls: toolCalls.length, - inputTokens: usage.inputTokens, - outputTokens: usage.outputTokens, - reasoningTokens: usage.reasoningTokens, - }; - if (result.stall !== undefined) details.stall = result.stall; - timings.record( - `agent.turn.${turn}.summary`, - 0, - result.stall === undefined ? "success" : "failure", - details, - ); - if (result.stall !== undefined) log(`[turn ${turn}] ${result.stall}`); - commands.push(...shellCommands(result.toolCalls)); - return { text: result.text, toolCalls }; - }, - }); - - const projectWorkspace = - authoringCase.projectDirectory === undefined - ? workspace - : `${workspace}/${authoringCase.projectDirectory}`; - const context = setupContext(activeSandbox, workspace); - const graderContext = setupContext(activeSandbox, projectWorkspace); - await prepareGraderDirectory(context); - await context.write( - `${AGENT_EVAL_DIRECTORY}/results.json`, - JSON.stringify({ o11y: { shellCommands: commands.map((command) => ({ command })) } }), - ); - await context.write( - `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`, - JSON.stringify(transcript), - ); - await Promise.all([ - graderContext.write( - `${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`, - readFileSync(`${fixturePath}/EVAL.ts`, "utf8"), - ), - graderContext.write( - `${POST_RUN_GRADER_DIRECTORY}/grader.ts`, - readFileSync(new URL("./grader.ts", import.meta.url), "utf8"), - ), - graderContext.write( - `${POST_RUN_GRADER_DIRECTORY}/paths.ts`, - readFileSync(new URL("./paths.ts", import.meta.url), "utf8"), - ), - graderContext.write( - `${POST_RUN_GRADER_DIRECTORY}/protocol.ts`, - readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"), - ), - ]); - - log("[grade] running deterministic assertions"); - const test = await timings.measure("validation.grader", () => - resultOf( - activeSandbox!, - `vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`, - projectWorkspace, - ), - ); - const scriptsResults = Object.fromEntries( - await Promise.all( - (options.scripts ?? []).map(async (script) => { - log(`[${script}] running`); - const result = await timings.measure(`validation.${script}`, () => - resultOf(activeSandbox!, `npm run ${shellQuote(script)}`, projectWorkspace), - ); - return [ - script, - { success: result.exitCode === 0, output: `${result.stdout}${result.stderr}` }, - ] as const; - }), - ), - ); - const scriptsPassed = Object.values(scriptsResults).every((result) => result.success); - timings.record("run.total", Date.now() - startedAt); - await context.write( - `${AGENT_EVAL_DIRECTORY}/timings.json`, - JSON.stringify(timings.entries), - ); - logTimingSummary(log, timings); - log(`[result] ${test.exitCode === 0 && scriptsPassed ? "passed" : "failed"}`); - return { - success: test.exitCode === 0 && scriptsPassed, - output: transcript.at(-1)?.content ?? "", - error: - test.exitCode === 0 && scriptsPassed ? undefined : `${test.stdout}\n${test.stderr}`, - duration: Date.now() - startedAt, - testResult: { success: test.exitCode === 0, output: `${test.stdout}${test.stderr}` }, - transcript: harnessTranscript(transcript), - scriptsResults, - sandboxId: activeSandbox.id, - generatedFiles: timingArtifact(timings), - }; - } catch (error) { - timings.record("run.total", Date.now() - startedAt, "failure"); - if (activeSandbox !== undefined && workspace !== undefined) { - await prepareGraderDirectory(setupContext(activeSandbox, workspace)).catch( - () => undefined, - ); - await Promise.all([ - setupContext(activeSandbox, workspace).write( - `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`, - JSON.stringify(transcript), - ), - setupContext(activeSandbox, workspace).write( - `${AGENT_EVAL_DIRECTORY}/timings.json`, - JSON.stringify(timings.entries), - ), - ]).catch(() => undefined); - } - const result: AgentRunResult = { - success: false, - output: transcript.at(-1)?.content ?? "", - error: error instanceof Error ? error.message : String(error), - duration: Date.now() - startedAt, - transcript: harnessTranscript(transcript), - scriptsResults: {}, - generatedFiles: timingArtifact(timings), - }; - if (activeSandbox !== undefined) result.sandboxId = activeSandbox.id; - return result; - } finally { - await session?.destroy(); - } - }, - }; -} - -interface AuthoringTurnResult extends AuthoringTurn { - readonly usage: AuthoringTokenUsage; - /** Set when the turn did not finish on its own and was cut short. */ - readonly stall?: string; -} - -// One streaming path for every turn, so a turn that never finishes still yields -// the text and tool calls it produced. The underlying runtime can leave a turn -// open indefinitely after the model's last message, and a turn that consumed the -// whole eval budget used to discard the transcript along with it. -async function runTurn(input: { - readonly agent: AuthoringHarnessAgent; - readonly session: HarnessAgentSession; - readonly prompt: string; - readonly timeout: number; - readonly verbose: boolean; - readonly abortSignal?: AbortSignal; -}): Promise { - const { agent, session, prompt, timeout, verbose } = input; - const steps: string[] = []; - const toolCalls: Array<{ name: string; input: unknown }> = []; - const usage = { - inputTokens: 0, - outputTokens: 0, - reasoningTokens: 0, - cachedInputTokens: 0, - cacheWriteTokens: 0, - }; - let current = ""; - let lineOpen = false; - const turnTimeout = Math.min(timeout, TURN_TIMEOUT_SECONDS); - let stall: string | undefined; - - // The runtime routinely leaves a turn open after the model's last message and - // honors neither the total nor the per-chunk budget the SDK passes down, so - // the harness has to decide when a turn is over. Text the model never follows - // with a tool call is its closing message, so silence after it ends the turn; - // silence anywhere else can still be work in flight. - const controller = new AbortController(); - const abort = () => controller.abort(); - input.abortSignal?.addEventListener("abort", abort, { once: true }); - const deadline = Date.now() + turnTimeout * 1000; - - try { - const result = await agent.stream({ - session, - prompt, - timeout: { totalMs: turnTimeout * 1000, chunkMs: TURN_STALL_SECONDS * 1000 }, - abortSignal: controller.signal, - }); - const stream = result.fullStream[Symbol.asyncIterator](); - const tracing = process.env.EVE_BENCHMARK_TRACE_PARTS === "1"; - let lastPartAt = Date.now(); - let pendingTools = 0; - let stepCalledTool = false; - let turnLooksDone = false; - for (;;) { - const idle = pendingTools > 0 ? Number.POSITIVE_INFINITY : idleBudget(turnLooksDone); - const budget = Math.min(idle, deadline - Date.now()); - const next = await withDeadline(stream.next(), budget); - if (next === "expired") { - if (Date.now() >= deadline) stall = `turn exceeded its ${turnTimeout}s budget`; - else if (!turnLooksDone) stall = `turn produced no output for ${TURN_STALL_SECONDS}s`; - await settleStalledTurn(stream, controller, stall ?? "turn ended on its closing message"); - break; - } - if (next.done === true) break; - const part = next.value; - if (tracing) { - const now = Date.now(); - console.log(`[part] +${((now - lastPartAt) / 1000).toFixed(1)}s ${part.type}`); - lastPartAt = now; - } - if (part.type === "text-delta") { - current += part.text; - // Text the model is not following with a tool call is it signing off. - // The runtime often emits nothing after that closing message, not even - // the step boundary, so the text itself has to be the signal. - turnLooksDone = true; - if (verbose) { - if (!lineOpen) process.stdout.write("[assistant] "); - lineOpen = true; - process.stdout.write(part.text); - } - continue; - } - if (verbose && lineOpen) process.stdout.write("\n"); - lineOpen = false; - if (part.type === "tool-call") { - // Only a tool call reopens the turn. The parts that bracket a message - // (`text-start`, `text-end`) are not work, and treating them as work is - // what used to hide a closing message behind the full stall budget. - turnLooksDone = false; - pendingTools += 1; - stepCalledTool = true; - toolCalls.push({ name: part.toolName, input: part.input }); - if (verbose) console.log(`[tool] ${formatToolCall(part.toolName, part.input)}`); - } else if (part.type === "tool-result" || part.type === "tool-error") { - pendingTools = Math.max(0, pendingTools - 1); - } else if (part.type === "finish-step") { - if (current.trim().length > 0) steps.push(current.trim()); - current = ""; - addUsage(usage, (part as { usage?: unknown }).usage); - turnLooksDone = !stepCalledTool; - stepCalledTool = false; - } else if (part.type === "finish") { - replaceUsage(usage, (part as { totalUsage?: unknown }).totalUsage); - turnLooksDone = true; - } - } - } catch (error) { - stall = `turn did not finish: ${error instanceof Error ? error.message : String(error)}`; - } finally { - input.abortSignal?.removeEventListener("abort", abort); - if (verbose && lineOpen) process.stdout.write("\n"); - } - - if (current.trim().length > 0) steps.push(current.trim()); - const turn: AuthoringTurnResult = { text: steps.join("\n\n"), toolCalls, usage }; - return stall === undefined ? turn : { ...turn, stall }; -} - -function idleBudget(turnLooksDone: boolean): number { - return turnLooksDone ? CLOSING_IDLE_MILLIS : TURN_STALL_SECONDS * 1000; -} - -// Aborting is what marks the turn finished. Walking away from the stream is not -// enough: the session keeps the turn open and rejects the next prompt as one -// already in progress, which loses every remaining turn of the case. Draining -// afterwards gives that settlement time to land before the next prompt. -async function settleStalledTurn( - stream: AsyncIterator, - controller: AbortController, - reason: string, -): Promise { - controller.abort(new Error(reason)); - await withDeadline( - (async () => { - try { - while ((await stream.next()).done !== true); - } catch { - // The abort surfaces here as a rejection, which is the settlement. - } - })(), - TURN_SETTLE_MILLIS, - ); -} - -async function withDeadline(promise: Promise, millis: number): Promise { - if (millis <= 0) return "expired"; - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise<"expired">((resolve) => { - timer = setTimeout(() => resolve("expired"), millis); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } -} - -type MutableAuthoringTokenUsage = { - -readonly [Key in keyof AuthoringTokenUsage]: AuthoringTokenUsage[Key]; -}; - -export function addUsage(total: MutableAuthoringTokenUsage, value: unknown): void { - if (typeof value !== "object" || value === null) return; - const usage = value as Record; - const input = usage.inputTokens; - const output = usage.outputTokens; - if (typeof input === "object" && input !== null) { - const tokens = input as Record; - total.inputTokens += usageNumber(tokens.noCache ?? tokens.total); - total.cachedInputTokens += usageNumber(tokens.cacheRead); - total.cacheWriteTokens += usageNumber(tokens.cacheWrite); - } else { - total.inputTokens += usageNumber(input); - } - if (typeof output === "object" && output !== null) { - total.outputTokens += usageNumber((output as Record).total); - } else { - total.outputTokens += usageNumber(output); - } - total.reasoningTokens += usageNumber(usage.reasoningTokens); - total.cachedInputTokens += usageNumber(usage.cachedInputTokens); - total.cacheWriteTokens += usageNumber(usage.cacheWriteTokens); -} - -function replaceUsage(total: MutableAuthoringTokenUsage, value: unknown): void { - if (typeof value !== "object" || value === null) return; - total.inputTokens = 0; - total.outputTokens = 0; - total.reasoningTokens = 0; - total.cachedInputTokens = 0; - total.cacheWriteTokens = 0; - addUsage(total, value); -} - -function usageNumber(value: unknown): number { - return typeof value === "number" && Number.isFinite(value) ? value : 0; -} - -function createHarness(harness: AuthoringBenchmarkModel["harness"], model: string): HarnessV1 { - if (harness === "Claude Code") { - return createClaudeCode({ auth: { gateway: {} }, model }); - } - return createOpenCode({ - auth: "ai-gateway", - model: openCodeModel(model), - port: HARNESS_BRIDGE_PORT, - }); -} - -function harnessId(harness: AuthoringBenchmarkModel["harness"]): string { - return harness === "Claude Code" ? "claude-code" : "opencode"; -} - -function openCodeModel(model: string): string { - const gatewayModel = model.includes("/") ? model : `anthropic/${model}`; - // OpenCode's Moonshot provider supplies the OpenAI-compatible transport; the - // canonical model ID still makes AI Gateway select the requested provider. - return `moonshotai/${gatewayModel}`; -} - -function formatToolCall(name: string, input: unknown): string { - if (typeof input === "object" && input !== null) { - const command = (input as { command?: unknown }).command; - if (typeof command === "string") return command; - } - return `${name} ${JSON.stringify(input)}`; -} - -async function bootstrapSubject( - sandbox: HarnessV1NetworkSandboxSession, - workspace: string, - workspaceKind: "scaffolded" | "empty", - archive: Uint8Array, - timings: BenchmarkTimings, -): Promise { - await timings.measure("subject.source-upload", async () => { - await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive }); - }); - await timings.measure("subject.source-install", () => - run( - sandbox, - `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`, - ), - ); - await timings.measure("subject.eve-build", () => - run(sandbox, `pnpm --dir ${SOURCE_ROOT} --filter eve build`), - ); - await timings.measure("subject.eve-pack-and-cli", () => - run( - sandbox, - `mkdir -p /tmp/eve-package && pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package && mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH} && ln -sf ${SOURCE_ROOT}/packages/eve/bin/eve.js /usr/local/bin/eve && command -v eve`, - ), - ); - const workspaceCommands: string[] = []; - if (workspaceKind === "scaffolded") { - workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`); - } - // Grader files are created after the agent finishes so an empty starting point remains empty. - workspaceCommands.push("command -v vitest >/dev/null"); - await timings.measure("subject.workspace-bootstrap", () => - run(sandbox, workspaceCommands.join(" && ")), - ); -} - -function setupContext( - sandbox: HarnessV1NetworkSandboxSession, - workspace: string, -): AuthoringSetupContext { - return { - sandbox, - workspace, - artifactsRoot: AUTHORING_EVAL_DIRECTORY, - run: async (command, workingDirectory = workspace) => { - await run(sandbox, command, workingDirectory); - }, - write: async (path, content) => { - await sandbox.writeTextFile({ - path: path.startsWith("/") ? path : `${workspace}/${path}`, - content, - }); - }, - }; -} - -// Created only after the agent's turns finish: an `empty` starting point has to -// look empty to `eve init .`, which refuses to scaffold into a directory that -// already holds entries it does not recognize. -async function ensureWorkspace( - context: AuthoringSetupContext, - workspaceKind: "scaffolded" | "empty", - archive: Uint8Array, - timings: BenchmarkTimings, -): Promise { - if (workspaceKind === "empty") { - await verifyWorkspace(context, workspaceKind); - return; - } - if (await hasPreparedWorkspace(context)) return; - - await timings.measure("session.workspace-recovery", () => - bootstrapSubject(context.sandbox, context.workspace, workspaceKind, archive, timings), - ); - await verifyWorkspace(context, workspaceKind); -} - -async function hasPreparedWorkspace(context: AuthoringSetupContext): Promise { - const result = await resultOf( - context.sandbox, - `test -f package.json && test -f agent/instructions.md && test -f AGENTS.md && test -f ${EVE_PACKAGE_PATH}`, - context.workspace, - ); - return result.exitCode === 0; -} - -async function verifyWorkspace( - context: AuthoringSetupContext, - workspaceKind: "scaffolded" | "empty", -): Promise { - const command = - workspaceKind === "empty" - ? "test ! -e package.json && test ! -e AGENTS.md && test ! -e agent && test ! -e .eve-grader" - : `for path in package.json agent/instructions.md AGENTS.md ${EVE_PACKAGE_PATH}; do test -f "$path" || { echo "Missing prepared workspace file: $path" >&2; exit 1; }; done`; - await context.run(command); -} - -async function prepareGraderDirectory(context: AuthoringSetupContext): Promise { - await context.run( - `mkdir -p ${AGENT_EVAL_DIRECTORY} && printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`, - ); -} - -async function installBaselineEveWrapper(context: AuthoringSetupContext): Promise { - await context.run(` -cli_path="/usr/local/bin/eve" -test -x "$cli_path" -real_cli="$cli_path.guided" -if [ ! -e "$real_cli" ]; then mv "$cli_path" "$real_cli"; fi -cat >"$cli_path" <<'EOF' -#!/bin/sh -"$0.guided" "$@" -status=$? -rm -f AGENTS.md CLAUDE.md GEMINI.md -exit "$status" -EOF -chmod +x "$cli_path" -`); -} - -function timingArtifact(timings: BenchmarkTimings): Record { - return { "benchmark/timings.json": `${JSON.stringify(timings.entries, null, 2)}\n` }; -} - -function logTimingSummary(log: (message: string) => void, timings: BenchmarkTimings): void { - for (const timing of timings.entries) { - log(`[timing] ${timing.phase}: ${timing.durationMs}ms (${timing.outcome})`); - } -} - -function harnessTranscript(transcript: ReadonlyArray): string { - return transcript - .map((entry) => { - const message: { - role: AuthoringTranscriptEntry["role"]; - content: unknown; - usage?: AuthoringTokenUsage; - } = { - role: entry.role, - content: - entry.role === "assistant" - ? [ - ...(entry.content ? [{ type: "text", text: entry.content }] : []), - ...(entry.toolCalls ?? []).map((call) => ({ - type: "tool_use", - name: call.name, - input: call.input, - })), - ] - : entry.content, - }; - if (entry.usage !== undefined) message.usage = entry.usage; - return JSON.stringify({ type: entry.role, message }); - }) - .join("\n"); -} - -function shellCommands(toolCalls: ReadonlyArray<{ input: unknown }>): string[] { - return toolCalls.flatMap((call) => { - if (typeof call.input !== "object" || call.input === null) return []; - const command = (call.input as { command?: unknown }).command; - return typeof command === "string" ? [command] : []; - }); -} - -function bootstrapHash(authoringCase: AuthoringCase, subject: { readonly digest: string }): string { - const setupIds = [authoringCase.startingPoint.setup, authoringCase.setup] - .filter((setup): setup is AuthoringSetup => setup !== undefined) - .map((setup) => setup.id) - .join("-"); - return `eve-authoring-${BOOTSTRAP_VERSION}-${subject.digest}-${authoringCase.startingPoint.id}-${setupIds}`; -} - -async function resultOf( - sandbox: HarnessV1NetworkSandboxSession, - command: string, - workingDirectory?: string, -) { - const options: { command: string; workingDirectory?: string } = { command }; - if (workingDirectory !== undefined) options.workingDirectory = workingDirectory; - return sandbox.run(options); -} - -async function run( - sandbox: HarnessV1NetworkSandboxSession, - command: string, - workingDirectory?: string, -): Promise { - const result = await resultOf(sandbox, command, workingDirectory); - if (result.exitCode !== 0) { - throw new Error(`${command} failed (${result.exitCode}):\n${result.stdout}\n${result.stderr}`); - } -} - -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'\\''`)}'`; -} diff --git a/apps/benchmarks/lib/native-authoring-setup.test.mjs b/apps/benchmarks/lib/native-authoring-setup.test.mjs new file mode 100644 index 000000000..f0ce0a6ad --- /dev/null +++ b/apps/benchmarks/lib/native-authoring-setup.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createJiti } from "jiti"; + +const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false }); +const { createNativeAuthoringSetup } = await jiti.import( + new URL("./native-authoring-setup.ts", import.meta.url).pathname, +); + +test("bootstraps the selected source before a native agent starts", async () => { + const commands = []; + const writes = []; + const setup = createNativeAuthoringSetup({ + packageSpec: "https://pkg.eve.dev/commit/eve.tgz", + revision: "commit", + treatment: "baseline", + }); + await setup({ + async readFile(path) { + assert.equal(path, ".eve-authoring-bootstrap.json"); + return JSON.stringify({ + startingPoint: "scaffolded", + revision: "commit", + setupIds: ["inventory-openapi-v1"], + }); + }, + async writeFiles(files) { + writes.push(files); + }, + async runCommand(command, args, options) { + commands.push({ command, args, options }); + return { stdout: "", stderr: "", exitCode: 0 }; + }, + getWorkingDirectory: () => "/workspace", + setWorkingDirectory() {}, + }); + + assert.match(commands[0].args[1], /rm -f \.eve-authoring-bootstrap\.json/u); + assert.deepEqual(Object.keys(writes[0]), ["/usr/local/bin/eve"]); + assert.match(writes[0]["/usr/local/bin/eve"], /pkg\.eve\.dev\/commit\/eve\.tgz/u); + assert.match(commands[1].args[1], /chmod \+x \/usr\/local\/bin\/eve/u); + assert.match( + commands[2].args[1], + /AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=.* eve init \. --model openai\/gpt-5\.5/u, + ); + assert.match(commands[3].args[1], /mkdir -p agent\/lib/u); + assert.match(commands[4].args[1], /rm -f AGENTS\.md CLAUDE\.md GEMINI\.md/u); + assert.match(commands[5].args[1], /git add \. && git commit --amend --no-edit --quiet/u); + assert.deepEqual(Object.keys(writes[1]), ["/workspace/agent/lib/inventory-openapi.ts"]); + assert.match(writes[1]["/workspace/agent/lib/inventory-openapi.ts"], /getStock/u); +}); diff --git a/apps/benchmarks/lib/native-authoring-setup.ts b/apps/benchmarks/lib/native-authoring-setup.ts new file mode 100644 index 000000000..8d9475824 --- /dev/null +++ b/apps/benchmarks/lib/native-authoring-setup.ts @@ -0,0 +1,106 @@ +import type { Sandbox } from "@vercel/agent-eval"; + +import type { AuthoringSetup, AuthoringSetupContext } from "./authoring-case.js"; +import { inventoryOpenApiSetup } from "./setups/inventory-openapi.js"; + +interface FixtureBootstrap { + readonly startingPoint: "scaffolded" | "empty"; + readonly projectDirectory?: string; + readonly revision: string; + readonly setupIds: readonly string[]; +} + +export function createNativeAuthoringSetup(options: { + readonly packageSpec: string; + readonly revision: string; + readonly treatment: "baseline" | "guided"; +}) { + return async (sandbox: Sandbox): Promise => { + const bootstrap = JSON.parse( + await sandbox.readFile(".eve-authoring-bootstrap.json"), + ) as FixtureBootstrap; + if (bootstrap.revision !== options.revision) { + throw new Error("Native fixture revision does not match the selected subject."); + } + await run( + sandbox, + "rm -f .eve-authoring-bootstrap.json CASE.ts package.json package-lock.json PROMPT.md", + "fixture cleanup", + ); + await sandbox.writeFiles({ + "/usr/local/bin/eve": `#!/bin/sh\nexec npx --yes --allow-remote=all --package=${shellQuote(options.packageSpec)} eve "$@"\n`, + }); + await run(sandbox, "chmod +x /usr/local/bin/eve", "eve canary wrapper"); + + if (bootstrap.startingPoint === "scaffolded") { + await run( + sandbox, + `AI_AGENT=claude EVE_INIT_PACKAGE_SPEC=${shellQuote(options.packageSpec)} eve init . --model openai/gpt-5.5`, + "workspace bootstrap", + ); + } + + if (bootstrap.startingPoint === "empty") { + if (bootstrap.projectDirectory === undefined) { + throw new Error("An empty native fixture must declare its project directory."); + } + await sandbox.writeFiles({ + "package.json": `${JSON.stringify({ + private: true, + scripts: { + typecheck: `npm --prefix ${bootstrap.projectDirectory} run typecheck`, + build: `npm --prefix ${bootstrap.projectDirectory} run build`, + }, + })}\n`, + }); + } + + const context = setupContext(sandbox, bootstrap.projectDirectory); + for (const setup of setupsFor(bootstrap.setupIds)) await setup.onSession?.(context); + if (options.treatment === "baseline") await run(sandbox, "rm -f AGENTS.md CLAUDE.md GEMINI.md"); + await run(sandbox, "git add . && git commit --amend --no-edit --quiet"); + }; +} + +function setupsFor(ids: readonly string[]): readonly AuthoringSetup[] { + return ids.flatMap((id) => (id === inventoryOpenApiSetup.id ? [inventoryOpenApiSetup] : [])); +} + +function setupContext(sandbox: Sandbox, projectDirectory?: string): AuthoringSetupContext { + const workspace = + projectDirectory === undefined + ? sandbox.getWorkingDirectory() + : `${sandbox.getWorkingDirectory()}/${projectDirectory}`; + return { + workspace, + artifactsRoot: "/tmp/photon", + run: async (command, workingDirectory = workspace) => + run(sandbox, command, command, undefined, workingDirectory), + write: async (path, content) => + sandbox.writeFiles({ [path.startsWith("/") ? path : `${workspace}/${path}`]: content }), + }; +} + +async function run( + sandbox: Sandbox, + command: string, + label = command, + env?: Record, + cwd?: string, +): Promise { + const result = await sandbox.runCommand( + "bash", + ["-lc", `timeout 240 bash -lc ${shellQuote(command)}`], + { + env, + cwd, + }, + ); + if (result.exitCode !== 0) { + throw new Error(`${label} failed (${result.exitCode}):\n${result.stdout}\n${result.stderr}`); + } +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} diff --git a/apps/benchmarks/lib/source.mjs b/apps/benchmarks/lib/source.mjs index 33ea86577..f0a0a42b4 100644 --- a/apps/benchmarks/lib/source.mjs +++ b/apps/benchmarks/lib/source.mjs @@ -1,108 +1,48 @@ import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -export function workingTreeSubject(repositoryRoot) { - return archiveSubject(repositoryRoot, "working tree", "current", (archivePath, environment) => { - git(repositoryRoot, ["read-tree", "HEAD"], environment); - git(repositoryRoot, ["add", "-A"], environment); - const tree = git(repositoryRoot, ["write-tree"], environment).trim(); - git(repositoryRoot, ["archive", "--format=tar.gz", `--output=${archivePath}`, tree]); - return tree; - }); -} +const PACKAGE_HOST = "https://pkg.eve.dev"; +const IMMUTABLE_PACKAGE_PATH = /^\/([0-9a-f]{40})\/eve\.tgz$/u; -export function revisionSubject(repositoryRoot, requestedRevision, label) { - const revision = git(repositoryRoot, [ - "rev-parse", - "--verify", - `${requestedRevision}^{commit}`, - ]).trim(); - return archiveSubject( - repositoryRoot, - revision.slice(0, 12), +/** Resolves a mutable canary ref to the immutable artifact all runs must share. */ +export function canarySubject(ref, label, resolve = resolveCanaryPackageSpec) { + const packageSpec = resolve(ref); + const revision = packageRevision(packageSpec); + return { label, - (archivePath) => { - const tree = git(repositoryRoot, ["rev-parse", `${revision}^{tree}`]).trim(); - git(repositoryRoot, ["archive", "--format=tar.gz", `--output=${archivePath}`, revision]); - return tree; - }, - { revision }, - ); + revision, + description: revision.slice(0, 12), + packageSpec, + }; } -function archiveSubject(repositoryRoot, description, label, createArchive, details = {}) { - const temporaryDirectory = mkdtempSync(join(tmpdir(), "eve-authoring-")); - const archivePath = join(temporaryDirectory, "source.tar.gz"); - const indexPath = join(temporaryDirectory, "index"); +export function resolveCanaryPackageSpec(ref) { + const requested = `${PACKAGE_HOST}/${encodeURIComponent(ref)}/eve.tgz`; + let resolved; try { - const digest = createArchive(archivePath, { ...process.env, GIT_INDEX_FILE: indexPath }); - const archive = readFileSync(archivePath); - return { - label, - description, - archive, - digest, - dependencyArchive: dependencyArchive(archive), - dependencyDigest: dependencyDigest(archive), - ...details, - }; - } finally { - rmSync(temporaryDirectory, { recursive: true, force: true }); + resolved = execFileSync( + "curl", + ["-fsSL", "-o", "/dev/null", "-w", "%{url_effective}", requested], + { + encoding: "utf8", + }, + ).trim(); + } catch { + throw new Error( + `No eve canary artifact is available for ${JSON.stringify(ref)}. Publish that revision or use a published canary ref such as "main".`, + ); } + packageRevision(resolved); + return resolved; } -function dependencyDigest(archive) { - const hash = createHash("sha256"); - for (const path of dependencyPaths(archive)) { - const content = execFileSync("tar", ["-xOzf", "-", path], { - input: archive, - maxBuffer: 10 * 1024 * 1024, - }); - hash.update(path).update("\0").update(content).update("\0"); +export function packageRevision(packageSpec) { + const url = new URL(packageSpec); + if (url.origin !== PACKAGE_HOST) { + throw new Error(`Eve canary resolved outside ${PACKAGE_HOST}: ${packageSpec}`); } - return hash.digest("hex"); -} - -function dependencyArchive(archive) { - const directory = mkdtempSync(join(tmpdir(), "eve-authoring-dependencies-")); - const archivePath = join(directory, "dependencies.tar.gz"); - try { - for (const path of dependencyPaths(archive)) { - const destination = join(directory, "input", path); - mkdirSync(join(destination, ".."), { recursive: true }); - writeFileSync( - destination, - execFileSync("tar", ["-xOzf", "-", path], { - input: archive, - maxBuffer: 10 * 1024 * 1024, - }), - ); - } - execFileSync("tar", ["-czf", archivePath, "-C", join(directory, "input"), "."]); - return readFileSync(archivePath); - } finally { - rmSync(directory, { recursive: true, force: true }); + const revision = url.pathname.match(IMMUTABLE_PACKAGE_PATH)?.[1]; + if (revision === undefined) { + throw new Error(`Eve canary did not resolve to an immutable revision: ${packageSpec}`); } -} - -function dependencyPaths(archive) { - const paths = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8" }) - .split("\n") - .filter(Boolean); - return paths.filter( - (path) => - path === ".npmrc" || - path === "package.json" || - path === "pnpm-lock.yaml" || - path === "pnpm-workspace.yaml" || - /(?:^|\/)package\.json$/u.test(path) || - path.startsWith("patches/"), - ); -} - -function git(cwd, args, env = process.env) { - return execFileSync("git", args, { cwd, env, encoding: "utf8" }); + return revision; } diff --git a/apps/benchmarks/lib/source.test.mjs b/apps/benchmarks/lib/source.test.mjs index 19c6df031..31b0ee71f 100644 --- a/apps/benchmarks/lib/source.test.mjs +++ b/apps/benchmarks/lib/source.test.mjs @@ -1,96 +1,37 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { test } from "node:test"; -import { revisionSubject, workingTreeSubject } from "./source.mjs"; +import { canarySubject, packageRevision } from "./source.mjs"; -test("archives the working tree without changing the index", () => { - const repository = repositoryFixture(); - try { - writeFileSync(join(repository, "tracked.txt"), "changed\n"); - writeFileSync(join(repository, "new.txt"), "new\n"); - writeFileSync(join(repository, "ignored.txt"), "ignored\n"); - rmSync(join(repository, "deleted.txt")); - const statusBefore = git(repository, ["status", "--porcelain"]); +const revision = "1234567890abcdef1234567890abcdef12345678"; +const packageSpec = `https://pkg.eve.dev/${revision}/eve.tgz`; - const subject = workingTreeSubject(repository); - withExtracted(subject.archive, (extracted) => { - assert.equal(readFileSync(join(extracted, "tracked.txt"), "utf8"), "changed\n"); - assert.equal(readFileSync(join(extracted, "new.txt"), "utf8"), "new\n"); - assert.equal(existsSync(join(extracted, "ignored.txt")), false); - assert.equal(existsSync(join(extracted, "deleted.txt")), false); - }); - withExtracted(subject.dependencyArchive, (extracted) => { - assert.equal(readFileSync(join(extracted, "package.json"), "utf8"), '{"private":true}\n'); - assert.equal( - readFileSync(join(extracted, "pnpm-lock.yaml"), "utf8"), - "lockfileVersion: '9.0'\n", - ); - assert.equal(existsSync(join(extracted, "tracked.txt")), false); - }); - assert.equal(git(repository, ["status", "--porcelain"]), statusBefore); - writeFileSync(join(repository, "tracked.txt"), "another source change\n"); - const sourceChange = workingTreeSubject(repository); - assert.notEqual(sourceChange.digest, subject.digest); - assert.equal(sourceChange.dependencyDigest, subject.dependencyDigest); - assert.deepEqual(sourceChange.dependencyArchive, subject.dependencyArchive); - writeFileSync(join(repository, "packages.json"), "not a manifest\n"); - assert.equal(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest); - writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n"); - assert.notEqual(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest); - } finally { - rmSync(repository, { recursive: true, force: true }); - } +test("resolves a canary alias once to an immutable subject", () => { + const calls = []; + const subject = canarySubject("main", "current", (ref) => { + calls.push(ref); + return packageSpec; + }); + + assert.deepEqual(calls, ["main"]); + assert.deepEqual(subject, { + label: "current", + revision, + description: revision.slice(0, 12), + packageSpec, + }); }); -test("archives a local revision", () => { - const repository = repositoryFixture(); - try { - writeFileSync(join(repository, "tracked.txt"), "working tree\n"); - const subject = revisionSubject(repository, "HEAD", "base"); - assert.equal(subject.label, "base"); - withExtracted(subject.archive, (extracted) => { - assert.equal(readFileSync(join(extracted, "tracked.txt"), "utf8"), "committed\n"); - }); - } finally { - rmSync(repository, { recursive: true, force: true }); - } +test("rejects a non-immutable canary URL", () => { + assert.throws( + () => packageRevision("https://pkg.eve.dev/main/eve.tgz"), + /did not resolve to an immutable revision/u, + ); }); -function repositoryFixture() { - const repository = mkdtempSync(join(tmpdir(), "eve-authoring-source-test-")); - git(repository, ["init", "--quiet"]); - git(repository, ["config", "user.name", "Test"]); - git(repository, ["config", "user.email", "test@example.com"]); - writeFileSync(join(repository, ".gitignore"), "ignored.txt\n"); - writeFileSync(join(repository, ".npmrc"), "link-workspace-packages=true\n"); - writeFileSync(join(repository, "package.json"), '{"private":true}\n'); - writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n"); - writeFileSync(join(repository, "pnpm-workspace.yaml"), "packages: []\n"); - writeFileSync(join(repository, "tracked.txt"), "committed\n"); - writeFileSync(join(repository, "deleted.txt"), "delete me\n"); - git(repository, ["add", "."]); - git(repository, ["commit", "--quiet", "-m", "fixture"]); - return repository; -} - -function withExtracted(archive, assertion) { - const directory = mkdtempSync(join(tmpdir(), "eve-authoring-archive-test-")); - const archivePath = join(directory, "source.tar.gz"); - const extracted = join(directory, "extracted"); - try { - writeFileSync(archivePath, archive); - execFileSync("mkdir", [extracted]); - execFileSync("tar", ["-xzf", archivePath, "-C", extracted]); - assertion(extracted); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -} - -function git(cwd, args) { - return execFileSync("git", args, { cwd, encoding: "utf8" }); -} +test("rejects a canary URL from another origin", () => { + assert.throws( + () => packageRevision(`https://example.com/${revision}/eve.tgz`), + /resolved outside/u, + ); +}); diff --git a/apps/benchmarks/lib/timing.test.mjs b/apps/benchmarks/lib/timing.test.mjs deleted file mode 100644 index adf42aae2..000000000 --- a/apps/benchmarks/lib/timing.test.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; - -import { BenchmarkTimings } from "./timing.ts"; - -test("records successful and failed phases", async () => { - const timings = new BenchmarkTimings(); - await timings.measure("success", async () => undefined, { turns: 1 }); - await assert.rejects(timings.measure("failure", async () => Promise.reject(new Error("nope")))); - - assert.equal(timings.entries.length, 2); - assert.deepEqual( - timings.entries.map(({ phase, outcome, details }) => ({ phase, outcome, details })), - [ - { phase: "success", outcome: "success", details: { turns: 1 } }, - { phase: "failure", outcome: "failure", details: undefined }, - ], - ); - assert.ok(timings.entries.every((entry) => entry.durationMs >= 0)); -}); diff --git a/apps/benchmarks/lib/timing.ts b/apps/benchmarks/lib/timing.ts deleted file mode 100644 index 15e1c0453..000000000 --- a/apps/benchmarks/lib/timing.ts +++ /dev/null @@ -1,55 +0,0 @@ -export interface BenchmarkTiming { - readonly phase: string; - readonly startedAt: string; - readonly durationMs: number; - readonly outcome: "success" | "failure"; - readonly details?: Readonly>; -} - -type TimingDetails = BenchmarkTiming["details"]; - -export class BenchmarkTimings { - readonly entries: BenchmarkTiming[] = []; - - async measure( - phase: string, - operation: () => Promise, - details?: TimingDetails, - ): Promise { - const startedAt = new Date().toISOString(); - const started = performance.now(); - try { - const result = await operation(); - this.add(phase, startedAt, performance.now() - started, "success", details); - return result; - } catch (error) { - this.add(phase, startedAt, performance.now() - started, "failure", details); - throw error; - } - } - - record( - phase: string, - durationMs: number, - outcome: BenchmarkTiming["outcome"] = "success", - details?: TimingDetails, - ): void { - this.add(phase, new Date(Date.now() - durationMs).toISOString(), durationMs, outcome, details); - } - - private add( - phase: string, - startedAt: string, - durationMs: number, - outcome: BenchmarkTiming["outcome"], - details: TimingDetails, - ): void { - const entry = { - phase, - startedAt, - durationMs: Math.round(durationMs), - outcome, - }; - this.entries.push(details === undefined ? entry : { ...entry, details }); - } -} diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.json index 348170ae9..e6852a193 100644 --- a/apps/benchmarks/package.json +++ b/apps/benchmarks/package.json @@ -12,12 +12,7 @@ "benchmark:timings": "node scripts/timings.mjs" }, "dependencies": { - "@ai-sdk/harness": "1.0.87", - "@ai-sdk/harness-claude-code": "1.0.90", - "@ai-sdk/harness-opencode": "1.0.88", - "@ai-sdk/sandbox-vercel": "1.0.87", - "@vercel/agent-eval": "1.5.0", - "@vercel/sandbox": "3.2.0", + "@vercel/agent-eval": "2.2.1", "jiti": "2.7.0", "zod": "catalog:" }, diff --git a/apps/benchmarks/publish.mjs b/apps/benchmarks/publish.mjs index 3d3905863..da865d9cb 100644 --- a/apps/benchmarks/publish.mjs +++ b/apps/benchmarks/publish.mjs @@ -12,13 +12,8 @@ import { publishedBenchmarkModels, publishedExperimentId, } from "./lib/benchmark-config.ts"; -import { - prepareFixtures, - resetExperiments, - writeExperiment, - writeSubjectArchives, -} from "./lib/experiment-files.mjs"; -import { revisionSubject } from "./lib/source.mjs"; +import { prepareFixtures, resetExperiments, writeExperiment } from "./lib/experiment-files.mjs"; +import { canarySubject } from "./lib/source.mjs"; const appRoot = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(appRoot, "../.."); @@ -51,14 +46,15 @@ if (values["allow-dirty"]) { ); } -const revision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim(); -const subject = revisionSubject(repositoryRoot, revision, "published"); +const requestedRevision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim(); +const subject = canarySubject(requestedRevision, "published"); +const revision = subject.revision; const benchmarks = selectedBenchmarks(values.models); const experimentNames = benchmarks.flatMap((benchmark) => authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)), ); -prepareFixtures(evalsRoot); +await prepareFixtures(evalsRoot, subject, publishedBenchmark.caseIds); writeExperiments(subject, revision, benchmarks); console.log(`> eve revision: ${revision}`); @@ -158,19 +154,11 @@ To inspect pending benchmark cells without publishing: function writeExperiments(subject, revision, benchmarks) { resetExperiments(experimentsRoot); - const { archiveName, dependencyArchiveName } = writeSubjectArchives( - experimentsRoot, - subject, - `published-${revision.slice(0, 12)}`, - ); - for (const benchmark of benchmarks) { for (const treatment of authoringTreatments) { writeExperiment(experimentsRoot, publishedExperimentId(benchmark, treatment), { - archiveName, - dependencyArchiveName, - digest: subject.digest, - dependencyDigest: subject.dependencyDigest, + revision: subject.revision, + packageSpec: subject.packageSpec, runs: publishedBenchmark.runs, evals: publishedBenchmark.caseIds, benchmark, diff --git a/apps/benchmarks/run.mjs b/apps/benchmarks/run.mjs index d12adbb1a..32553b65d 100644 --- a/apps/benchmarks/run.mjs +++ b/apps/benchmarks/run.mjs @@ -6,14 +6,13 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; -import { findBenchmarkModel, parseAuthoringTreatment } from "./lib/benchmark-config.ts"; import { - prepareFixtures, - resetExperiments, - writeExperiment, - writeSubjectArchives, -} from "./lib/experiment-files.mjs"; -import { revisionSubject, workingTreeSubject } from "./lib/source.mjs"; + findBenchmarkModel, + parseAuthoringTreatment, + publishedBenchmark, +} from "./lib/benchmark-config.ts"; +import { prepareFixtures, resetExperiments, writeExperiment } from "./lib/experiment-files.mjs"; +import { canarySubject } from "./lib/source.mjs"; const appRoot = dirname(fileURLToPath(import.meta.url)); const repositoryRoot = resolve(appRoot, "../.."); @@ -23,8 +22,7 @@ const { values, positionals } = parseArgs({ args: process.argv.slice(2), allowPositionals: true, options: { - base: { type: "string" }, - head: { type: "string" }, + canary: { type: "string", default: "main" }, dry: { type: "boolean" }, runs: { type: "string" }, model: { type: "string", default: "claude-sonnet-5" }, @@ -44,32 +42,23 @@ if (values.help) { process.exit(0); } if (positionals.length > 1) throw new Error("Expected at most one ."); -if (values.head !== undefined && values.base === undefined) { - throw new Error("--head requires --base."); -} const runs = parseRuns(values.runs); const selectedEval = positionals[0]; const treatment = parseAuthoringTreatment(values.treatment); const benchmark = findBenchmarkModel(values.model); -if (values.verbose && (selectedEval === undefined || runs !== 1 || values.base !== undefined)) { - throw new Error("--verbose requires one eval, one run, and no revision comparison."); +if (values.verbose && (selectedEval === undefined || runs !== 1)) { + throw new Error("--verbose requires one eval and one run."); } if (selectedEval !== undefined && !existsSync(join(evalsRoot, selectedEval, "CASE.ts"))) { throw new Error(`Unknown eval ${JSON.stringify(selectedEval)}.`); } -const workingTree = () => workingTreeSubject(repositoryRoot); -const subjects = - values.base === undefined - ? [workingTree()] - : [ - revisionSubject(repositoryRoot, values.base, "base"), - values.head === undefined - ? { ...workingTree(), label: "head" } - : revisionSubject(repositoryRoot, values.head, "head"), - ]; - -prepareFixtures(evalsRoot, selectedEval === undefined ? undefined : [selectedEval]); +const subjects = [canarySubject(values.canary, "current")]; +await prepareFixtures( + evalsRoot, + subjects[0], + selectedEval === undefined ? publishedBenchmark.caseIds : [selectedEval], +); mkdirSync(join(appRoot, "results"), { recursive: true }); writeExperiments(subjects, runs, benchmark, treatment, values.verbose ?? false); const executable = join(appRoot, "node_modules/.bin/agent-eval"); @@ -82,7 +71,8 @@ for (const subject of subjects) console.log(`> ${subject.label}: ${subject.descr const result = spawnSync(executable, args, { cwd: appRoot, stdio: "inherit", - env: { ...process.env, EVE_BENCHMARK_EVAL: selectedEval ?? "*" }, + env: + selectedEval === undefined ? process.env : { ...process.env, EVE_BENCHMARK_EVAL: selectedEval }, }); if (result.error) throw result.error; process.exit(result.status ?? 1); @@ -90,16 +80,9 @@ process.exit(result.status ?? 1); function writeExperiments(subjects, runs, benchmark, treatment, verbose) { resetExperiments(experimentsRoot); for (const subject of subjects) { - const { archiveName, dependencyArchiveName } = writeSubjectArchives( - experimentsRoot, - subject, - subject.label, - ); writeExperiment(experimentsRoot, subject.label, { - archiveName, - dependencyArchiveName, - digest: subject.digest, - dependencyDigest: subject.dependencyDigest, + revision: subject.revision, + packageSpec: subject.packageSpec, runs, benchmark, treatment, @@ -118,6 +101,5 @@ function parseRuns(value) { function usage() { console.log(`Usage: - pnpm benchmark [eval-name] [--model ] [--runs N] [--treatment baseline|guided] [--dry] [--verbose] [--keep-failures] - pnpm benchmark [eval-name] --base [--head ] [--model ] [--runs N] [--treatment baseline|guided] [--dry]`); + pnpm benchmark [eval-name] [--canary main] [--model ] [--runs N] [--treatment baseline|guided] [--dry] [--verbose] [--keep-failures]`); } diff --git a/apps/benchmarks/scripts/cost.mjs b/apps/benchmarks/scripts/cost.mjs index 3a4e840c3..f00b00fde 100644 --- a/apps/benchmarks/scripts/cost.mjs +++ b/apps/benchmarks/scripts/cost.mjs @@ -86,20 +86,19 @@ export const modelPricing = { const number = (value) => (typeof value === "number" && Number.isFinite(value) ? value : 0); const object = (value) => (value !== null && typeof value === "object" ? value : {}); -export function extractRunUsage(raw) { +export function extractRunUsage(raw, harness) { const usage = { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 }; let found = false; for (const line of raw.split("\n")) { try { const event = JSON.parse(line); - if (event.type !== "assistant") continue; - const value = object(object(event.message).usage); - if (Object.keys(value).length === 0) continue; - usage.input += number(value.inputTokens); - usage.output += number(value.outputTokens); - usage.reasoning += number(value.reasoningTokens); - usage.cacheRead += number(value.cachedInputTokens); - usage.cacheWrite += number(value.cacheWriteTokens); + const value = usageForEvent(event, harness); + if (value === undefined) continue; + usage.input += value.input; + usage.output += value.output; + usage.reasoning += value.reasoning; + usage.cacheRead += value.cacheRead; + usage.cacheWrite += value.cacheWrite; found = true; } catch { // Ignore malformed transcript lines. @@ -108,6 +107,59 @@ export function extractRunUsage(raw) { return found && tokenConsumption(usage) > 0 ? usage : null; } +function usageForEvent(event, harness) { + const legacy = object(object(event.message).usage); + if (event.type === "assistant" && "inputTokens" in legacy) { + return { + input: number(legacy.inputTokens), + output: number(legacy.outputTokens), + reasoning: number(legacy.reasoningTokens), + cacheRead: number(legacy.cachedInputTokens), + cacheWrite: number(legacy.cacheWriteTokens), + }; + } + + if (harness === "OpenCode" && event.type === "step_finish") { + const tokens = object(object(event.part).tokens); + const cache = object(tokens.cache); + return { + input: number(tokens.input), + output: number(tokens.output), + reasoning: number(tokens.reasoning), + cacheRead: number(cache.read), + cacheWrite: number(cache.write), + }; + } + + if (harness === "Claude Code" && event.type === "assistant") { + const usage = object(object(event.message).usage); + if (Object.keys(usage).length === 0) return undefined; + return { + input: number(usage.input_tokens), + output: number(usage.output_tokens), + reasoning: number(usage.reasoning_tokens), + cacheRead: number(usage.cache_read_input_tokens), + cacheWrite: number(usage.cache_creation_input_tokens), + }; + } + + if (harness === "Codex" && event.type === "turn.completed") { + const usage = object(event.usage); + const cacheRead = number(usage.cached_input_tokens); + return { + // Codex reports total input tokens. Cache reads are a subset, unlike the + // other runners which report non-cached input separately. + input: Math.max(0, number(usage.input_tokens) - cacheRead), + output: number(usage.output_tokens), + reasoning: number(usage.reasoning_output_tokens ?? usage.reasoning_tokens), + cacheRead, + cacheWrite: number(usage.cache_write_input_tokens ?? usage.cache_creation_input_tokens), + }; + } + + return undefined; +} + export function tokenConsumption(usage) { return usage.input + usage.output + usage.reasoning; } @@ -117,10 +169,18 @@ export function countToolInvocations(raw) { for (const line of raw.split("\n")) { try { const event = JSON.parse(line); + if (event.type === "tool_use") { + count++; + continue; + } + if (event.type === "item.started" && object(event.item).type === "command_execution") { + count++; + continue; + } if (event.type !== "assistant") continue; const content = object(event.message).content; - if (!Array.isArray(content)) continue; - count += content.filter((part) => object(part).type === "tool_use").length; + if (Array.isArray(content)) + count += content.filter((part) => object(part).type === "tool_use").length; } catch { // Ignore malformed transcript lines. } diff --git a/apps/benchmarks/scripts/cost.test.mjs b/apps/benchmarks/scripts/cost.test.mjs index 2350bbfa1..d6371f803 100644 --- a/apps/benchmarks/scripts/cost.test.mjs +++ b/apps/benchmarks/scripts/cost.test.mjs @@ -32,6 +32,57 @@ test("extracts usage from authoring harness transcripts", () => { }); }); +test("extracts OpenCode step usage", () => { + const usage = extractRunUsage( + JSON.stringify({ + type: "step_finish", + part: { + tokens: { input: 100, output: 20, reasoning: 5, cache: { read: 50, write: 10 } }, + }, + }), + "OpenCode", + ); + + assert.deepEqual(usage, { input: 100, output: 20, reasoning: 5, cacheRead: 50, cacheWrite: 10 }); +}); + +test("extracts Claude Code message usage", () => { + const usage = extractRunUsage( + JSON.stringify({ + type: "assistant", + message: { + usage: { + input_tokens: 100, + output_tokens: 20, + cache_read_input_tokens: 50, + cache_creation_input_tokens: 10, + }, + }, + }), + "Claude Code", + ); + + assert.deepEqual(usage, { input: 100, output: 20, reasoning: 0, cacheRead: 50, cacheWrite: 10 }); +}); + +test("extracts Codex completion usage without double-counting cached input", () => { + const usage = extractRunUsage( + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 150, + cached_input_tokens: 50, + cache_write_input_tokens: 10, + output_tokens: 20, + reasoning_output_tokens: 5, + }, + }), + "Codex", + ); + + assert.deepEqual(usage, { input: 100, output: 20, reasoning: 5, cacheRead: 50, cacheWrite: 10 }); +}); + test("calculates token consumption without double-counting cache details", () => { assert.equal( tokenConsumption({ @@ -45,6 +96,15 @@ test("calculates token consumption without double-counting cache details", () => ); }); +test("counts native OpenCode and Codex tool invocations", () => { + const raw = [ + JSON.stringify({ type: "tool_use", part: { tool: "bash" } }), + JSON.stringify({ type: "item.started", item: { type: "command_execution" } }), + ].join("\n"); + + assert.equal(countToolInvocations(raw), 2); +}); + test("counts tool invocations in authoring harness transcripts", () => { const raw = [ JSON.stringify({ type: "user", message: { role: "user", content: "Hi" } }), diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjs index bf1e4f505..d0aa7df3b 100644 --- a/apps/benchmarks/scripts/export-results.mjs +++ b/apps/benchmarks/scripts/export-results.mjs @@ -81,7 +81,7 @@ for (const benchmark of benchmarks) { caseId, status: status ?? "current", ...result, - ...meanRunMetrics(summaryPath, benchmark.model), + ...meanRunMetrics(summaryPath, benchmark.model, benchmark.harness), }); } } @@ -227,8 +227,8 @@ function latestValidResult(experimentId, caseId) { return undefined; } -function meanRunMetrics(summaryPath, model) { - const performanceRuns = runMetrics(summaryPath); +function meanRunMetrics(summaryPath, model, harness) { + const performanceRuns = runMetrics(summaryPath, harness); const performanceUsage = performanceRuns.flatMap((run) => run.usage === null ? [] : [run.usage], ); @@ -248,14 +248,14 @@ function meanRunMetrics(summaryPath, model) { return result; } -function runMetrics(summaryPath) { +function runMetrics(summaryPath, harness) { return readdirSync(dirname(summaryPath), { withFileTypes: true }) .filter((entry) => entry.isDirectory() && /^run-\d+$/u.test(entry.name)) .flatMap((entry) => { const transcriptPath = join(dirname(summaryPath), entry.name, "transcript-raw.jsonl"); if (!existsSync(transcriptPath)) return []; const raw = readFileSync(transcriptPath, "utf8"); - return [{ usage: extractRunUsage(raw), toolInvocations: countToolInvocations(raw) }]; + return [{ usage: extractRunUsage(raw, harness), toolInvocations: countToolInvocations(raw) }]; }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6aa95a4e7..00e643b13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -138,24 +138,9 @@ importers: apps/benchmarks: dependencies: - '@ai-sdk/harness': - specifier: 1.0.87 - version: 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4) - '@ai-sdk/harness-claude-code': - specifier: 1.0.90 - version: 1.0.90(bufferutil@4.1.0)(zod@4.5.4) - '@ai-sdk/harness-opencode': - specifier: 1.0.88 - version: 1.0.88(bufferutil@4.1.0)(zod@4.5.4) - '@ai-sdk/sandbox-vercel': - specifier: 1.0.87 - version: 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4) '@vercel/agent-eval': - specifier: 1.5.0 - version: 1.5.0(supports-color@10.2.2) - '@vercel/sandbox': - specifier: 3.2.0 - version: 3.2.0 + specifier: 2.2.1 + version: 2.2.1(supports-color@10.2.2) jiti: specifier: 2.7.0 version: 2.7.0 @@ -1670,28 +1655,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/harness-claude-code@1.0.90': - resolution: {integrity: sha512-SL77y5l3596n6cxuLN4OB19iTkvwg6zJq2KxIV9MAPwAMYEEXHz7YGRXxrAd7GOixIVqcsXEw1HOLN1zcG1L3A==} - engines: {node: '>=22'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - - '@ai-sdk/harness-opencode@1.0.88': - resolution: {integrity: sha512-bA22/rBaTbsO/o/x91P78RvLpvgwnCoW6Ie7ATsC7Z6BxGYZpFrS6BGWD1DQL2pN3jy61KWnDLDwy0egg7ypZg==} - engines: {node: '>=22'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - - '@ai-sdk/harness@1.0.87': - resolution: {integrity: sha512-OY+M1B58lLv+fRHB0dh/FJ6lCzmg9HQmTe1z+XS1WTPF8PnTMEr2E/6vNTUoORN7iZMkTyVkszA4DOhod1eFQw==} - engines: {node: '>=22'} - peerDependencies: - ws: ^8.21.0 - zod: ^3.25.76 || ^4.1.8 - peerDependenciesMeta: - ws: - optional: true - '@ai-sdk/mcp@2.0.29': resolution: {integrity: sha512-cK8mYpjKGp7/gmsTYtq/dkdmNlBeWQrwcfsah/W9hN6Vr1yNzunLnAZAqL5tMzumLDBueSz6y+vrb5z8DJ8TMg==} engines: {node: '>=22'} @@ -1794,10 +1757,6 @@ packages: peerDependencies: react: ^18 || ~19.0.1 || ~19.1.2 || ^19.2.1 - '@ai-sdk/sandbox-vercel@1.0.87': - resolution: {integrity: sha512-rW8mtwvH5lcoXWPn8hEdh2Kns/2qAQokS9xG/UeqZOgp79ifKJQp8dvWpUSzRNZKgPtqsXNmVFac6J/UnlmO+Q==} - engines: {node: '>=22'} - '@ai-sdk/togetherai@1.0.49': resolution: {integrity: sha512-g4BpEatN7flh3GZ0CN9KvAUX6uLPmIqGSrKKFvAmC3HZdnF940zl+ChXs3atdbtpr6+cwirxM5RACbUzr0uYhA==} engines: {node: '>=18'} @@ -7846,8 +7805,8 @@ packages: peerDependencies: chat: ^4.0.0 - '@vercel/agent-eval@1.5.0': - resolution: {integrity: sha512-evbgSkCv6CoQxgIr6dfjh6/3K54NxXoykZN5stXS97kTVm4NuU2y7su1WR71Y55tR4l0xgiUTsVp+BlLVh6zZw==} + '@vercel/agent-eval@2.2.1': + resolution: {integrity: sha512-TVowf/Q60kw8anu4oAIhb1fc4y8k4jhwjCPwdfNoy9m9W1LHBHZYaIi81QZ+7w2S77hvBi4CAoOnohovjA2Mow==} engines: {node: '>=18.0.0'} '@vercel/agent-readability@0.6.0': @@ -15764,18 +15723,6 @@ packages: utf-8-validate: optional: true - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.21.3: resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} engines: {node: '>=10.0.0'} @@ -16098,44 +16045,6 @@ snapshots: zod: 4.4.3 optional: true - '@ai-sdk/harness-claude-code@1.0.90(bufferutil@4.1.0)(zod@4.5.4)': - dependencies: - '@ai-sdk/harness': 1.0.87(ws@8.21.3(bufferutil@4.1.0))(zod@4.5.4) - '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4) - ws: 8.21.3(bufferutil@4.1.0) - zod: 4.5.4 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@ai-sdk/harness-opencode@1.0.88(bufferutil@4.1.0)(zod@4.5.4)': - dependencies: - '@ai-sdk/harness': 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4) - '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4) - ws: 8.21.0(bufferutil@4.1.0) - zod: 4.5.4 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@ai-sdk/harness@1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)': - dependencies: - '@ai-sdk/provider': 4.0.8 - '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4) - ai: 7.0.79(zod@4.5.4) - zod: 4.5.4 - optionalDependencies: - ws: 8.21.0(bufferutil@4.1.0) - - '@ai-sdk/harness@1.0.87(ws@8.21.3(bufferutil@4.1.0))(zod@4.5.4)': - dependencies: - '@ai-sdk/provider': 4.0.8 - '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4) - ai: 7.0.79(zod@4.5.4) - zod: 4.5.4 - optionalDependencies: - ws: 8.21.3(bufferutil@4.1.0) - '@ai-sdk/mcp@2.0.29(zod@4.5.4)': dependencies: '@ai-sdk/provider': 4.0.7 @@ -16270,17 +16179,6 @@ snapshots: transitivePeerDependencies: - zod - '@ai-sdk/sandbox-vercel@1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4)': - dependencies: - '@ai-sdk/harness': 1.0.87(ws@8.21.0(bufferutil@4.1.0))(zod@4.5.4) - '@ai-sdk/provider-utils': 5.0.30(zod@4.5.4) - '@vercel/sandbox': 3.2.0 - transitivePeerDependencies: - - bare-abort-controller - - react-native-b4a - - ws - - zod - '@ai-sdk/togetherai@1.0.49(zod@4.4.3)': dependencies: '@ai-sdk/openai-compatible': 1.0.46(zod@4.4.3) @@ -22322,7 +22220,7 @@ snapshots: - supports-color - zod - '@vercel/agent-eval@1.5.0(supports-color@10.2.2)': + '@vercel/agent-eval@2.2.1(supports-color@10.2.2)': dependencies: '@ai-sdk/anthropic': 1.2.12(zod@3.25.76) '@vercel/sandbox': 1.10.2 @@ -33064,10 +32962,6 @@ snapshots: optionalDependencies: bufferutil: 4.1.0 - ws@8.21.0(bufferutil@4.1.0): - optionalDependencies: - bufferutil: 4.1.0 - ws@8.21.3(bufferutil@4.1.0): optionalDependencies: bufferutil: 4.1.0