mirror of
https://github.com/vercel/eve.git
synced 2026-09-20 05:35:39 +08:00
perf: profile authoring benchmark setup (#2219)
Signed-off-by: owenkephart <owen.kephart@vercel.com>
This commit is contained in:
OwenKephart
@@ -35,15 +35,26 @@ pnpm benchmark author-000-imessage \
|
||||
```
|
||||
|
||||
The runner archives each subject locally and uploads it to the sandbox. Revisions and local-only
|
||||
commits do not need to be pushed. Dependency downloads are cached by lockfile, so source-only
|
||||
changes reuse the prepared pnpm store. For one eval and one run, `--verbose` streams setup
|
||||
phases, assistant text, tool calls, grading, and build progress.
|
||||
commits do not need to be pushed. It maintains two persistent snapshot layers: a dependency
|
||||
snapshot keyed by package-manager inputs, and a subject snapshot keyed by the source tree,
|
||||
starting point, setup IDs, and bootstrap version. Source-only changes reuse the dependency snapshot
|
||||
but create a new subject snapshot. For one eval and one run, `--verbose` streams setup phases,
|
||||
assistant text, tool calls, grading, and build progress.
|
||||
|
||||
Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated
|
||||
by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.
|
||||
|
||||
Results are written under `apps/benchmarks/results/`. Each run includes the transcript,
|
||||
grader output, summary, and copied project files. Vercel Sandbox and AI Gateway credentials are
|
||||
grader output, summary, copied project files, and `project/benchmark/timings.json`. The timing
|
||||
artifact records the snapshot-cache outcome, source installation and build phases, workspace setup,
|
||||
each user turn with token and tool-call counts, and grading and validation durations. Use it to
|
||||
separate sandbox setup time from agent time when comparing runs. Print a compact local report with:
|
||||
|
||||
```sh
|
||||
pnpm benchmark:timings results/current/<timestamp>/<case>/run-1
|
||||
```
|
||||
|
||||
Pass `--json` to print the original timing artifact. Vercel Sandbox and AI Gateway credentials are
|
||||
required.
|
||||
|
||||
## Publish canonical results
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { dependencySnapshotId } from "./dependency-snapshot.ts";
|
||||
|
||||
test("reuses a dependency sandbox's current snapshot", async () => {
|
||||
const snapshotId = await dependencySnapshotId({
|
||||
currentSnapshotId: "snapshot-current",
|
||||
snapshot: async () => {
|
||||
throw new Error("snapshot should not be called");
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(snapshotId, "snapshot-current");
|
||||
});
|
||||
|
||||
test("explicitly snapshots a dependency sandbox without a current snapshot", async () => {
|
||||
let expiration;
|
||||
const snapshotId = await dependencySnapshotId({
|
||||
currentSnapshotId: undefined,
|
||||
snapshot: async (options) => {
|
||||
expiration = options?.expiration;
|
||||
return { snapshotId: "snapshot-created" };
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(snapshotId, "snapshot-created");
|
||||
assert.equal(expiration, 0);
|
||||
});
|
||||
@@ -3,16 +3,19 @@ import { createVercelSandbox } from "@ai-sdk/sandbox-vercel";
|
||||
import { Sandbox } from "@vercel/sandbox";
|
||||
|
||||
import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";
|
||||
import type { BenchmarkTimings } from "./timing.js";
|
||||
|
||||
const dependencySnapshots = new Map<string, Promise<string>>();
|
||||
const subjectSnapshots = new Map<string, Promise<string>>();
|
||||
|
||||
export function createDependencyCachedSandbox(options: {
|
||||
readonly archive: Uint8Array;
|
||||
readonly dependencyArchive: Uint8Array;
|
||||
readonly dependencyDigest: string;
|
||||
readonly ports: ReadonlyArray<number>;
|
||||
readonly env: Readonly<Record<string, string>>;
|
||||
readonly log: (message: string) => void;
|
||||
readonly timings: BenchmarkTimings;
|
||||
}): HarnessV1SandboxProvider {
|
||||
const sessionProvider = (snapshotId: string) =>
|
||||
createVercelSandbox({
|
||||
@@ -29,19 +32,22 @@ export function createDependencyCachedSandbox(options: {
|
||||
async createSession(request = {}) {
|
||||
options.log("[setup] preparing dependency cache");
|
||||
const dependencies = await dependencySnapshot(
|
||||
options.archive,
|
||||
options.dependencyArchive,
|
||||
options.dependencyDigest,
|
||||
options.log,
|
||||
options.timings,
|
||||
);
|
||||
if (request.identity === undefined || request.onFirstCreate === undefined) {
|
||||
return sessionProvider(dependencies).createSession(request);
|
||||
}
|
||||
const subject = await subjectSnapshot(
|
||||
dependencies,
|
||||
options.archive,
|
||||
request.identity,
|
||||
options.env,
|
||||
request.onFirstCreate,
|
||||
request.abortSignal,
|
||||
options.timings,
|
||||
);
|
||||
return sessionProvider(subject).createSession({
|
||||
sessionId: request.sessionId,
|
||||
@@ -50,9 +56,10 @@ export function createDependencyCachedSandbox(options: {
|
||||
},
|
||||
async resumeSession(request) {
|
||||
const dependencies = await dependencySnapshot(
|
||||
options.archive,
|
||||
options.dependencyArchive,
|
||||
options.dependencyDigest,
|
||||
options.log,
|
||||
options.timings,
|
||||
);
|
||||
const provider = sessionProvider(dependencies);
|
||||
if (provider.resumeSession === undefined) {
|
||||
@@ -67,11 +74,15 @@ function dependencySnapshot(
|
||||
archive: Uint8Array,
|
||||
digest: string,
|
||||
log: (message: string) => void,
|
||||
timings: BenchmarkTimings,
|
||||
): Promise<string> {
|
||||
const name = `eve-benchmark-dependencies-v4-${digest.slice(0, 24)}`;
|
||||
const name = `eve-benchmark-dependencies-v5-${digest.slice(0, 24)}`;
|
||||
let snapshot = dependencySnapshots.get(name);
|
||||
if (snapshot !== undefined) return snapshot;
|
||||
snapshot = createDependencySnapshot(name, archive, log);
|
||||
if (snapshot !== undefined) {
|
||||
timings.record("dependency-snapshot.memory-cache", 0, "success", { name });
|
||||
return snapshot;
|
||||
}
|
||||
snapshot = createDependencySnapshot(name, archive, log, timings);
|
||||
dependencySnapshots.set(name, snapshot);
|
||||
return snapshot;
|
||||
}
|
||||
@@ -80,49 +91,70 @@ async function createDependencySnapshot(
|
||||
name: string,
|
||||
archive: Uint8Array,
|
||||
log: (message: string) => void,
|
||||
timings: BenchmarkTimings,
|
||||
): Promise<string> {
|
||||
let created = false;
|
||||
const sandbox = await Sandbox.getOrCreate({
|
||||
name,
|
||||
runtime: "node24",
|
||||
timeout: 15 * 60_000,
|
||||
persistent: true,
|
||||
snapshotExpiration: 0,
|
||||
networkPolicy: "allow-all",
|
||||
async onCreate(current) {
|
||||
created = true;
|
||||
log("[setup] fetching workspace dependencies");
|
||||
await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
|
||||
const command = await current.runCommand("bash", [
|
||||
"-lc",
|
||||
`mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global pnpm@11.15.0 vitest@4.1.10 && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
|
||||
]);
|
||||
if (command.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
|
||||
return stopWithSnapshot(sandbox, "Dependency");
|
||||
const sandbox = await timings.measure("dependency-snapshot.get-or-create", () =>
|
||||
Sandbox.getOrCreate({
|
||||
name,
|
||||
runtime: "node24",
|
||||
timeout: 15 * 60_000,
|
||||
persistent: true,
|
||||
snapshotExpiration: 0,
|
||||
networkPolicy: "allow-all",
|
||||
async onCreate(current) {
|
||||
created = true;
|
||||
log("[setup] fetching workspace dependencies");
|
||||
await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
|
||||
const command = await current.runCommand("bash", [
|
||||
"-lc",
|
||||
`mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global pnpm@11.15.0 vitest@4.1.10 && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
|
||||
]);
|
||||
if (command.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (!created && sandbox.currentSnapshotId !== undefined) {
|
||||
timings.record("dependency-snapshot.reused", 0, "success", { name });
|
||||
return sandbox.currentSnapshotId;
|
||||
}
|
||||
return timings.measure("dependency-snapshot.publish", () =>
|
||||
stopWithSnapshot(sandbox, "Dependency"),
|
||||
);
|
||||
}
|
||||
|
||||
// Publish bootstrap mutations explicitly; a layered Vercel template can expose
|
||||
// its inherited snapshot ID before those mutations receive a new snapshot.
|
||||
function subjectSnapshot(
|
||||
dependencySnapshotId: string,
|
||||
archive: Uint8Array,
|
||||
identity: string,
|
||||
env: Readonly<Record<string, string>>,
|
||||
bootstrap: NonNullable<
|
||||
NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
|
||||
>,
|
||||
abortSignal?: AbortSignal,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
timings: BenchmarkTimings,
|
||||
): Promise<string> {
|
||||
const name = `eve-benchmark-subject-${identity}`;
|
||||
let snapshot = subjectSnapshots.get(name);
|
||||
if (snapshot !== undefined) return snapshot;
|
||||
snapshot = createSubjectSnapshot(name, dependencySnapshotId, env, bootstrap, abortSignal);
|
||||
if (snapshot !== undefined) {
|
||||
timings.record("subject-snapshot.memory-cache", 0, "success", { name });
|
||||
return snapshot;
|
||||
}
|
||||
snapshot = createSubjectSnapshot(
|
||||
name,
|
||||
dependencySnapshotId,
|
||||
archive,
|
||||
env,
|
||||
bootstrap,
|
||||
abortSignal,
|
||||
timings,
|
||||
);
|
||||
subjectSnapshots.set(name, snapshot);
|
||||
return snapshot;
|
||||
}
|
||||
@@ -130,31 +162,41 @@ function subjectSnapshot(
|
||||
async function createSubjectSnapshot(
|
||||
name: string,
|
||||
dependencySnapshotId: string,
|
||||
archive: Uint8Array,
|
||||
env: Readonly<Record<string, string>>,
|
||||
bootstrap: NonNullable<
|
||||
NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
|
||||
>,
|
||||
abortSignal?: AbortSignal,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
timings: BenchmarkTimings,
|
||||
): Promise<string> {
|
||||
let created = false;
|
||||
const sandbox = await Sandbox.getOrCreate({
|
||||
name,
|
||||
source: { type: "snapshot", snapshotId: dependencySnapshotId },
|
||||
timeout: 15 * 60_000,
|
||||
env: { ...env },
|
||||
persistent: true,
|
||||
snapshotExpiration: 0,
|
||||
networkPolicy: "allow-all",
|
||||
signal: abortSignal,
|
||||
async onCreate(current) {
|
||||
created = true;
|
||||
const provider = createVercelSandbox({ sandbox: current });
|
||||
const session = await provider.createSession({ abortSignal });
|
||||
await bootstrap(session.restricted(), { abortSignal });
|
||||
},
|
||||
});
|
||||
if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
|
||||
return stopWithSnapshot(sandbox, "Subject", abortSignal);
|
||||
const sandbox = await timings.measure("subject-snapshot.get-or-create", () =>
|
||||
Sandbox.getOrCreate({
|
||||
name,
|
||||
source: { type: "snapshot", snapshotId: dependencySnapshotId },
|
||||
timeout: 15 * 60_000,
|
||||
env: { ...env },
|
||||
persistent: true,
|
||||
snapshotExpiration: 0,
|
||||
networkPolicy: "allow-all",
|
||||
signal: abortSignal,
|
||||
async onCreate(current) {
|
||||
created = true;
|
||||
await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
|
||||
const provider = createVercelSandbox({ sandbox: current });
|
||||
const session = await provider.createSession({ abortSignal });
|
||||
await bootstrap(session.restricted(), { abortSignal });
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (!created && sandbox.currentSnapshotId !== undefined) {
|
||||
timings.record("subject-snapshot.reused", 0, "success", { name });
|
||||
return sandbox.currentSnapshotId;
|
||||
}
|
||||
return timings.measure("subject-snapshot.publish", () =>
|
||||
stopWithSnapshot(sandbox, "Subject", abortSignal),
|
||||
);
|
||||
}
|
||||
|
||||
async function stopWithSnapshot(
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
import type { Sandbox } from "@vercel/sandbox";
|
||||
|
||||
export async function dependencySnapshotId(
|
||||
sandbox: Pick<Sandbox, "currentSnapshotId" | "snapshot">,
|
||||
): Promise<string> {
|
||||
if (sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
|
||||
return (await sandbox.snapshot({ expiration: 0 })).snapshotId;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
export function fixtureNames(evalsRoot) {
|
||||
return readdirSync(evalsRoot, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
}
|
||||
|
||||
export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) {
|
||||
for (const name of names) {
|
||||
const fixtureRoot = join(evalsRoot, name);
|
||||
writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
|
||||
writeFileSync(
|
||||
join(fixtureRoot, "package.json"),
|
||||
`${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function resetExperiments(experimentsRoot) {
|
||||
rmSync(experimentsRoot, { recursive: true, force: true });
|
||||
mkdirSync(experimentsRoot, { recursive: true });
|
||||
}
|
||||
|
||||
export function writeSubjectArchives(experimentsRoot, subject, name) {
|
||||
const archiveName = `${name}.source.tar.gz`;
|
||||
const dependencyArchiveName = `${name}.dependencies.tar.gz`;
|
||||
writeFileSync(join(experimentsRoot, archiveName), subject.archive);
|
||||
writeFileSync(join(experimentsRoot, dependencyArchiveName), subject.dependencyArchive);
|
||||
return { archiveName, dependencyArchiveName };
|
||||
}
|
||||
|
||||
export function writeExperiment(experimentsRoot, name, options) {
|
||||
writeFileSync(
|
||||
join(experimentsRoot, `${name}.ts`),
|
||||
`import { readFileSync } from "node:fs";\n` +
|
||||
`import { authoringExperiment } from "../lib/experiment.js";\n\n` +
|
||||
`export default authoringExperiment({\n` +
|
||||
` archive: readFileSync(new URL(${JSON.stringify(`./${options.archiveName}`)}, import.meta.url)),\n` +
|
||||
` dependencyArchive: readFileSync(new URL(${JSON.stringify(`./${options.dependencyArchiveName}`)}, import.meta.url)),\n` +
|
||||
` digest: ${JSON.stringify(options.digest)},\n` +
|
||||
` dependencyDigest: ${JSON.stringify(options.dependencyDigest)},\n` +
|
||||
` runs: ${options.runs},\n` +
|
||||
(options.evals === undefined ? "" : ` evals: ${JSON.stringify(options.evals)},\n`) +
|
||||
` benchmark: ${JSON.stringify(options.benchmark)},\n` +
|
||||
` treatment: ${JSON.stringify(options.treatment)},\n` +
|
||||
(options.verbose === undefined ? "" : ` verbose: ${options.verbose},\n`) +
|
||||
`});\n`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
|
||||
import {
|
||||
fixtureNames,
|
||||
prepareFixtures,
|
||||
resetExperiments,
|
||||
writeExperiment,
|
||||
writeSubjectArchives,
|
||||
} from "./experiment-files.mjs";
|
||||
|
||||
const subject = {
|
||||
archive: Buffer.from("source"),
|
||||
dependencyArchive: Buffer.from("dependencies"),
|
||||
digest: "source-digest",
|
||||
dependencyDigest: "dependency-digest",
|
||||
};
|
||||
const benchmark = {
|
||||
id: "test",
|
||||
model: "test/model",
|
||||
displayName: "Test",
|
||||
harness: "OpenCode",
|
||||
support: "supported",
|
||||
};
|
||||
|
||||
test("materializes fixtures and complete experiment inputs", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "eve-benchmark-experiments-"));
|
||||
const evals = join(root, "evals");
|
||||
const experiments = join(root, "experiments");
|
||||
try {
|
||||
writeCase(evals, "author-002-second");
|
||||
writeCase(evals, "author-001-first");
|
||||
writeFileSync(join(evals, "not-a-case"), "ignored");
|
||||
assert.deepEqual(fixtureNames(evals), ["author-001-first", "author-002-second"]);
|
||||
|
||||
prepareFixtures(evals);
|
||||
assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "");
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(evals, "author-001-first", "package.json"))), {
|
||||
name: "eve-authoring-author-001-first",
|
||||
private: true,
|
||||
type: "module",
|
||||
});
|
||||
|
||||
resetExperiments(experiments);
|
||||
const archives = writeSubjectArchives(experiments, subject, "published-deadbeef");
|
||||
writeExperiment(experiments, "test-opencode--guided", {
|
||||
...archives,
|
||||
digest: subject.digest,
|
||||
dependencyDigest: subject.dependencyDigest,
|
||||
runs: 3,
|
||||
evals: ["author-001-first"],
|
||||
benchmark,
|
||||
treatment: "guided",
|
||||
});
|
||||
|
||||
assert.equal(readFileSync(join(experiments, archives.archiveName), "utf8"), "source");
|
||||
assert.equal(
|
||||
readFileSync(join(experiments, archives.dependencyArchiveName), "utf8"),
|
||||
"dependencies",
|
||||
);
|
||||
const experiment = readFileSync(join(experiments, "test-opencode--guided.ts"), "utf8");
|
||||
assert.match(experiment, /dependencyArchive: readFileSync/u);
|
||||
assert.match(experiment, /published-deadbeef\.dependencies\.tar\.gz/u);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function writeCase(evals, name) {
|
||||
const root = join(evals, name);
|
||||
mkdirSync(root, { recursive: true });
|
||||
writeFileSync(join(root, "CASE.ts"), "export default {};\n");
|
||||
assert.ok(existsSync(root));
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { createAuthoringAgent } from "./harness-agent.js";
|
||||
|
||||
export function authoringExperiment(options: {
|
||||
readonly archive: Uint8Array;
|
||||
readonly dependencyArchive: Uint8Array;
|
||||
readonly digest: string;
|
||||
readonly dependencyDigest: string;
|
||||
readonly runs?: number;
|
||||
@@ -17,6 +18,7 @@ export function authoringExperiment(options: {
|
||||
const agent = createAuthoringAgent({
|
||||
model: options.benchmark.model,
|
||||
archive: options.archive,
|
||||
dependencyArchive: options.dependencyArchive,
|
||||
digest: options.digest,
|
||||
dependencyDigest: options.dependencyDigest,
|
||||
});
|
||||
|
||||
@@ -23,12 +23,14 @@ import {
|
||||
WORKSPACE,
|
||||
} from "./paths.js";
|
||||
import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.js";
|
||||
import { BenchmarkTimings } from "./timing.js";
|
||||
|
||||
const HARNESS_BRIDGE_PORT = 4172;
|
||||
const BOOTSTRAP_VERSION = "v5";
|
||||
const BOOTSTRAP_VERSION = "v6";
|
||||
export function createAuthoringAgent(subject: {
|
||||
readonly model: string;
|
||||
readonly archive: Uint8Array;
|
||||
readonly dependencyArchive: Uint8Array;
|
||||
readonly digest: string;
|
||||
readonly dependencyDigest: string;
|
||||
}): Agent {
|
||||
@@ -57,8 +59,18 @@ export function createAuthoringAgent(subject: {
|
||||
const setups = [authoringCase.startingPoint.setup, authoringCase.setup].filter(
|
||||
(setup): setup is AuthoringSetup => setup !== undefined,
|
||||
);
|
||||
const timings = new BenchmarkTimings();
|
||||
timings.record("run.context", 0, "success", {
|
||||
sourceArchiveBytes: subject.archive.length,
|
||||
dependencyArchiveBytes: subject.dependencyArchive.length,
|
||||
sourceDigest: subject.digest,
|
||||
dependencyDigest: subject.dependencyDigest,
|
||||
startingPoint: authoringCase.startingPoint.id,
|
||||
setupCount: setups.length,
|
||||
});
|
||||
const sandbox = createDependencyCachedSandbox({
|
||||
archive: subject.archive,
|
||||
dependencyArchive: subject.dependencyArchive,
|
||||
dependencyDigest: subject.dependencyDigest,
|
||||
ports: [HARNESS_BRIDGE_PORT, ...new Set(setups.flatMap((setup) => setup.ports ?? []))],
|
||||
env: {
|
||||
@@ -67,6 +79,7 @@ export function createAuthoringAgent(subject: {
|
||||
...Object.assign({}, ...setups.map((setup) => setup.environment ?? {})),
|
||||
},
|
||||
log,
|
||||
timings,
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
const commands: string[] = [];
|
||||
@@ -89,22 +102,27 @@ export function createAuthoringAgent(subject: {
|
||||
onBootstrap: async ({ session: bootstrap, workDir }) => {
|
||||
const bootstrapSandbox = bootstrap as HarnessV1NetworkSandboxSession;
|
||||
const context = setupContext(bootstrapSandbox, workDir);
|
||||
for (const setup of setups) await setup.onBootstrap?.(context);
|
||||
await timings.measure("subject.case-bootstrap", async () => {
|
||||
for (const setup of setups) await setup.onBootstrap?.(context);
|
||||
});
|
||||
log("[setup] building the selected eve source");
|
||||
await bootstrapSubject(
|
||||
bootstrapSandbox,
|
||||
workDir,
|
||||
authoringCase.startingPoint.workspace,
|
||||
subject.archive,
|
||||
timings,
|
||||
);
|
||||
},
|
||||
onSession: async ({ session: current, sessionWorkDir }) => {
|
||||
activeSandbox = current as HarnessV1NetworkSandboxSession;
|
||||
workspace = sessionWorkDir;
|
||||
const context = setupContext(activeSandbox, workspace);
|
||||
for (const setup of setups) await setup.onSession?.(context);
|
||||
await timings.measure("session.setup", async () => {
|
||||
for (const setup of setups) await setup.onSession?.(context);
|
||||
if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);
|
||||
});
|
||||
if (options.agentOptions?.agentsMd !== true) {
|
||||
await installBaselineEveWrapper(context);
|
||||
await context.run("rm -f AGENTS.md CLAUDE.md GEMINI.md");
|
||||
}
|
||||
},
|
||||
@@ -113,7 +131,7 @@ export function createAuthoringAgent(subject: {
|
||||
});
|
||||
|
||||
try {
|
||||
session = await withBootstrapInitialization(bootstrapHash(authoringCase, subject), () =>
|
||||
session = await timings.measure("session.create", () =>
|
||||
agent.createSession({ abortSignal: options.signal }),
|
||||
);
|
||||
if (activeSandbox === undefined || workspace === undefined) {
|
||||
@@ -126,22 +144,25 @@ export function createAuthoringAgent(subject: {
|
||||
send: async (prompt) => {
|
||||
transcript.push({ role: "user", content: prompt });
|
||||
if (verbose) console.log(`[user] ${prompt}`);
|
||||
const result = verbose
|
||||
? await streamTurn(agent, session!, prompt, options.timeout, options.signal)
|
||||
: await agent.generate({
|
||||
session: session!,
|
||||
prompt,
|
||||
timeout: options.timeout,
|
||||
abortSignal: options.signal,
|
||||
});
|
||||
transcript.push({
|
||||
role: "assistant",
|
||||
content: result.text,
|
||||
toolCalls: authoringToolCalls(result.toolCalls),
|
||||
usage: normalizeUsage(result.usage),
|
||||
const turn = transcript.filter((entry) => entry.role === "user").length;
|
||||
const result = await timings.measure(`agent.turn.${turn}`, () =>
|
||||
verbose
|
||||
? streamTurn(agent, session!, prompt, options.timeout, options.signal)
|
||||
: generateTurn(agent, session!, prompt, options.timeout, options.signal),
|
||||
);
|
||||
const toolCalls = result.toolCalls;
|
||||
const usage = result.usage;
|
||||
transcript.push({ role: "assistant", content: result.text, toolCalls, usage });
|
||||
timings.record(`agent.turn.${turn}.summary`, 0, "success", {
|
||||
promptCharacters: prompt.length,
|
||||
responseCharacters: result.text.length,
|
||||
toolCalls: toolCalls.length,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
reasoningTokens: usage.reasoningTokens,
|
||||
});
|
||||
commands.push(...shellCommands(result.toolCalls));
|
||||
return { text: result.text, toolCalls: authoringToolCalls(result.toolCalls) };
|
||||
return { text: result.text, toolCalls };
|
||||
},
|
||||
});
|
||||
|
||||
@@ -178,19 +199,19 @@ export function createAuthoringAgent(subject: {
|
||||
]);
|
||||
|
||||
log("[grade] running deterministic assertions");
|
||||
const test = await resultOf(
|
||||
activeSandbox,
|
||||
`ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
|
||||
projectWorkspace,
|
||||
const test = await timings.measure("validation.grader", () =>
|
||||
resultOf(
|
||||
activeSandbox!,
|
||||
`ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
|
||||
projectWorkspace,
|
||||
),
|
||||
);
|
||||
const scriptsResults = Object.fromEntries(
|
||||
await Promise.all(
|
||||
(options.scripts ?? []).map(async (script) => {
|
||||
log(`[${script}] running`);
|
||||
const result = await resultOf(
|
||||
activeSandbox!,
|
||||
`npm run ${shellQuote(script)}`,
|
||||
projectWorkspace,
|
||||
const result = await timings.measure(`validation.${script}`, () =>
|
||||
resultOf(activeSandbox!, `npm run ${shellQuote(script)}`, projectWorkspace),
|
||||
);
|
||||
return [
|
||||
script,
|
||||
@@ -200,6 +221,12 @@ export function createAuthoringAgent(subject: {
|
||||
),
|
||||
);
|
||||
const scriptsPassed = Object.values(scriptsResults).every((result) => result.success);
|
||||
timings.record("run.total", Date.now() - startedAt);
|
||||
await context.write(
|
||||
`${AGENT_EVAL_DIRECTORY}/timings.json`,
|
||||
JSON.stringify(timings.entries),
|
||||
);
|
||||
logTimingSummary(log, timings);
|
||||
log(`[result] ${test.exitCode === 0 && scriptsPassed ? "passed" : "failed"}`);
|
||||
return {
|
||||
success: test.exitCode === 0 && scriptsPassed,
|
||||
@@ -211,12 +238,21 @@ export function createAuthoringAgent(subject: {
|
||||
transcript: harnessTranscript(transcript),
|
||||
scriptsResults,
|
||||
sandboxId: activeSandbox.id,
|
||||
generatedFiles: timingArtifact(timings),
|
||||
};
|
||||
} catch (error) {
|
||||
timings.record("run.total", Date.now() - startedAt, "failure");
|
||||
if (activeSandbox !== undefined && workspace !== undefined) {
|
||||
await setupContext(activeSandbox, workspace)
|
||||
.write(`${AGENT_EVAL_DIRECTORY}/harness-transcript.json`, JSON.stringify(transcript))
|
||||
.catch(() => undefined);
|
||||
await Promise.all([
|
||||
setupContext(activeSandbox, workspace).write(
|
||||
`${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,
|
||||
JSON.stringify(transcript),
|
||||
),
|
||||
setupContext(activeSandbox, workspace).write(
|
||||
`${AGENT_EVAL_DIRECTORY}/timings.json`,
|
||||
JSON.stringify(timings.entries),
|
||||
),
|
||||
]).catch(() => undefined);
|
||||
}
|
||||
const result: AgentRunResult = {
|
||||
success: false,
|
||||
@@ -225,6 +261,7 @@ export function createAuthoringAgent(subject: {
|
||||
duration: Date.now() - startedAt,
|
||||
transcript: harnessTranscript(transcript),
|
||||
scriptsResults: {},
|
||||
generatedFiles: timingArtifact(timings),
|
||||
};
|
||||
if (activeSandbox !== undefined) result.sandboxId = activeSandbox.id;
|
||||
return result;
|
||||
@@ -235,6 +272,21 @@ export function createAuthoringAgent(subject: {
|
||||
};
|
||||
}
|
||||
|
||||
async function generateTurn(
|
||||
agent: HarnessAgent,
|
||||
session: HarnessAgentSession,
|
||||
prompt: string,
|
||||
timeout: number,
|
||||
abortSignal?: AbortSignal,
|
||||
): Promise<AuthoringTurn & { usage: AuthoringTokenUsage }> {
|
||||
const result = await agent.generate({ session, prompt, timeout, abortSignal });
|
||||
return {
|
||||
text: result.text,
|
||||
toolCalls: authoringToolCalls(result.toolCalls),
|
||||
usage: normalizeUsage(result.usage),
|
||||
};
|
||||
}
|
||||
|
||||
async function streamTurn(
|
||||
agent: HarnessAgent,
|
||||
session: HarnessAgentSession,
|
||||
@@ -309,22 +361,25 @@ async function bootstrapSubject(
|
||||
workspace: string,
|
||||
workspaceKind: "scaffolded" | "empty",
|
||||
archive: Uint8Array,
|
||||
timings: BenchmarkTimings,
|
||||
): Promise<void> {
|
||||
await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });
|
||||
await run(
|
||||
sandbox,
|
||||
[
|
||||
`rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT}`,
|
||||
`tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT}`,
|
||||
`pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,
|
||||
`pnpm --dir ${SOURCE_ROOT} --filter eve build`,
|
||||
"mkdir -p /tmp/eve-package",
|
||||
`pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package`,
|
||||
`mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH}`,
|
||||
`npm install --global --package-lock=false ${EVE_PACKAGE_PATH}`,
|
||||
'ln -sf "$(npm prefix --global)/bin/eve" /usr/local/bin/eve',
|
||||
"command -v eve",
|
||||
].join(" && "),
|
||||
await timings.measure("subject.source-upload", async () => {
|
||||
await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });
|
||||
});
|
||||
await timings.measure("subject.source-install", () =>
|
||||
run(
|
||||
sandbox,
|
||||
`rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,
|
||||
),
|
||||
);
|
||||
await timings.measure("subject.eve-build", () =>
|
||||
run(sandbox, `pnpm --dir ${SOURCE_ROOT} --filter eve build`),
|
||||
);
|
||||
await timings.measure("subject.eve-pack-and-cli", () =>
|
||||
run(
|
||||
sandbox,
|
||||
`mkdir -p /tmp/eve-package && pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package && mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH} && ln -sf ${SOURCE_ROOT}/packages/eve/bin/eve.js /usr/local/bin/eve && command -v eve`,
|
||||
),
|
||||
);
|
||||
const artifactsRoot = `${workspace}/${AGENT_EVAL_DIRECTORY}`;
|
||||
const workspaceCommands: string[] = [];
|
||||
@@ -339,7 +394,9 @@ async function bootstrapSubject(
|
||||
if (workspaceKind === "scaffolded") {
|
||||
workspaceCommands.push(`test -f ${workspace}/package.json`);
|
||||
}
|
||||
await run(sandbox, workspaceCommands.join(" && "));
|
||||
await timings.measure("subject.workspace-bootstrap", () =>
|
||||
run(sandbox, workspaceCommands.join(" && ")),
|
||||
);
|
||||
}
|
||||
|
||||
function setupContext(
|
||||
@@ -364,7 +421,7 @@ function setupContext(
|
||||
|
||||
async function installBaselineEveWrapper(context: AuthoringSetupContext): Promise<void> {
|
||||
await context.run(`
|
||||
cli_path="$(npm prefix --global)/bin/eve"
|
||||
cli_path="/usr/local/bin/eve"
|
||||
test -x "$cli_path"
|
||||
real_cli="$cli_path.guided"
|
||||
if [ ! -e "$real_cli" ]; then mv "$cli_path" "$real_cli"; fi
|
||||
@@ -379,6 +436,16 @@ chmod +x "$cli_path"
|
||||
`);
|
||||
}
|
||||
|
||||
function timingArtifact(timings: BenchmarkTimings): Record<string, string> {
|
||||
return { "benchmark/timings.json": `${JSON.stringify(timings.entries, null, 2)}\n` };
|
||||
}
|
||||
|
||||
function logTimingSummary(log: (message: string) => void, timings: BenchmarkTimings): void {
|
||||
for (const timing of timings.entries) {
|
||||
log(`[timing] ${timing.phase}: ${timing.durationMs}ms (${timing.outcome})`);
|
||||
}
|
||||
}
|
||||
|
||||
function harnessTranscript(transcript: ReadonlyArray<AuthoringTranscriptEntry>): string {
|
||||
return transcript
|
||||
.map((entry) => {
|
||||
@@ -422,43 +489,6 @@ function bootstrapHash(authoringCase: AuthoringCase, subject: { readonly digest:
|
||||
return `eve-authoring-${BOOTSTRAP_VERSION}-${subject.digest}-${authoringCase.startingPoint.id}-${setupIds}`;
|
||||
}
|
||||
|
||||
const bootstrapCoordination = globalThis as typeof globalThis & {
|
||||
__eveAuthoringBootstrapLocks?: Map<string, Promise<void>>;
|
||||
__eveAuthoringBootstrapsReady?: Set<string>;
|
||||
};
|
||||
|
||||
async function withBootstrapInitialization<T>(
|
||||
key: string,
|
||||
operation: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const ready = (bootstrapCoordination.__eveAuthoringBootstrapsReady ??= new Set());
|
||||
if (ready.has(key)) return operation();
|
||||
|
||||
const locks = (bootstrapCoordination.__eveAuthoringBootstrapLocks ??= new Map());
|
||||
const previous = locks.get(key) ?? Promise.resolve();
|
||||
let release = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const tail = previous.then(() => current);
|
||||
locks.set(key, tail);
|
||||
|
||||
await previous;
|
||||
if (ready.has(key)) {
|
||||
release();
|
||||
if (locks.get(key) === tail) locks.delete(key);
|
||||
return operation();
|
||||
}
|
||||
try {
|
||||
const result = await operation();
|
||||
ready.add(key);
|
||||
return result;
|
||||
} finally {
|
||||
release();
|
||||
if (locks.get(key) === tail) locks.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
async function resultOf(
|
||||
sandbox: HarnessV1NetworkSandboxSession,
|
||||
command: string,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { readdir, stat } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -12,7 +12,7 @@ const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: fa
|
||||
const { loadAuthoringCase } = await jiti.import(resolve(libRoot, "load-authoring-case.ts"));
|
||||
|
||||
for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
if (!entry.isDirectory() || !(await hasCase(entry.name))) continue;
|
||||
|
||||
test(`loads ${entry.name}/CASE.ts`, async () => {
|
||||
const authoringCase = await loadAuthoringCase(resolve(evalsRoot, entry.name));
|
||||
@@ -21,6 +21,14 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
|
||||
});
|
||||
}
|
||||
|
||||
async function hasCase(name) {
|
||||
try {
|
||||
return (await stat(resolve(evalsRoot, name, "CASE.ts"))).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
test("loads the named project output directory", async () => {
|
||||
const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));
|
||||
assert.equal(authoringCase.projectDirectory, "wayfinder");
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
export const WORKSPACE = "workspace";
|
||||
export const AGENT_EVAL_DIRECTORY = "/tmp/eve-grader";
|
||||
export const WORKSPACE_ENV = "EVE_BENCHMARK_WORKSPACE";
|
||||
export const AGENT_EVAL_DIRECTORY = ".eve-grader";
|
||||
export const AUTHORING_EVAL_DIRECTORY = "/tmp/photon";
|
||||
export const WORLD_EVENTS_PATH = `${AUTHORING_EVAL_DIRECTORY}/world-events.jsonl`;
|
||||
export const SOURCE_ROOT = "/tmp/eve-source";
|
||||
export const SOURCE_ARCHIVE_PATH = "/tmp/eve-source.tar.gz";
|
||||
export const EVE_PACKAGE_PATH = "/tmp/eve-package/eve.tgz";
|
||||
export const AUTHORING_EVAL_DIRECTORY_ENV = "EVE_AUTHORING_EVAL_DIRECTORY";
|
||||
export const AUTHORING_MODEL = "openai/gpt-5.5";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -45,6 +45,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detai
|
||||
description,
|
||||
archive,
|
||||
digest,
|
||||
dependencyArchive: dependencyArchive(archive),
|
||||
dependencyDigest: dependencyDigest(archive),
|
||||
...details,
|
||||
};
|
||||
@@ -55,7 +56,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detai
|
||||
|
||||
function dependencyDigest(archive) {
|
||||
const hash = createHash("sha256");
|
||||
for (const path of [".npmrc", "package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"]) {
|
||||
for (const path of dependencyPaths(archive)) {
|
||||
const content = execFileSync("tar", ["-xOzf", "-", path], {
|
||||
input: archive,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
@@ -65,6 +66,43 @@ function dependencyDigest(archive) {
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
function dependencyArchive(archive) {
|
||||
const directory = mkdtempSync(join(tmpdir(), "eve-authoring-dependencies-"));
|
||||
const archivePath = join(directory, "dependencies.tar.gz");
|
||||
try {
|
||||
for (const path of dependencyPaths(archive)) {
|
||||
const destination = join(directory, "input", path);
|
||||
mkdirSync(join(destination, ".."), { recursive: true });
|
||||
writeFileSync(
|
||||
destination,
|
||||
execFileSync("tar", ["-xOzf", "-", path], {
|
||||
input: archive,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
}),
|
||||
);
|
||||
}
|
||||
execFileSync("tar", ["-czf", archivePath, "-C", join(directory, "input"), "."]);
|
||||
return readFileSync(archivePath);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function dependencyPaths(archive) {
|
||||
const paths = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8" })
|
||||
.split("\n")
|
||||
.filter(Boolean);
|
||||
return paths.filter(
|
||||
(path) =>
|
||||
path === ".npmrc" ||
|
||||
path === "package.json" ||
|
||||
path === "pnpm-lock.yaml" ||
|
||||
path === "pnpm-workspace.yaml" ||
|
||||
/(?:^|\/)package\.json$/u.test(path) ||
|
||||
path.startsWith("patches/"),
|
||||
);
|
||||
}
|
||||
|
||||
function git(cwd, args, env = process.env) {
|
||||
return execFileSync("git", args, { cwd, env, encoding: "utf8" });
|
||||
}
|
||||
|
||||
@@ -23,11 +23,22 @@ test("archives the working tree without changing the index", () => {
|
||||
assert.equal(existsSync(join(extracted, "ignored.txt")), false);
|
||||
assert.equal(existsSync(join(extracted, "deleted.txt")), false);
|
||||
});
|
||||
withExtracted(subject.dependencyArchive, (extracted) => {
|
||||
assert.equal(readFileSync(join(extracted, "package.json"), "utf8"), '{"private":true}\n');
|
||||
assert.equal(
|
||||
readFileSync(join(extracted, "pnpm-lock.yaml"), "utf8"),
|
||||
"lockfileVersion: '9.0'\n",
|
||||
);
|
||||
assert.equal(existsSync(join(extracted, "tracked.txt")), false);
|
||||
});
|
||||
assert.equal(git(repository, ["status", "--porcelain"]), statusBefore);
|
||||
writeFileSync(join(repository, "tracked.txt"), "another source change\n");
|
||||
const sourceChange = workingTreeSubject(repository);
|
||||
assert.notEqual(sourceChange.digest, subject.digest);
|
||||
assert.equal(sourceChange.dependencyDigest, subject.dependencyDigest);
|
||||
assert.deepEqual(sourceChange.dependencyArchive, subject.dependencyArchive);
|
||||
writeFileSync(join(repository, "packages.json"), "not a manifest\n");
|
||||
assert.equal(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
|
||||
writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n");
|
||||
assert.notEqual(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { BenchmarkTimings } from "./timing.ts";
|
||||
|
||||
test("records successful and failed phases", async () => {
|
||||
const timings = new BenchmarkTimings();
|
||||
await timings.measure("success", async () => undefined, { turns: 1 });
|
||||
await assert.rejects(timings.measure("failure", async () => Promise.reject(new Error("nope"))));
|
||||
|
||||
assert.equal(timings.entries.length, 2);
|
||||
assert.deepEqual(
|
||||
timings.entries.map(({ phase, outcome, details }) => ({ phase, outcome, details })),
|
||||
[
|
||||
{ phase: "success", outcome: "success", details: { turns: 1 } },
|
||||
{ phase: "failure", outcome: "failure", details: undefined },
|
||||
],
|
||||
);
|
||||
assert.ok(timings.entries.every((entry) => entry.durationMs >= 0));
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
export interface BenchmarkTiming {
|
||||
readonly phase: string;
|
||||
readonly startedAt: string;
|
||||
readonly durationMs: number;
|
||||
readonly outcome: "success" | "failure";
|
||||
readonly details?: Readonly<Record<string, string | number | boolean>>;
|
||||
}
|
||||
|
||||
type TimingDetails = BenchmarkTiming["details"];
|
||||
|
||||
export class BenchmarkTimings {
|
||||
readonly entries: BenchmarkTiming[] = [];
|
||||
|
||||
async measure<T>(
|
||||
phase: string,
|
||||
operation: () => Promise<T>,
|
||||
details?: TimingDetails,
|
||||
): Promise<T> {
|
||||
const startedAt = new Date().toISOString();
|
||||
const started = performance.now();
|
||||
try {
|
||||
const result = await operation();
|
||||
this.add(phase, startedAt, performance.now() - started, "success", details);
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.add(phase, startedAt, performance.now() - started, "failure", details);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
record(
|
||||
phase: string,
|
||||
durationMs: number,
|
||||
outcome: BenchmarkTiming["outcome"] = "success",
|
||||
details?: TimingDetails,
|
||||
): void {
|
||||
this.add(phase, new Date(Date.now() - durationMs).toISOString(), durationMs, outcome, details);
|
||||
}
|
||||
|
||||
private add(
|
||||
phase: string,
|
||||
startedAt: string,
|
||||
durationMs: number,
|
||||
outcome: BenchmarkTiming["outcome"],
|
||||
details: TimingDetails,
|
||||
): void {
|
||||
const entry = {
|
||||
phase,
|
||||
startedAt,
|
||||
durationMs: Math.round(durationMs),
|
||||
outcome,
|
||||
};
|
||||
this.entries.push(details === undefined ? entry : { ...entry, details });
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,8 @@
|
||||
"benchmark:publish": "node publish.mjs",
|
||||
"benchmark:export": "node scripts/export-results.mjs",
|
||||
"test": "node --test lib/*.test.mjs lib/setups/*.test.mjs scripts/*.test.mjs",
|
||||
"typecheck": "tsc --noEmit"
|
||||
"typecheck": "tsc --noEmit",
|
||||
"benchmark:timings": "node scripts/timings.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/harness": "1.0.72",
|
||||
|
||||
+23
-39
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
@@ -13,6 +12,12 @@ import {
|
||||
publishedBenchmarkModels,
|
||||
publishedExperimentId,
|
||||
} from "./lib/benchmark-config.ts";
|
||||
import {
|
||||
prepareFixtures,
|
||||
resetExperiments,
|
||||
writeExperiment,
|
||||
writeSubjectArchives,
|
||||
} from "./lib/experiment-files.mjs";
|
||||
import { revisionSubject } from "./lib/source.mjs";
|
||||
|
||||
const appRoot = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -53,7 +58,7 @@ const experimentNames = benchmarks.flatMap((benchmark) =>
|
||||
authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),
|
||||
);
|
||||
|
||||
prepareFixtures();
|
||||
prepareFixtures(evalsRoot);
|
||||
writeExperiments(subject, revision, benchmarks);
|
||||
|
||||
console.log(`> eve revision: ${revision}`);
|
||||
@@ -151,47 +156,26 @@ To inspect pending benchmark cells without publishing:
|
||||
return false;
|
||||
}
|
||||
|
||||
function prepareFixtures() {
|
||||
for (const name of fixtureNames()) {
|
||||
const fixtureRoot = join(evalsRoot, name);
|
||||
writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
|
||||
writeFileSync(
|
||||
join(fixtureRoot, "package.json"),
|
||||
`${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function fixtureNames() {
|
||||
return readdirSync(evalsRoot, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function writeExperiments(subject, revision, benchmarks) {
|
||||
rmSync(experimentsRoot, { recursive: true, force: true });
|
||||
mkdirSync(experimentsRoot, { recursive: true });
|
||||
const archiveName = `published-${revision.slice(0, 12)}.source.tar.gz`;
|
||||
writeFileSync(join(experimentsRoot, archiveName), subject.archive);
|
||||
resetExperiments(experimentsRoot);
|
||||
const { archiveName, dependencyArchiveName } = writeSubjectArchives(
|
||||
experimentsRoot,
|
||||
subject,
|
||||
`published-${revision.slice(0, 12)}`,
|
||||
);
|
||||
|
||||
for (const benchmark of benchmarks) {
|
||||
for (const treatment of authoringTreatments) {
|
||||
const experimentName = publishedExperimentId(benchmark, treatment);
|
||||
writeFileSync(
|
||||
join(experimentsRoot, `${experimentName}.ts`),
|
||||
`import { readFileSync } from "node:fs";\n` +
|
||||
`import { authoringExperiment } from "../lib/experiment.js";\n\n` +
|
||||
`export default authoringExperiment({\n` +
|
||||
` archive: readFileSync(new URL(${JSON.stringify(`./${archiveName}`)}, import.meta.url)),\n` +
|
||||
` digest: ${JSON.stringify(subject.digest)},\n` +
|
||||
` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
|
||||
` runs: ${publishedBenchmark.runs},\n` +
|
||||
` evals: ${JSON.stringify(publishedBenchmark.caseIds)},\n` +
|
||||
` benchmark: ${JSON.stringify(benchmark)},\n` +
|
||||
` treatment: ${JSON.stringify(treatment)},\n` +
|
||||
`});\n`,
|
||||
);
|
||||
writeExperiment(experimentsRoot, publishedExperimentId(benchmark, treatment), {
|
||||
archiveName,
|
||||
dependencyArchiveName,
|
||||
digest: subject.digest,
|
||||
dependencyDigest: subject.dependencyDigest,
|
||||
runs: publishedBenchmark.runs,
|
||||
evals: publishedBenchmark.caseIds,
|
||||
benchmark,
|
||||
treatment,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+23
-37
@@ -1,12 +1,18 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import { findBenchmarkModel, parseAuthoringTreatment } from "./lib/benchmark-config.ts";
|
||||
import {
|
||||
prepareFixtures,
|
||||
resetExperiments,
|
||||
writeExperiment,
|
||||
writeSubjectArchives,
|
||||
} from "./lib/experiment-files.mjs";
|
||||
import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";
|
||||
|
||||
const appRoot = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -59,7 +65,7 @@ const subjects =
|
||||
: revisionSubject(repositoryRoot, values.head, "head"),
|
||||
];
|
||||
|
||||
prepareFixtures();
|
||||
prepareFixtures(evalsRoot, selectedEval === undefined ? undefined : [selectedEval]);
|
||||
mkdirSync(join(appRoot, "results"), { recursive: true });
|
||||
writeExperiments(subjects, runs, benchmark, treatment, values.verbose ?? false);
|
||||
const executable = join(appRoot, "node_modules/.bin/agent-eval");
|
||||
@@ -75,44 +81,24 @@ const result = spawnSync(executable, args, {
|
||||
if (result.error) throw result.error;
|
||||
process.exit(result.status ?? 1);
|
||||
|
||||
function prepareFixtures() {
|
||||
const names = selectedEval === undefined ? fixtureNames() : [selectedEval];
|
||||
for (const name of names) {
|
||||
const fixtureRoot = join(evalsRoot, name);
|
||||
writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
|
||||
writeFileSync(
|
||||
join(fixtureRoot, "package.json"),
|
||||
`${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function fixtureNames() {
|
||||
return readdirSync(evalsRoot, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
|
||||
.map((entry) => entry.name);
|
||||
}
|
||||
|
||||
function writeExperiments(subjects, runs, benchmark, treatment, verbose) {
|
||||
rmSync(experimentsRoot, { recursive: true, force: true });
|
||||
mkdirSync(experimentsRoot, { recursive: true });
|
||||
resetExperiments(experimentsRoot);
|
||||
for (const subject of subjects) {
|
||||
const archivePath = join(experimentsRoot, `${subject.label}.source.tar.gz`);
|
||||
writeFileSync(archivePath, subject.archive);
|
||||
writeFileSync(
|
||||
join(experimentsRoot, `${subject.label}.ts`),
|
||||
`import { readFileSync } from "node:fs";\n` +
|
||||
`import { authoringExperiment } from "../lib/experiment.js";\n\n` +
|
||||
`export default authoringExperiment({\n` +
|
||||
` archive: readFileSync(new URL(${JSON.stringify(`./${subject.label}.source.tar.gz`)}, import.meta.url)),\n` +
|
||||
` digest: ${JSON.stringify(subject.digest)},\n` +
|
||||
` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
|
||||
` runs: ${runs},\n` +
|
||||
` benchmark: ${JSON.stringify(benchmark)},\n` +
|
||||
` treatment: ${JSON.stringify(treatment)},\n` +
|
||||
` verbose: ${verbose},\n` +
|
||||
`});\n`,
|
||||
const { archiveName, dependencyArchiveName } = writeSubjectArchives(
|
||||
experimentsRoot,
|
||||
subject,
|
||||
subject.label,
|
||||
);
|
||||
writeExperiment(experimentsRoot, subject.label, {
|
||||
archiveName,
|
||||
dependencyArchiveName,
|
||||
digest: subject.digest,
|
||||
dependencyDigest: subject.dependencyDigest,
|
||||
runs,
|
||||
benchmark,
|
||||
treatment,
|
||||
verbose,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
const { positionals, values } = parseArgs({
|
||||
args: process.argv.slice(2),
|
||||
allowPositionals: true,
|
||||
options: { json: { type: "boolean" }, help: { type: "boolean", short: "h" } },
|
||||
strict: true,
|
||||
});
|
||||
|
||||
if (values.help || positionals.length !== 1) {
|
||||
console.log("Usage: node scripts/timings.mjs <run-directory-or-timings.json> [--json]");
|
||||
process.exit(values.help ? 0 : 1);
|
||||
}
|
||||
|
||||
const timingsPath = resolveTimingsPath(positionals[0]);
|
||||
const timings = JSON.parse(readFileSync(timingsPath, "utf8"));
|
||||
if (!Array.isArray(timings)) throw new Error(`${timingsPath} must contain a timing array.`);
|
||||
|
||||
if (values.json) {
|
||||
console.log(JSON.stringify(timings, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const phases = timings.filter((timing) => timing.phase !== "run.context");
|
||||
const width = Math.max(...phases.map((timing) => timing.phase.length));
|
||||
for (const timing of phases) {
|
||||
const details = timing.details === undefined ? "" : ` ${JSON.stringify(timing.details)}`;
|
||||
console.log(
|
||||
`${timing.phase.padEnd(width)} ${formatDuration(timing.durationMs).padStart(8)} ${timing.outcome}${details}`,
|
||||
);
|
||||
}
|
||||
|
||||
const byPhase = new Map(phases.map((timing) => [timing.phase, timing.durationMs]));
|
||||
const setup = byPhase.get("session.create") ?? 0;
|
||||
const agent = phases
|
||||
.filter((timing) => /^agent\.turn\.\d+$/u.test(timing.phase))
|
||||
.reduce((total, timing) => total + timing.durationMs, 0);
|
||||
const grader = byPhase.get("validation.grader") ?? 0;
|
||||
const checks = Math.max(
|
||||
0,
|
||||
...phases
|
||||
.filter(
|
||||
(timing) => timing.phase.startsWith("validation.") && timing.phase !== "validation.grader",
|
||||
)
|
||||
.map((timing) => timing.durationMs),
|
||||
);
|
||||
console.log(`\nSetup: ${formatDuration(setup)}`);
|
||||
console.log(`Agent: ${formatDuration(agent)}`);
|
||||
console.log(`Grader: ${formatDuration(grader)}`);
|
||||
console.log(`Checks: ${formatDuration(checks)}`);
|
||||
console.log(`Validation: ${formatDuration(grader + checks)}`);
|
||||
console.log(`Total: ${formatDuration(byPhase.get("run.total") ?? 0)}`);
|
||||
|
||||
function resolveTimingsPath(input) {
|
||||
const path = resolve(input);
|
||||
if (path.endsWith(".json")) return path;
|
||||
const candidates = [
|
||||
join(path, "project/benchmark/timings.json"),
|
||||
join(dirname(path), "project/benchmark/timings.json"),
|
||||
];
|
||||
const timingPath = candidates.find(existsSync);
|
||||
if (timingPath === undefined) {
|
||||
throw new Error(`Could not find project/benchmark/timings.json under ${path}.`);
|
||||
}
|
||||
return timingPath;
|
||||
}
|
||||
|
||||
function formatDuration(durationMs) {
|
||||
return `${(durationMs / 1000).toFixed(1)}s`;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
|
||||
const script = new URL("./timings.mjs", import.meta.url);
|
||||
|
||||
test("reports timing phases and aggregate durations", () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-timings-"));
|
||||
const path = join(directory, "timings.json");
|
||||
try {
|
||||
writeFileSync(
|
||||
path,
|
||||
JSON.stringify([
|
||||
{ phase: "run.context", durationMs: 0, outcome: "success", details: { cache: "cold" } },
|
||||
{ phase: "dependency-snapshot.get-or-create", durationMs: 1_000, outcome: "success" },
|
||||
{ phase: "subject-snapshot.get-or-create", durationMs: 2_000, outcome: "success" },
|
||||
{ phase: "session.create", durationMs: 6_000, outcome: "success" },
|
||||
{ phase: "agent.turn.1", durationMs: 3_000, outcome: "success" },
|
||||
{ phase: "validation.grader", durationMs: 6_000, outcome: "success" },
|
||||
{ phase: "validation.typecheck", durationMs: 2_000, outcome: "success" },
|
||||
{ phase: "validation.build", durationMs: 4_000, outcome: "success" },
|
||||
{ phase: "run.total", durationMs: 13_000, outcome: "success" },
|
||||
]),
|
||||
);
|
||||
const output = execFileSync(process.execPath, [script.pathname, path], { encoding: "utf8" });
|
||||
assert.match(output, /dependency-snapshot\.get-or-create\s+1\.0s\s+success/u);
|
||||
assert.match(output, /Setup:\s+6\.0s/u);
|
||||
assert.match(output, /Agent:\s+3\.0s/u);
|
||||
assert.match(output, /Grader:\s+6\.0s/u);
|
||||
assert.match(output, /Checks:\s+4\.0s/u);
|
||||
assert.match(output, /Validation:\s+10\.0s/u);
|
||||
assert.match(output, /Total:\s+13\.0s/u);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user