skills/vercel__eve
1
0
Fork 0
mirror of https://github.com/vercel/eve.git synced 2026-09-20 05:35:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: isolate authoring benchmark artifacts to reduce distraction for benchmark agents (#2240)

Browse Source 
Signed-off-by: Colton Padden <colton.padden@vercel.com>
This commit is contained in:
Colton Padden
2026-08-18 16:56:40 -04:00
committed by GitHub
parent 5678b9ee07
commit 30ec0f8b14
1 changed files with 17 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/benchmarks/lib/harness-agent.ts
+17 -10
View File
@@ -27,6 +27,7 @@ import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.j
import { BenchmarkTimings } from "./timing.js";
const HARNESS_BRIDGE_PORT = 4172;
const POST_RUN_GRADER_DIRECTORY = ".eve-grader";
const BOOTSTRAP_VERSION = "v8";
// A turn that stops producing output should end the turn, not the eval: the
// remaining turns still run and the graders still see what the agent did.
@@ -224,6 +225,7 @@ export function createAuthoringAgent(subject: {
? workspace
: `${workspace}/${authoringCase.projectDirectory}`;
const context = setupContext(activeSandbox, workspace);
const graderContext = setupContext(activeSandbox, projectWorkspace);
await prepareGraderDirectory(context);
await context.write(
`${AGENT_EVAL_DIRECTORY}/results.json`,
@@ -234,20 +236,20 @@ export function createAuthoringAgent(subject: {
JSON.stringify(transcript),
);
await Promise.all([
context.write(
`${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,
graderContext.write(
`${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
readFileSync(`${fixturePath}/EVAL.ts`, "utf8"),
),
context.write(
`${AGENT_EVAL_DIRECTORY}/grader.ts`,
graderContext.write(
`${POST_RUN_GRADER_DIRECTORY}/grader.ts`,
readFileSync(new URL("./grader.ts", import.meta.url), "utf8"),
),
context.write(
`${AGENT_EVAL_DIRECTORY}/paths.ts`,
graderContext.write(
`${POST_RUN_GRADER_DIRECTORY}/paths.ts`,
readFileSync(new URL("./paths.ts", import.meta.url), "utf8"),
),
context.write(
`${AGENT_EVAL_DIRECTORY}/protocol.ts`,
graderContext.write(
`${POST_RUN_GRADER_DIRECTORY}/protocol.ts`,
readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"),
),
]);
@@ -256,7 +258,7 @@ export function createAuthoringAgent(subject: {
const test = await timings.measure("validation.grader", () =>
resultOf(
activeSandbox!,
`ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
`vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`,
projectWorkspace,
),
);
@@ -544,7 +546,12 @@ async function bootstrapSubject(
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`);
}
workspaceCommands.push("command -v vitest >/dev/null");
// Grader files must not change what subject commands observe in the project directory.
workspaceCommands.push(
`rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`,
`printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`,
"command -v vitest >/dev/null",
);
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`test -f ${workspace}/package.json`);
}