diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts index b15b036c7..3eb1beb56 100644 --- a/apps/benchmarks/lib/harness-agent.ts +++ b/apps/benchmarks/lib/harness-agent.ts @@ -27,6 +27,7 @@ import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.j import { BenchmarkTimings } from "./timing.js"; const HARNESS_BRIDGE_PORT = 4172; +const POST_RUN_GRADER_DIRECTORY = ".eve-grader"; const BOOTSTRAP_VERSION = "v8"; // A turn that stops producing output should end the turn, not the eval: the // remaining turns still run and the graders still see what the agent did. @@ -224,6 +225,7 @@ export function createAuthoringAgent(subject: { ? workspace : `${workspace}/${authoringCase.projectDirectory}`; const context = setupContext(activeSandbox, workspace); + const graderContext = setupContext(activeSandbox, projectWorkspace); await prepareGraderDirectory(context); await context.write( `${AGENT_EVAL_DIRECTORY}/results.json`, @@ -234,20 +236,20 @@ export function createAuthoringAgent(subject: { JSON.stringify(transcript), ); await Promise.all([ - context.write( - `${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`, + graderContext.write( + `${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`, readFileSync(`${fixturePath}/EVAL.ts`, "utf8"), ), - context.write( - `${AGENT_EVAL_DIRECTORY}/grader.ts`, + graderContext.write( + `${POST_RUN_GRADER_DIRECTORY}/grader.ts`, readFileSync(new URL("./grader.ts", import.meta.url), "utf8"), ), - context.write( - `${AGENT_EVAL_DIRECTORY}/paths.ts`, + graderContext.write( + `${POST_RUN_GRADER_DIRECTORY}/paths.ts`, readFileSync(new URL("./paths.ts", import.meta.url), "utf8"), ), - context.write( - `${AGENT_EVAL_DIRECTORY}/protocol.ts`, + graderContext.write( + `${POST_RUN_GRADER_DIRECTORY}/protocol.ts`, readFileSync(new URL("./protocol.ts", import.meta.url), "utf8"), ), ]); @@ -256,7 +258,7 @@ export function createAuthoringAgent(subject: { const test = await timings.measure("validation.grader", () => resultOf( activeSandbox!, - `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`, + `vitest run ${POST_RUN_GRADER_DIRECTORY}/EVAL.test.ts`, projectWorkspace, ), ); @@ -544,7 +546,12 @@ async function bootstrapSubject( if (workspaceKind === "scaffolded") { workspaceCommands.push(`cd ${shellQuote(workspace)} && AI_AGENT=benchmark eve init .`); } - workspaceCommands.push("command -v vitest >/dev/null"); + // Grader files must not change what subject commands observe in the project directory. + workspaceCommands.push( + `rm -rf ${AGENT_EVAL_DIRECTORY} && mkdir -p ${AGENT_EVAL_DIRECTORY}`, + `printf '{"private":true,"type":"module"}\\n' >${AGENT_EVAL_DIRECTORY}/package.json`, + "command -v vitest >/dev/null", + ); if (workspaceKind === "scaffolded") { workspaceCommands.push(`test -f ${workspace}/package.json`); }