test: allow eve init <agent-name> outside of workspace in project creation benchmark (#2241)

Signed-off-by: Colton Padden <colton.padden@vercel.com>

apps/benchmarks/evals/author-002-new-project/CASE.ts

@@ -2,9 +2,10 @@ import { defineAuthoringCase, emptyProject } from "../../lib/authoring-case.js";
 
 export default defineAuthoringCase({
   startingPoint: emptyProject,
+  projectDirectory: "wayfinder",
   async interact({ send }) {
     await send(
-      "Create a new eve project directly in the current working directory for a concise travel assistant called Wayfinder. Do not create a subdirectory: initialize the current directory (for example, with `eve init .`). It should use the default model and be ready to build.",
+      "Create a new eve project named `wayfinder` for a concise travel assistant called Wayfinder. It should use the default model and be ready to build.",
     );
   },
 });

apps/benchmarks/evals/author-002-new-project/EVAL.ts

@@ -4,7 +4,7 @@ import { expect, test } from "vitest";
 
 import { subjectDefaultAgentModel } from "./grader.js";
 
-test("creates a complete eve project in place", () => {
+test("creates a complete eve project", () => {
   expect(existsSync("agent/agent.ts")).toBe(true);
   expect(existsSync("agent/channels/eve.ts")).toBe(true);
   expect(existsSync("agent/instructions.md")).toBe(true);

apps/benchmarks/lib/authoring-case.ts

@@ -41,6 +41,8 @@ export interface AuthoringInteractionContext {
 
 export interface AuthoringCase {
   readonly startingPoint: AuthoringStartingPoint;
+  /** Directory created by the agent, relative to the starting workspace. */
+  readonly projectDirectory?: string;
   readonly setup?: AuthoringSetup;
   readonly interact: (context: AuthoringInteractionContext) => Promise<void>;
 }

apps/benchmarks/lib/harness-agent.ts

@@ -145,6 +145,10 @@ export function createAuthoringAgent(subject: {
           },
         });
 
+        const projectWorkspace =
+          authoringCase.projectDirectory === undefined
+            ? workspace
+            : `${workspace}/${authoringCase.projectDirectory}`;
         const context = setupContext(activeSandbox, workspace);
         await context.write(
           `${AGENT_EVAL_DIRECTORY}/results.json`,
@@ -176,8 +180,8 @@ export function createAuthoringAgent(subject: {
         log("[grade] running deterministic assertions");
         const test = await resultOf(
           activeSandbox,
-          `vitest run ${workspace}/${AGENT_EVAL_DIRECTORY}/EVAL.test.ts`,
-          workspace,
+          `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
+          projectWorkspace,
         );
         const scriptsResults = Object.fromEntries(
           await Promise.all(
@@ -186,7 +190,7 @@ export function createAuthoringAgent(subject: {
               const result = await resultOf(
                 activeSandbox!,
                 `npm run ${shellQuote(script)}`,
-                workspace,
+                projectWorkspace,
               );
               return [
                 script,

apps/benchmarks/lib/load-authoring-case.test.mjs

@@ -21,6 +21,11 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
   });
 }
 
+test("loads the named project output directory", async () => {
+  const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));
+  assert.equal(authoringCase.projectDirectory, "wayfinder");
+});
+
 test("requires an iMessage phone-number request before supplying the number", async () => {
   const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-000-imessage"));
   const prompts = [];