[1/3] test(e2e): cover self-mod tool generation and repair (#3327)

Signed-off-by: benpankow <ben.pankow@vercel.com>
2026-09-20 05:35:39 +08:00 · 2026-09-16 08:25:51 -07:00
parent 7b17b27f45
commit 8dbfe3f3eb
25 changed files with 1554 additions and 81 deletions
@@ -14,95 +14,123 @@
 //                         `additionalModels` and make selected legs non-blocking
 //                         through `optionalModels`.
 //   world_matrix_<world>  `{ name, dir[, world_package] }` entries for that
-//                         world's suite workflow, which runs every fixture
-//                         once with mock models (EVE_E2E_MODEL=mock).
+//                         world's suite workflow, which runs fixtures that
+//                         select that world once with mock models
+//                         (EVE_E2E_MODEL=mock).
 import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { fileURLToPath } from "node:url";
 import { join } from "node:path";

-const roots = ["e2e/fixtures", "apps/fixtures"];
-
-const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8"));
-const models = validateNamedEntries(registry.models, "models", ["id"]);
-const worlds = validateNamedEntries(registry.worlds, "worlds", []);
-
-const fixtures = [];
-for (const root of roots) {
-  if (!existsSync(root)) continue;
-  for (const entry of readdirSync(root).sort()) {
-    const dir = join(root, entry);
-    if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue;
-
-    let modelMatrix = "default";
-    let additionalModels = [];
-    let optionalModels = [];
-    const packageJsonPath = join(dir, "package.json");
-    if (existsSync(packageJsonPath)) {
-      const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8"));
-      modelMatrix = pkg.e2e?.modelMatrix ?? "default";
-      additionalModels = validateNamedEntries(
-        pkg.e2e?.additionalModels ?? [],
-        `${packageJsonPath}: e2e.additionalModels`,
-        ["id"],
-        { allowEmpty: true },
-      );
-      optionalModels = validateNames(
-        pkg.e2e?.optionalModels ?? [],
-        `${packageJsonPath}: e2e.optionalModels`,
-      );
-    }
-    if (modelMatrix !== "default" && modelMatrix !== "full") {
-      throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);
-    }
-
-    fixtures.push({ name: entry, dir, modelMatrix, additionalModels, optionalModels });
+export function discoverE2eFixtures({ registry, fixtures }) {
+  const models = validateNamedEntries(registry.models, "models", ["id"]);
+  const worlds = validateNamedEntries(registry.worlds, "worlds", []);
+  const normalizedFixtures = fixtures.map((fixture) => normalizeFixture(fixture, worlds));
+  if (normalizedFixtures.length === 0) {
+    throw new Error("No e2e fixtures with an evals/ directory were found.");
  }
-}

-if (fixtures.length === 0) {
-  console.error("No e2e fixtures with an evals/ directory were found.");
-  process.exit(1);
-}
-
-const modelMatrix = fixtures.flatMap(
-  ({ name, dir, modelMatrix, additionalModels, optionalModels }) => {
-    const fixtureModels = uniqueModels([
-      ...(modelMatrix === "full" ? models : models.slice(0, 1)),
-      ...additionalModels,
-    ]);
-    const selectedNames = new Set(fixtureModels.map((model) => model.name));
-    for (const optionalModel of optionalModels) {
-      if (!selectedNames.has(optionalModel)) {
-        throw new Error(
-          `${dir}/package.json: optional model "${optionalModel}" is not selected by this fixture.`,
-        );
+  const modelMatrix = normalizedFixtures.flatMap(
+    ({ name, dir, modelMatrix, additionalModels, optionalModels }) => {
+      const fixtureModels = uniqueModels([
+        ...(modelMatrix === "full" ? models : models.slice(0, 1)),
+        ...additionalModels,
+      ]);
+      const selectedNames = new Set(fixtureModels.map((model) => model.name));
+      for (const optionalModel of optionalModels) {
+        if (!selectedNames.has(optionalModel)) {
+          throw new Error(
+            `${dir}/package.json: optional model "${optionalModel}" is not selected by this fixture.`,
+          );
+        }
      }
-    }
-    return fixtureModels.map((model) => ({
-      name,
-      dir,
-      model_name: model.name,
-      model_id: model.id,
-      optional: optionalModels.includes(model.name),
-    }));
-  },
-);
-
-const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];
-for (const world of worlds) {
-  const legs = fixtures.map(({ name, dir }) =>
-    world.package === undefined ? { name, dir } : { name, dir, world_package: world.package },
+      return fixtureModels.map((model) => ({
+        name,
+        dir,
+        model_name: model.name,
+        model_id: model.id,
+        optional: optionalModels.includes(model.name),
+      }));
+    },
  );
-  outputs.push(`world_matrix_${world.name}=${JSON.stringify(legs)}`);
+
+  const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];
+  for (const world of worlds) {
+    const legs = normalizedFixtures
+      .filter(({ selectedWorlds }) => selectedWorlds.includes(world.name))
+      .map(({ name, dir }) =>
+        world.package === undefined ? { name, dir } : { name, dir, world_package: world.package },
+      );
+    if (legs.length === 0) {
+      throw new Error(`No e2e fixtures select the registered world "${world.name}".`);
+    }
+    outputs.push(`world_matrix_${world.name}=${JSON.stringify(legs)}`);
+  }
+  return { lines: `${outputs.join("\n")}\n`, modelMatrix, worlds, fixtures: normalizedFixtures };
 }

-console.error(
-  `Discovered ${fixtures.length} fixtures (${modelMatrix.length} model-suite jobs, ${worlds.length} worlds).`,
-);
-const lines = `${outputs.join("\n")}\n`;
-if (process.env.GITHUB_OUTPUT) {
-  appendFileSync(process.env.GITHUB_OUTPUT, lines);
-} else {
-  process.stdout.write(lines);
+function normalizeFixture({ name, dir, packageJson }, worlds) {
+  const pkg = packageJson ?? {};
+  const packageJsonPath = join(dir, "package.json");
+  const selectedWorlds = validateNames(
+    pkg.e2e?.worlds ?? worlds.map((world) => world.name),
+    `${packageJsonPath}: e2e.worlds`,
+  );
+  for (const selectedWorld of selectedWorlds) {
+    if (!worlds.some((world) => world.name === selectedWorld)) {
+      throw new Error(`${packageJsonPath}: unknown e2e world "${selectedWorld}".`);
+    }
+  }
+  const modelMatrix = pkg.e2e?.modelMatrix ?? "default";
+  if (modelMatrix !== "default" && modelMatrix !== "full") {
+    throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);
+  }
+  return {
+    name,
+    dir,
+    modelMatrix,
+    additionalModels: validateNamedEntries(
+      pkg.e2e?.additionalModels ?? [],
+      `${packageJsonPath}: e2e.additionalModels`,
+      ["id"],
+      { allowEmpty: true },
+    ),
+    optionalModels: validateNames(
+      pkg.e2e?.optionalModels ?? [],
+      `${packageJsonPath}: e2e.optionalModels`,
+    ),
+    selectedWorlds,
+  };
+}
+
+function discoverFromDisk() {
+  const roots = ["e2e/fixtures", "apps/fixtures"];
+  const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8"));
+  const fixtures = [];
+  for (const root of roots) {
+    if (!existsSync(root)) continue;
+    for (const name of readdirSync(root).sort()) {
+      const dir = join(root, name);
+      if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue;
+      const packageJsonPath = join(dir, "package.json");
+      fixtures.push({
+        name,
+        dir,
+        packageJson: existsSync(packageJsonPath)
+          ? JSON.parse(readFileSync(packageJsonPath, "utf8"))
+          : undefined,
+      });
+    }
+  }
+  return { registry, fixtures };
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  const result = discoverE2eFixtures(discoverFromDisk());
+  console.error(
+    `Discovered ${result.fixtures.length} fixtures (${result.modelMatrix.length} model-suite jobs, ${result.worlds.length} worlds).`,
+  );
+  if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, result.lines);
+  else process.stdout.write(result.lines);
 }

 function validateNamedEntries(entries, key, requiredFields, options = {}) {
@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { discoverE2eFixtures } from "./discover-e2e-fixtures.mjs";
+
+const registry = {
+  models: [{ name: "model-a", id: "provider/model-a" }],
+  worlds: [{ name: "vercel" }, { name: "postgres", package: "world-postgres" }],
+};
+
+function discover(packageJson, additionalFixtures = []) {
+  return discoverE2eFixtures({
+    registry,
+    fixtures: [{ name: "fixture", dir: "fixtures/fixture", packageJson }, ...additionalFixtures],
+  });
+}
+
+function worldMatrix(result, world) {
+  return JSON.parse(result.lines.match(new RegExp(`world_matrix_${world}=(.*)`))[1]);
+}
+
+test("omitted e2e.worlds selects every registered world", () => {
+  const result = discover({});
+
+  assert.deepEqual(worldMatrix(result, "vercel"), [{ name: "fixture", dir: "fixtures/fixture" }]);
+  assert.deepEqual(worldMatrix(result, "postgres"), [
+    { name: "fixture", dir: "fixtures/fixture", world_package: "world-postgres" },
+  ]);
+});
+
+test("a registered world must retain at least one fixture", () => {
+  assert.throws(
+    () => discover({ e2e: { worlds: [] } }),
+    /No e2e fixtures select the registered world "vercel"/u,
+  );
+});
+
+test("a valid e2e.worlds subset selects only that world", () => {
+  const result = discover({ e2e: { worlds: ["postgres"] } }, [
+    {
+      name: "vercel-fixture",
+      dir: "fixtures/vercel-fixture",
+      packageJson: { e2e: { worlds: ["vercel"] } },
+    },
+  ]);
+
+  assert.deepEqual(worldMatrix(result, "vercel"), [
+    { name: "vercel-fixture", dir: "fixtures/vercel-fixture" },
+  ]);
+  assert.deepEqual(worldMatrix(result, "postgres"), [
+    { name: "fixture", dir: "fixtures/fixture", world_package: "world-postgres" },
+  ]);
+});
+
+test("an unknown e2e.worlds entry is rejected", () => {
+  assert.throws(() => discover({ e2e: { worlds: ["missing"] } }), /unknown e2e world "missing"/u);
+});
@@ -92,6 +92,9 @@ jobs:
      - name: Run unit tests
        run: pnpm test:unit

+      - name: Test e2e fixture discovery
+        run: node --test .github/scripts/discover-e2e-fixtures.test.mjs
+
  test-integration:
    name: test-integration (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
@@ -134,6 +134,7 @@ jobs:
        run: |
          mkdir -p "$EVE_EVAL_JUNIT_DIR"
          cd "${{ matrix.dir }}"
+          pnpm run --if-present e2e:prepare
          pnpm exec eve eval --strict --verbose \
            --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml"

@@ -136,6 +136,12 @@ high-volume session execution and repeated session resumption respectively.

 ## Fixtures

+The [`agent-self-modification`](./fixtures/agent-self-modification/README.md)
+fixture contains source-generation and repair examples using `eve eval`. It
+checks generated tools through real calls in fresh sessions and restores source
+after retiring the parent and child sessions. Routing-only self-modification
+coverage stays in `agent-subagents`.
+
 E2E fixtures live under `e2e/fixtures/*`. Fixture discovery also accepts
 `apps/fixtures/*` apps with an `evals/` directory, but shared development apps
 should stay out of the e2e matrix unless they intentionally own evals.
@@ -164,7 +170,9 @@ matrices from the registry:
  `e2e.optionalModels` can name selected model legs that should still run and
  report failures without blocking the aggregate check.
 - `world_matrix_<world>` — one leg per fixture for that world's suite
-  workflow. A registered world's `package` reaches the job as
+  workflow. A fixture can set `e2e.worlds` to a subset of registered world
+  names, or to `[]` when its evals require local dev behavior; omitting it
+  selects every world. A registered world's `package` reaches the job as
  `EVE_E2E_WORKFLOW_WORLD` (worlds without one, like `vercel`, use the
  deploy target's default).

@@ -183,9 +191,14 @@ once per leg, then runs one fixture directory with the leg's real model:
 ```sh
 pnpm --filter eve run build
 cd "$FIXTURE_DIR"
+pnpm run --if-present e2e:prepare
 EVE_E2E_MODEL="$MODEL" pnpm exec eve eval --strict --junit "$JUNIT_PATH"
 ```

+Fixtures with generated source can define an `e2e:prepare` script. The local
+model suite runs it before starting the eval server; the self-modification
+fixture uses it to copy the checkout's standard registry scaffold.
+
 Always build with the full `build` script (not `build:js`); only the full
 build stamps the package version into `dist`.

@@ -0,0 +1,13 @@
+node_modules
+.env*
+.eve
+.vercel
+.next
+.output
+.nitro
+dist
+.DS_Store
+*.tsbuildinfo
+.env*.local
+/agent/subagents/self-modification/
+/.eve-self-modification-eval.lock/
@@ -0,0 +1,6 @@
+node_modules
+.eve
+.next
+.output
+.nitro
+dist
@@ -0,0 +1,42 @@
+# Self-modification e2e fixture
+
+This fixture uses `eve eval` to test a parent delegating a source change to the real self-modification child, rebuilding the agent, and using the result in a new conversation. CI runs the default root model and the standard self-modification child with eve's default model. Independent parent/child model selection is not part of this fixture.
+
+The workspace root declares `@vercel/connect` as a development dependency so the bundler can resolve the self-modification extension's optional deployed credential provider through the workspace-linked `eve` package. These local cases do not use Connect credentials.
+
+Routing-only cases remain in [`agent-subagents`](../agent-subagents/evals/self-modification/), where an acceptance-only child avoids performing real integration installs.
+
+## Fixture preparation
+
+`pnpm run e2e:prepare` copies the `eve/self-modification` scaffold from this checkout using the source and target paths in `apps/docs/registry.json`. The generated `agent/subagents/self-modification/` directory is gitignored and replaced on each preparation; no registry fetch, dependency installation, or credential setup runs.
+
+The fixture's `build`, `dev`, `typecheck`, and `test:e2e` scripts prepare the scaffold before starting eve. The local e2e CI workflow also runs `e2e:prepare` before invoking `eve eval` directly. For a direct CLI invocation, prepare first. Do not prepare while an eval or dev server is running: preparation replaces the generated subtree.
+
+This exercises the current standard scaffold without duplicating it in the fixture. Registry installer behavior is outside these evals' scope. Dependencies remain declared in the fixture's `package.json`.
+
+## Cases
+
+- `create-incident-triage.eval.ts` creates an incident-triage tool and checks precedence and threshold rules across typed inputs.
+- `create-shipping-quote.eval.ts` creates a quote calculator and checks destination, started-kilogram, free-shipping, and expedited pricing boundaries.
+- `repair-order-total.eval.ts` first reproduces quantity and discount errors in the fixture's existing tool, asks self-mod to investigate the incorrect invoice, and verifies the fix, cent rounding, and single-item/empty-order regressions.
+
+Each case checks actual tool inputs and outputs, not the assistant's claim that the work succeeded. Cases also reject source changes outside their specified tool file. The arithmetic cases use synthetic data and do not require external services. These checks cover the specified behavior; they are not a general security audit of generated code.
+
+## Adding a case
+
+Wrap source-mutating cases in `withSelfModification(t, async (selfMod) => { ... })` from `evals/self-modification/harness.ts`:
+
+1. Establish the initial state. For a repair, invoke the existing tool and assert its known incorrect output before asking for a fix.
+2. Use `selfMod.request(prompt)` to start the parent and follow the delegated child. Describe the user-visible requirement or symptom, not a prescribed patch.
+3. Use `selfMod.assertOnlyChanged(paths)` to check the permitted edit scope, then `selfMod.apply()` to force a runtime rebuild.
+4. Use `selfMod.verify(prompt)` to invoke the tool in a fresh session. Assert the call's input and structured output with `requireToolCall`, including boundary cases and previously working behavior.
+
+The harness snapshots the complete `agent/` tree, tracks sessions, and retires them before restoring source. Restoration removes unexpected files and restores deleted files, binary contents, and file modes. Source watching is suspended during restoration. If session retirement or restoration fails, later mutation cases fail instead of continuing against uncertain state; failed retirement retains the backup path in the error.
+
+Keep `maxConcurrency: 1`. The harness also serializes cleanup that continues after an eval timeout and acquires a checkout lock before snapshotting source. A concurrent `eve eval` process fails before mutation. If cleanup cannot safely restore source, the lock remains with owner diagnostics; remove it only after inspecting the retained backup and confirming no eval or development server is mutating the fixture.
+
+Forced rebuilds isolate source-authoring and runtime correctness. These cases do not verify automatic hot-reload timing or deployed proposal/merge behavior. Real-model e2e runs belong in CI. The fixture-only cleanup tests need no model or running server:
+
+```sh
+pnpm --filter agent-self-modification test:scenario
+```
@@ -0,0 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
+import { defineAgent } from "eve";
+
+export default defineAgent({
+  ...e2eAgentConfig(),
+  reasoning: "high",
+});
@@ -0,0 +1,12 @@
+import { eveChannel } from "eve/channels/eve";
+
+export default eveChannel({
+  auth: () => ({
+    attributes: { fixture: "self-modification" },
+    authenticator: "e2e-fixture",
+    issuer: "e2e",
+    principalId: "self-modification-e2e-user",
+    principalType: "user",
+    subject: "self-modification-e2e-user",
+  }),
+});
@@ -0,0 +1,3 @@
+# Identity
+
+You are a helpful assistant.
@@ -0,0 +1,30 @@
+import { defineTool } from "eve/tools";
+import { never } from "eve/tools/approval";
+import { z } from "zod";
+
+export default defineTool({
+  description:
+    "Calculate an order subtotal and percentage discount in cents without placing or charging an order.",
+  inputSchema: z.object({
+    items: z
+      .array(
+        z.object({
+          sku: z.string().min(1).max(64),
+          unitPriceCents: z.number().int().min(0).max(1000000),
+          quantity: z.number().int().min(1).max(100),
+        }),
+      )
+      .max(100),
+    discountBps: z.number().int().min(0).max(5000),
+  }),
+  approval: never(),
+  async execute({ items, discountBps }) {
+    const subtotalCents = items.reduce((total, item) => total + item.unitPriceCents, 0);
+    const discountCents = Math.floor((subtotalCents * discountBps) / 10000);
+    return {
+      subtotalCents,
+      discountCents,
+      totalCents: subtotalCents - discountCents,
+    };
+  },
+});
@@ -0,0 +1,8 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
+import { defineEvalConfig } from "eve/evals";
+
+export default defineEvalConfig({
+  judge: { model: e2eJudgeModel() },
+  maxConcurrency: 1,
+  timeoutMs: 240_000,
+});
@@ -0,0 +1,100 @@
+import { defineEval } from "eve/evals";
+
+import { withSelfModification } from "./harness";
+
+const TOOL_NAME = "eval_triage_incident";
+
+export default defineEval({
+  tags: ["real-model"],
+  description:
+    "Self-mod creates an incident-triage tool with typed inputs and deterministic priority rules.",
+
+  async test(t) {
+    await withSelfModification(t, async (selfMod) => {
+      await selfMod.request(
+        [
+          `Alice's support team needs a reusable ${TOOL_NAME} action for classifying incidents in future conversations.`,
+          "Accept impact as one of outage, degraded, or cosmetic; affectedUsers as an integer from 0 through 1000000; and dataAtRisk and workaroundAvailable as booleans.",
+          "Assign P0 when data is at risk, or when an outage affects at least 100 users and no workaround exists.",
+          "Otherwise assign P1 for any outage or when degraded service affects at least 1000 users; assign P2 for all other incidents.",
+          "Return structured data with priority and responseMinutes, using 15 minutes for P0, 60 for P1, and 480 for P2.",
+          "This action only classifies the supplied facts. It must not page responders, modify incidents, or contact external services.",
+        ].join(" "),
+      );
+      await selfMod.readSource(`tools/${TOOL_NAME}.ts`);
+      await selfMod.assertOnlyChanged([`tools/${TOOL_NAME}.ts`]);
+      await selfMod.apply();
+
+      await Promise.all(
+        (
+          [
+            [
+              {
+                impact: "outage",
+                affectedUsers: 100,
+                dataAtRisk: false,
+                workaroundAvailable: false,
+              },
+              "P0",
+              15,
+            ],
+            [
+              { impact: "cosmetic", affectedUsers: 1, dataAtRisk: true, workaroundAvailable: true },
+              "P0",
+              15,
+            ],
+            [
+              {
+                impact: "outage",
+                affectedUsers: 99,
+                dataAtRisk: false,
+                workaroundAvailable: false,
+              },
+              "P1",
+              60,
+            ],
+            [
+              {
+                impact: "degraded",
+                affectedUsers: 1000,
+                dataAtRisk: false,
+                workaroundAvailable: true,
+              },
+              "P1",
+              60,
+            ],
+            [
+              {
+                impact: "degraded",
+                affectedUsers: 999,
+                dataAtRisk: false,
+                workaroundAvailable: false,
+              },
+              "P2",
+              480,
+            ],
+            [
+              {
+                impact: "cosmetic",
+                affectedUsers: 0,
+                dataAtRisk: false,
+                workaroundAvailable: false,
+              },
+              "P2",
+              480,
+            ],
+          ] as const
+        ).map(async ([input, priority, responseMinutes]) => {
+          const turn = await selfMod.verify(
+            `Classify this incident once with ${TOOL_NAME}: ${JSON.stringify(input)}. Report the classification without taking operational action.`,
+          );
+          turn.requireToolCall(TOOL_NAME, {
+            input,
+            output: { priority, responseMinutes },
+          });
+        }),
+      );
+      t.succeeded();
+    });
+  },
+});
@@ -0,0 +1,87 @@
+import { defineEval } from "eve/evals";
+
+import { withSelfModification } from "./harness";
+
+const TOOL_NAME = "eval_shipping_quote";
+
+export default defineEval({
+  tags: ["real-model"],
+  description:
+    "A generated shipping tool applies destination, weight, threshold, and expedited pricing rules.",
+
+  async test(t) {
+    await withSelfModification(t, async (selfMod) => {
+      await selfMod.request(
+        [
+          `Alice needs a reusable ${TOOL_NAME} action for preparing order quotes in future conversations.`,
+          "Accept subtotalCents as an integer from 0 through 1000000, weightGrams as an integer from 1 through 50000, destination as domestic or international, and expedited as a boolean.",
+          "Domestic standard shipping is 500 cents plus 100 cents for each started kilogram. Waive that entire standard charge when the subtotal is at least 5000 cents.",
+          "International standard shipping is 1500 cents plus 300 cents for each started kilogram and is never waived.",
+          "Expedited delivery adds 1200 cents after calculating the standard charge, including when domestic standard shipping is waived.",
+          "Return structured data with standardShippingCents, expeditedSurchargeCents, shippingCents, and totalCents.",
+          "This action only calculates a quote. It must not place orders, charge customers, or contact external services.",
+        ].join(" "),
+      );
+      await selfMod.readSource(`tools/${TOOL_NAME}.ts`);
+      await selfMod.assertOnlyChanged([`tools/${TOOL_NAME}.ts`]);
+      await selfMod.apply();
+
+      await Promise.all(
+        (
+          [
+            [
+              { subtotalCents: 4999, weightGrams: 1000, destination: "domestic", expedited: false },
+              600,
+              0,
+            ],
+            [
+              { subtotalCents: 5000, weightGrams: 1001, destination: "domestic", expedited: false },
+              0,
+              0,
+            ],
+            [
+              { subtotalCents: 5000, weightGrams: 1001, destination: "domestic", expedited: true },
+              0,
+              1200,
+            ],
+            [
+              {
+                subtotalCents: 10000,
+                weightGrams: 2001,
+                destination: "international",
+                expedited: false,
+              },
+              2400,
+              0,
+            ],
+            [
+              {
+                subtotalCents: 2500,
+                weightGrams: 1,
+                destination: "international",
+                expedited: true,
+              },
+              1800,
+              1200,
+            ],
+          ] as const
+        ).map(async ([input, standardShippingCents, expeditedSurchargeCents]) => {
+          const shippingCents = standardShippingCents + expeditedSurchargeCents;
+          const turn = await selfMod.verify(
+            `Please use ${TOOL_NAME} once to quote this order: ${JSON.stringify(input)}. Report the quote without placing an order.`,
+          );
+          turn.requireToolCall(TOOL_NAME, {
+            input,
+            output: {
+              standardShippingCents,
+              expeditedSurchargeCents,
+              shippingCents,
+              totalCents: input.subtotalCents + shippingCents,
+            },
+          });
+        }),
+      );
+      t.succeeded();
+    });
+  },
+});
@@ -0,0 +1,331 @@
+import { randomUUID } from "node:crypto";
+import {
+  cp,
+  lstat,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readdir,
+  readlink,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+
+import type { EveEvalContext, EveEvalLiveTurn, EveEvalTurn } from "eve/evals";
+
+const SELF_MODIFICATION_AGENT = "self-modification";
+const CLEANUP_TIMEOUT_MS = 30_000;
+const LOCK_DIRECTORY = ".eve-self-modification-eval.lock";
+// Each eval entry bundles its relative imports separately. Share the lock across those copies.
+const shared = globalThis as typeof globalThis & {
+  __eveSelfModificationEvalIsolation?: { previousCleanup: Promise<void>; failure?: unknown };
+};
+const isolation = (shared.__eveSelfModificationEvalIsolation ??= {
+  previousCleanup: Promise.resolve(),
+});
+
+export interface SelfModificationRun {
+  readonly child: EveEvalTurn;
+  readonly parent: EveEvalTurn;
+}
+
+/** Serializes source mutation, including cleanup that outlives an eval timeout. */
+export async function withSelfModification(
+  t: EveEvalContext,
+  test: (harness: SelfModificationHarness) => Promise<void>,
+): Promise<void> {
+  if (t.target.kind !== "local") {
+    t.skip("Self-modification evals require a local development target.");
+  }
+
+  const preceding = isolation.previousCleanup;
+  let release!: () => void;
+  isolation.previousCleanup = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  await preceding;
+  let checkoutLock: Awaited<ReturnType<typeof acquireCheckoutLock>> | undefined;
+  let retainCheckoutLock = false;
+  try {
+    t.signal.throwIfAborted();
+    if (isolation.failure !== undefined) {
+      throw new Error("A previous self-modification eval could not clean up safely.", {
+        cause: isolation.failure,
+      });
+    }
+    checkoutLock = await acquireCheckoutLock(process.cwd());
+    const harness = await SelfModificationHarness.create(t, {
+      onBackupCreated: checkoutLock.recordBackup,
+    });
+    let failure: { error: unknown } | undefined;
+    try {
+      await test(harness);
+    } catch (error) {
+      failure = { error };
+    }
+    try {
+      await harness.close();
+    } catch (error) {
+      isolation.failure = error;
+      retainCheckoutLock = true;
+      if (failure !== undefined) {
+        throw new AggregateError(
+          [failure.error, error],
+          "Self-modification eval and cleanup failed.",
+        );
+      }
+      throw error;
+    }
+    if (failure !== undefined) throw failure.error;
+  } finally {
+    try {
+      if (!retainCheckoutLock) await checkoutLock?.release();
+    } finally {
+      release();
+    }
+  }
+}
+
+export class SelfModificationHarness {
+  readonly #t: EveEvalContext;
+  readonly #sourceRoot: string;
+  readonly #backupRoot: string;
+  readonly #turns = new Set<EveEvalLiveTurn>();
+
+  private constructor(t: EveEvalContext, sourceRoot: string, backupRoot: string) {
+    this.#t = t;
+    this.#sourceRoot = sourceRoot;
+    this.#backupRoot = backupRoot;
+  }
+
+  static async create(
+    t: EveEvalContext,
+    sourceRootOrOptions:
+      | string
+      | {
+          sourceRoot?: string;
+          onBackupCreated?: (backupRoot: string) => Promise<void>;
+        } = {},
+  ): Promise<SelfModificationHarness> {
+    const options =
+      typeof sourceRootOrOptions === "string"
+        ? { sourceRoot: sourceRootOrOptions }
+        : sourceRootOrOptions;
+    const sourceRoot = options.sourceRoot ?? join(process.cwd(), "agent");
+    const backupRoot = await mkdtemp(join(tmpdir(), "eve-selfmod-eval-"));
+    try {
+      await options.onBackupCreated?.(backupRoot);
+      await cp(sourceRoot, join(backupRoot, "agent"), { recursive: true, verbatimSymlinks: true });
+      return new SelfModificationHarness(t, sourceRoot, backupRoot);
+    } catch (error) {
+      await rm(backupRoot, { recursive: true, force: true });
+      throw error;
+    }
+  }
+
+  async request(prompt: string): Promise<SelfModificationRun> {
+    const liveParent = await this.#t.start(prompt);
+    this.#turns.add(liveParent);
+    let continuation: EveEvalLiveTurn | undefined;
+    const called = await liveParent
+      .waitForEvent("subagent.called", {
+        data: { name: SELF_MODIFICATION_AGENT },
+      })
+      .catch(async (error) => {
+        this.#t.signal.throwIfAborted();
+        const parent = await liveParent.result();
+        parent.expectOk();
+        parent.requireToolCall(SELF_MODIFICATION_AGENT);
+        const startIndex = liveParent.session.state?.streamIndex;
+        if (startIndex === undefined) throw error;
+        continuation = this.#t.target.watchTurn(parent.sessionId, { startIndex });
+        this.#turns.add(continuation);
+        return continuation.waitForEvent("subagent.called", {
+          data: { name: SELF_MODIFICATION_AGENT },
+        });
+      });
+    const liveChild = this.#t.target.watchTurn(called.data.childSessionId);
+    this.#turns.add(liveChild);
+    const [parent, child] = await Promise.all([
+      liveParent.result(),
+      liveChild.result(),
+      continuation?.result().then((turn) => turn.expectOk()),
+    ]);
+    parent.expectOk();
+    this.#t.calledSubagent(SELF_MODIFICATION_AGENT);
+    child.expectOk();
+    return { child, parent };
+  }
+
+  /** Uses a fresh conversation so the model cannot answer from the authoring exchange alone. */
+  async verify(prompt: string): Promise<EveEvalTurn> {
+    const live = await this.#t.newSession().start(prompt);
+    this.#turns.add(live);
+    const turn = await live.result();
+    turn.expectOk();
+    return turn;
+  }
+
+  async apply(): Promise<void> {
+    const response = await this.#post("rebuild?force=1", this.#t.signal);
+    const body = (await response.json()) as { revision?: unknown };
+    if (typeof body.revision !== "string" || body.revision.length === 0) {
+      throw new Error("Self-modification rebuild did not return a runtime revision.");
+    }
+    this.#t.log(`Self-modification runtime revision: ${body.revision}`);
+  }
+
+  async assertOnlyChanged(sourcePaths: readonly string[]): Promise<void> {
+    const original = await sourceFiles(join(this.#backupRoot, "agent"));
+    const current = await sourceFiles(this.#sourceRoot);
+    for (const path of new Set([...original.keys(), ...current.keys()])) {
+      if (sourcePaths.includes(path)) continue;
+      const before = original.get(path);
+      const after = current.get(path);
+      if (before === undefined || after === undefined || !before.equals(after)) {
+        throw new Error(`Self-modification changed an unrelated source file: ${path}`);
+      }
+    }
+  }
+
+  readSource(sourcePath: string): Promise<string> {
+    return readFile(this.#resolve(sourcePath), "utf8");
+  }
+
+  async writeSource(sourcePath: string, content: string): Promise<void> {
+    const path = this.#resolve(sourcePath);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, content, "utf8");
+  }
+
+  async close(): Promise<void> {
+    // Reset retires sessions and waits for their command hooks to close; cancel only requests it.
+    // Discover even children emitted just before a failed/aborted parent stream was observed.
+    const sessionIds = new Set<string>();
+    for (const turn of this.#turns) {
+      sessionIds.add(turn.sessionId);
+      for (const event of turn.events) {
+        if (event.type === "subagent.called") sessionIds.add(event.data.childSessionId);
+      }
+    }
+    const signal = AbortSignal.timeout(CLEANUP_TIMEOUT_MS);
+    const results = await Promise.allSettled(
+      [...sessionIds].map(async (sessionId) => {
+        const response = await this.#t.target.fetch(
+          `/eve/v1/session/${encodeURIComponent(sessionId)}/reset`,
+          {
+            method: "POST",
+            signal,
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify({ reason: "Self-modification eval cleanup" }),
+          },
+        );
+        if (!response.ok)
+          throw new Error(`Failed to retire self-modification session: ${response.status}`);
+      }),
+    );
+    const failures = results.flatMap((result) =>
+      result.status === "rejected" ? [result.reason] : [],
+    );
+    if (failures.length > 0) {
+      throw new AggregateError(
+        failures,
+        `Sessions could not be retired; source backup retained at ${this.#backupRoot}`,
+      );
+    }
+
+    const lease = randomUUID();
+    await this.#post(`suspend?lease=${lease}`, signal);
+    try {
+      await rm(this.#sourceRoot, { recursive: true, force: true });
+      await cp(join(this.#backupRoot, "agent"), this.#sourceRoot, {
+        recursive: true,
+        verbatimSymlinks: true,
+      });
+    } finally {
+      await this.#post(`resume?lease=${lease}`, signal);
+    }
+    await this.#post("rebuild?force=1", signal);
+    await rm(this.#backupRoot, { recursive: true, force: true });
+  }
+
+  #resolve(sourcePath: string): string {
+    if (!sourcePath || sourcePath.startsWith("/") || sourcePath.split(/[\\/]/u).includes("..")) {
+      throw new Error(`Self-modification eval source paths must be relative: ${sourcePath}`);
+    }
+    return join(this.#sourceRoot, sourcePath);
+  }
+
+  async #post(operation: string, signal: AbortSignal): Promise<Response> {
+    const response = await this.#t.target.fetch(`/eve/v1/dev/runtime-artifacts/${operation}`, {
+      method: "POST",
+      signal,
+    });
+    if (!response.ok) throw new Error(`Self-modification ${operation} failed: ${response.status}`);
+    return response;
+  }
+}
+
+async function acquireCheckoutLock(root: string): Promise<{
+  recordBackup: (backupRoot: string) => Promise<void>;
+  release: () => Promise<void>;
+}> {
+  const lockDirectory = join(root, LOCK_DIRECTORY);
+  const owner = { pid: process.pid, startedAt: new Date().toISOString() };
+  try {
+    await mkdir(lockDirectory);
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "EEXIST") {
+      throw new Error(
+        `Another self-modification eval owns this checkout. Remove ${lockDirectory} only after confirming no eval or development server is mutating it.`,
+      );
+    }
+    throw error;
+  }
+  const writeOwner = (backupRoot?: string) =>
+    writeFile(
+      join(lockDirectory, "owner.json"),
+      `${JSON.stringify({ ...owner, backupRoot }, null, 2)}\n`,
+    );
+  try {
+    await writeOwner();
+  } catch (error) {
+    await rm(lockDirectory, { recursive: true, force: true });
+    throw error;
+  }
+  return {
+    recordBackup: writeOwner,
+    release: () => rm(lockDirectory, { recursive: true, force: true }),
+  };
+}
+
+async function sourceFiles(root: string): Promise<Map<string, Buffer>> {
+  const files = new Map<string, Buffer>();
+  let entries = 0;
+  let bytes = 0;
+  async function visit(directory: string, depth: number): Promise<void> {
+    if (depth > 16) throw new Error("Self-modification source exceeds 16 nested directories.");
+    for (const entry of await readdir(join(root, directory), { withFileTypes: true })) {
+      if (++entries > 256) throw new Error("Self-modification source exceeds 256 entries.");
+      const path = directory ? `${directory}/${entry.name}` : entry.name;
+      const absolute = join(root, path);
+      if (entry.isDirectory()) {
+        await visit(path, depth + 1);
+      } else {
+        const stat = await lstat(absolute);
+        bytes += stat.size;
+        if (bytes > 4 * 1024 * 1024) throw new Error("Self-modification source exceeds 4 MiB.");
+        if (!stat.isFile() && !stat.isSymbolicLink())
+          throw new Error(`Unexpected source entry: ${path}`);
+        const content = entry.isSymbolicLink()
+          ? Buffer.from(`symlink:${await readlink(absolute)}`)
+          : await readFile(absolute);
+        files.set(path, Buffer.concat([Buffer.from(`${stat.mode}:`), content]));
+      }
+    }
+  }
+  await visit("", 0);
+  return files;
+}
@@ -0,0 +1,79 @@
+import { defineEval } from "eve/evals";
+
+import { withSelfModification } from "./harness";
+
+const TOOL_NAME = "eval_order_total";
+const ORDER = {
+  items: [
+    { sku: "desk-lamp", unitPriceCents: 1200, quantity: 3 },
+    { sku: "bulb-pack", unitPriceCents: 500, quantity: 2 },
+  ],
+  discountBps: 1250,
+};
+
+export default defineEval({
+  tags: ["real-model"],
+  description:
+    "Self-mod diagnoses incorrect quantity and discount calculations without regressing order boundaries.",
+
+  async test(t) {
+    await withSelfModification(t, async (selfMod) => {
+      const before = await selfMod.verify(
+        `Alice is checking an order. Call ${TOOL_NAME} once with ${JSON.stringify(ORDER)} and report the result as returned, without correcting it or changing the tool.`,
+      );
+      before.requireToolCall(TOOL_NAME, {
+        input: ORDER,
+        output: { subtotalCents: 1700, discountCents: 212, totalCents: 1488 },
+      });
+
+      await selfMod.request(
+        [
+          `Alice found a problem with ${TOOL_NAME}: three 1200-cent desk lamps and two 500-cent bulb packs with a 12.5% discount came back with a 1700-cent subtotal and a 1488-cent total.`,
+          "The invoice should use quantities, then round the percentage discount down to whole cents: subtotal 4600, discount 575, total 4025.",
+          "Please investigate and fix the tool for future orders rather than correcting just this response.",
+          "Keep its existing input validation and calculation-only behavior; it must not place orders or charge customers.",
+        ].join(" "),
+      );
+      await selfMod.assertOnlyChanged([`tools/${TOOL_NAME}.ts`]);
+      await selfMod.apply();
+
+      await Promise.all(
+        (
+          [
+            [ORDER, 4600, 575],
+            [
+              {
+                items: [{ sku: "monitor", unitPriceCents: 333, quantity: 3 }],
+                discountBps: 3333,
+              },
+              999,
+              332,
+            ],
+            [
+              {
+                items: [{ sku: "cable", unitPriceCents: 250, quantity: 1 }],
+                discountBps: 0,
+              },
+              250,
+              0,
+            ],
+            [{ items: [], discountBps: 5000 }, 0, 0],
+          ] as const
+        ).map(async ([input, subtotalCents, discountCents]) => {
+          const turn = await selfMod.verify(
+            `Call ${TOOL_NAME} once with ${JSON.stringify(input)} and report its result.`,
+          );
+          turn.requireToolCall(TOOL_NAME, {
+            input,
+            output: {
+              subtotalCents,
+              discountCents,
+              totalCents: subtotalCents - discountCents,
+            },
+          });
+        }),
+      );
+      t.succeeded();
+    });
+  },
+});
@@ -0,0 +1,30 @@
+{
+  "name": "agent-self-modification",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "e2e:prepare": "node scripts/prepare.mjs",
+    "build": "pnpm run e2e:prepare && eve build",
+    "dev": "pnpm run e2e:prepare && eve dev",
+    "start": "eve start",
+    "typecheck": "pnpm run build && tsc",
+    "test:e2e": "pnpm run e2e:prepare && eve eval --strict",
+    "test:scenario": "node --test test/*.scenario.test.mjs"
+  },
+  "dependencies": {
+    "@eve-e2e/config": "workspace:*",
+    "@vercel/connect": "catalog:",
+    "@workflow/world-postgres": "catalog:",
+    "eve": "workspace:*",
+    "just-bash": "3.1.0",
+    "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@types/node": "catalog:",
+    "typescript": "catalog:"
+  },
+  "e2e": {
+    "worlds": []
+  }
+}
@@ -0,0 +1,46 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const fixtureRoot = fileURLToPath(new URL("../", import.meta.url));
+const repoRoot = resolve(fixtureRoot, "../../..");
+const targetDirectory = "agent/subagents/self-modification";
+
+/** Materialize the checkout's registry scaffold without network access or installer setup. */
+export async function prepareSelfModification(options = {}) {
+  const fixture = options.fixtureRoot ?? fixtureRoot;
+  const docs = resolve(options.repoRoot ?? repoRoot, "apps/docs");
+  const registry = JSON.parse(await readFile(resolve(docs, "registry.json"), "utf8"));
+  const item = registry.items.find((item) => item.name === "eve/self-modification");
+  if (!item?.files?.length)
+    throw new Error("eve/self-modification has no registry scaffold files.");
+
+  const destination = resolve(fixture, targetDirectory);
+  const files = await Promise.all(
+    item.files.map(async (file) => {
+      const source = resolve(docs, file.path);
+      const target = resolve(fixture, file.target);
+      requireDescendant(resolve(docs, "registry"), source);
+      requireDescendant(destination, target);
+      return { target, contents: await readFile(source) };
+    }),
+  );
+
+  // Only this generated subtree is disposable; leave the fixture's authored tools intact.
+  await rm(destination, { recursive: true, force: true });
+  for (const { target, contents } of files) {
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, contents);
+  }
+}
+
+function requireDescendant(root, path) {
+  const child = relative(root, path);
+  if (!child || child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) {
+    throw new Error(`Registry scaffold path is outside ${root}: ${path}`);
+  }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  await prepareSelfModification();
+}
@@ -0,0 +1,417 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+import {
+  SelfModificationHarness,
+  withSelfModification,
+} from "../evals/self-modification/harness.ts";
+
+async function sourceTree() {
+  const root = await mkdtemp(join(tmpdir(), "eve-selfmod-test-"));
+  await mkdir(join(root, "nested"), { recursive: true });
+  await writeFile(join(root, "keep.txt"), "baseline");
+  await writeFile(join(root, "nested", "delete.txt"), "delete me");
+  await writeFile(join(root, "nested", "binary.bin"), Buffer.from([0, 1, 2, 255]));
+  return root;
+}
+
+async function temporaryCheckout(t, prefix) {
+  const root = await mkdtemp(join(tmpdir(), prefix));
+  const cwd = process.cwd();
+  await mkdir(join(root, "agent"));
+  process.chdir(root);
+  t.after(async () => {
+    process.chdir(cwd);
+    await rm(root, { recursive: true, force: true });
+  });
+  return root;
+}
+
+function response(body = {}) {
+  return new Response(JSON.stringify(body), { status: 200 });
+}
+
+function context(target, signal = new AbortController().signal) {
+  return {
+    signal,
+    target,
+    log() {},
+    calledSubagent() {},
+    start: async () => {
+      throw new Error("start was not expected");
+    },
+    newSession: () => {
+      throw new Error("newSession was not expected");
+    },
+  };
+}
+
+function completedTurn(sessionId, events = []) {
+  return {
+    sessionId,
+    events,
+    expectOk() {},
+    calledSubagent() {},
+    requireToolCall() {},
+  };
+}
+
+function liveTurn(sessionId, events = []) {
+  return {
+    sessionId,
+    events,
+    async result() {
+      return completedTurn(sessionId, events);
+    },
+  };
+}
+
+function targetFor({ calls = [], turns = {} } = {}) {
+  return {
+    kind: "local",
+    async fetch(path, options = {}) {
+      calls.push({ path, options });
+      return response({ revision: "revision-after-cleanup" });
+    },
+    watchTurn(sessionId) {
+      return turns[sessionId];
+    },
+  };
+}
+
+async function withHarness(run, setup = {}, makeContext = context) {
+  const root = await sourceTree();
+  const calls = [];
+  const target = targetFor({ ...setup, calls });
+  const harness = await SelfModificationHarness.create(makeContext(target), root);
+  let closed = false;
+  const close = async () => {
+    if (!closed) {
+      closed = true;
+      await harness.close();
+    }
+  };
+  try {
+    return await run({ harness, root, calls, target, close });
+  } finally {
+    await close();
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+async function assertTreeRestored(root) {
+  assert.equal(await readFile(join(root, "keep.txt"), "utf8"), "baseline");
+  assert.equal(await readFile(join(root, "nested", "delete.txt"), "utf8"), "delete me");
+  assert.deepEqual(await readFile(join(root, "nested", "binary.bin")), Buffer.from([0, 1, 2, 255]));
+  await assert.rejects(readFile(join(root, "unexpected.txt")));
+}
+
+test("close retires every session before restoring the complete source tree", async () => {
+  const child = liveTurn("child");
+  const verification = liveTurn("verification");
+  const parentEvent = {
+    type: "subagent.called",
+    data: { name: "self-modification", childSessionId: child.sessionId },
+  };
+  const parent = liveTurn("parent", [parentEvent]);
+
+  await withHarness(
+    async ({ harness, root, calls, target, close }) => {
+      let parentFinished = false;
+      parent.result = async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+        parentFinished = true;
+        return completedTurn(parent.sessionId, [parentEvent]);
+      };
+      parent.waitForEvent = async () => ({ data: parentEvent.data });
+      target.watchTurn = () => {
+        assert.equal(parentFinished, false);
+        return child;
+      };
+
+      await harness.request("make a change");
+      await harness.verify("verify it");
+      await writeFile(join(root, "keep.txt"), "changed");
+      await rm(join(root, "nested", "delete.txt"));
+      await writeFile(join(root, "nested", "binary.bin"), Buffer.from([9, 8, 7]));
+      await writeFile(join(root, "unexpected.txt"), "mutation");
+
+      const retiring = Promise.withResolvers();
+      const released = Promise.withResolvers();
+      const fetch = target.fetch;
+      target.fetch = async (path, options) => {
+        if (path.endsWith("/child/reset")) {
+          retiring.resolve();
+          await released.promise;
+        }
+        return fetch(path, options);
+      };
+      const closing = close();
+      await retiring.promise;
+      assert.equal(await readFile(join(root, "unexpected.txt"), "utf8"), "mutation");
+      assert.equal(
+        calls.some((call) => call.path.includes("/suspend?")),
+        false,
+      );
+      released.resolve();
+      await closing;
+
+      assert.deepEqual(
+        calls
+          .filter((call) => call.path.endsWith("/reset"))
+          .map((call) => call.path)
+          .sort(),
+        [parent, child, verification]
+          .map((turn) => `/eve/v1/session/${turn.sessionId}/reset`)
+          .sort(),
+      );
+      assert.ok(
+        calls.findLastIndex((call) => call.path.endsWith("/reset")) <
+          calls.findIndex((call) => call.path.includes("/suspend?lease=")),
+      );
+      await assertTreeRestored(root);
+    },
+    {},
+    (target) => {
+      const t = context(target);
+      t.start = async () => parent;
+      t.newSession = () => ({ start: async () => verification });
+      return t;
+    },
+  );
+});
+
+test("one reset failure still retires other sessions and leaves unsafe source untouched", async () => {
+  const root = await sourceTree();
+  const calls = [];
+  const target = targetFor({ calls });
+  target.fetch = async (path, options = {}) => {
+    calls.push({ path, options });
+    if (path.includes("bad/reset")) return new Response("no", { status: 500 });
+    return response({ revision: "revision" });
+  };
+  const parentEvent = {
+    type: "subagent.called",
+    data: { name: "self-modification", childSessionId: "bad" },
+  };
+  // The public request path is used to populate both tracked sessions.
+  const liveParent = {
+    sessionId: "parent",
+    events: [parentEvent],
+    async waitForEvent() {
+      return { data: parentEvent.data };
+    },
+    async result() {
+      return completedTurn("parent", [parentEvent]);
+    },
+  };
+  target.watchTurn = () => ({
+    sessionId: "bad",
+    events: [],
+    async result() {
+      return completedTurn("bad");
+    },
+  });
+  const t = context(target);
+  t.start = async () => liveParent;
+  const populated = await SelfModificationHarness.create(t, root);
+  await populated.request("mutate");
+  await writeFile(join(root, "keep.txt"), "unsafe mutation");
+  await assert.rejects(populated.close(), /could not be retired/);
+  assert.equal(await readFile(join(root, "keep.txt"), "utf8"), "unsafe mutation");
+  assert.equal(calls.filter((call) => call.path.endsWith("/reset")).length, 2);
+  assert.equal(
+    calls.some((call) => call.path.includes("/suspend?")),
+    false,
+  );
+  target.fetch = targetFor({ calls }).fetch;
+  await populated.close();
+  await rm(root, { recursive: true, force: true });
+});
+
+test("close uses its own live cleanup signal after the eval signal is aborted", async () => {
+  const controller = new AbortController();
+  const calls = [];
+  const root = await sourceTree();
+  const target = targetFor({ calls });
+  const harness = await SelfModificationHarness.create(context(target, controller.signal), root);
+  controller.abort();
+  await harness.close();
+  assert.ok(calls.length > 0);
+  assert.ok(
+    calls.every(({ options }) => options.signal !== controller.signal && !options.signal.aborted),
+  );
+  await rm(root, { recursive: true, force: true });
+});
+
+test("source paths reject absolute paths and traversal", async () => {
+  await withHarness(async ({ harness }) => {
+    for (const path of ["../outside", "nested/../../outside", "/absolute", "nested\\..\\outside"]) {
+      assert.throws(() => harness.readSource(path), /must be relative/);
+      await assert.rejects(harness.writeSource(path, "bad"), /must be relative/);
+    }
+  });
+});
+
+test("assertOnlyChanged allows listed modifications and rejects unexpected or deleted files", async () => {
+  await withHarness(async ({ harness, root, close }) => {
+    await writeFile(join(root, "keep.txt"), "allowed");
+    await harness.assertOnlyChanged(["keep.txt"]);
+    await writeFile(join(root, "unexpected.txt"), "bad");
+    await assert.rejects(
+      harness.assertOnlyChanged(["keep.txt"]),
+      /unrelated source file: unexpected.txt/,
+    );
+    await rm(join(root, "nested", "delete.txt"));
+    await assert.rejects(
+      harness.assertOnlyChanged(["keep.txt"]),
+      /unrelated source file: nested\/delete.txt/,
+    );
+    await close();
+  });
+});
+
+test("separate eval bundles serialize cleanup before the next source snapshot", async (t) => {
+  const otherBundle = await import("../evals/self-modification/harness.ts?other-eval");
+  const root = await temporaryCheckout(t, "eve-selfmod-serial-");
+  await writeFile(join(root, "agent", "instructions.md"), "baseline");
+  const restoring = Promise.withResolvers();
+  const release = Promise.withResolvers();
+  const target = targetFor();
+  const fetch = target.fetch;
+  target.fetch = async (path, options) => {
+    if (path.includes("/suspend?")) {
+      restoring.resolve();
+      await release.promise;
+    }
+    return fetch(path, options);
+  };
+  const first = withSelfModification(context(target), async (harness) => {
+    await harness.writeSource("instructions.md", "changed");
+  });
+  await restoring.promise;
+  let secondStarted = false;
+  const second = otherBundle.withSelfModification(context(targetFor()), async (harness) => {
+    secondStarted = true;
+    assert.equal(await harness.readSource("instructions.md"), "baseline");
+  });
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal(secondStarted, false);
+  release.resolve();
+  await Promise.all([first, second]);
+  assert.equal(secondStarted, true);
+});
+
+test("checkout lock rejects another eval process before source mutation", async (t) => {
+  const root = await temporaryCheckout(t, "eve-selfmod-locked-");
+  await mkdir(join(root, ".eve-self-modification-eval.lock"));
+  await writeFile(join(root, "agent", "instructions.md"), "baseline");
+  let started = false;
+  await assert.rejects(
+    withSelfModification(context(targetFor()), async () => {
+      started = true;
+    }),
+    /Another self-modification eval owns this checkout/,
+  );
+  assert.equal(started, false);
+  assert.equal(await readFile(join(root, "agent", "instructions.md"), "utf8"), "baseline");
+});
+
+test("a failed eval restores source, releases its checkout lock, and preserves the failure", async (t) => {
+  const root = await temporaryCheckout(t, "eve-selfmod-failed-");
+  await writeFile(join(root, "agent", "instructions.md"), "baseline");
+  const failure = new Error("authoring failed");
+  await assert.rejects(
+    withSelfModification(context(targetFor()), async (harness) => {
+      await harness.writeSource("instructions.md", "changed");
+      await harness.writeSource("tools/unexpected.ts", "unexpected");
+      throw failure;
+    }),
+    (error) => error === failure,
+  );
+  assert.equal(await readFile(join(root, "agent", "instructions.md"), "utf8"), "baseline");
+  await assert.rejects(readFile(join(root, "agent", "tools", "unexpected.ts")));
+  await assert.rejects(readFile(join(root, ".eve-self-modification-eval.lock", "owner.json")));
+});
+
+test("apply rejects a rebuild response without a runtime revision", async () => {
+  await withHarness(async ({ harness, target, close }) => {
+    let apply = true;
+    target.fetch = async () => {
+      if (apply) {
+        apply = false;
+        return response({});
+      }
+      return response({ revision: "cleanup-revision" });
+    };
+    await assert.rejects(harness.apply(), /did not return a runtime revision/);
+    await close();
+  });
+});
+
+test("request falls back to a parent-boundary watch when the initial event is missed", async () => {
+  const root = await sourceTree();
+  const calls = [];
+  const continuation = {
+    sessionId: "parent",
+    events: [],
+    async waitForEvent() {
+      return { data: { name: "self-modification", childSessionId: "child" } };
+    },
+    async result() {
+      return completedTurn("parent");
+    },
+  };
+  const child = {
+    sessionId: "child",
+    events: [],
+    async result() {
+      return completedTurn("child");
+    },
+  };
+  const target = targetFor({ calls, turns: { parent: continuation, child } });
+  const parent = {
+    sessionId: "parent",
+    events: [],
+    session: { state: { streamIndex: 10 } },
+    async waitForEvent() {
+      throw new Error("stream boundary");
+    },
+    async result() {
+      return completedTurn("parent");
+    },
+  };
+  const t = context(target);
+  t.start = async () => parent;
+  const harness = await SelfModificationHarness.create(t, root);
+  await harness.request("make a change");
+  await harness.close();
+  await rm(root, { recursive: true, force: true });
+});
+
+test("cleanup failure retains a lock that identifies the source backup", async (t) => {
+  const root = await temporaryCheckout(t, "eve-selfmod-cleanup-failed-");
+  await writeFile(join(root, "agent", "instructions.md"), "baseline");
+  const target = targetFor();
+  target.fetch = async (path) =>
+    path.includes("/suspend?")
+      ? new Response("no", { status: 500 })
+      : response({ revision: "revision" });
+  await assert.rejects(
+    withSelfModification(context(target), async (harness) => {
+      await harness.writeSource("instructions.md", "changed");
+    }),
+    /suspend.*failed: 500/,
+  );
+  const owner = JSON.parse(
+    await readFile(join(root, ".eve-self-modification-eval.lock", "owner.json"), "utf8"),
+  );
+  t.after(() => rm(owner.backupRoot, { recursive: true, force: true }));
+  assert.equal(
+    await readFile(join(owner.backupRoot, "agent", "instructions.md"), "utf8"),
+    "baseline",
+  );
+});
@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { test } from "node:test";
+
+import { prepareSelfModification } from "../scripts/prepare.mjs";
+
+const repoRoot = fileURLToPath(new URL("../../../../", import.meta.url));
+
+async function temporaryFixture(t) {
+  const fixtureRoot = await mkdtemp(join(tmpdir(), "eve-selfmod-scaffold-"));
+  t.after(() => rm(fixtureRoot, { recursive: true, force: true }));
+  return fixtureRoot;
+}
+
+async function put(root, path, content) {
+  await mkdir(dirname(join(root, path)), { recursive: true });
+  await writeFile(join(root, path), content);
+}
+
+async function files(root, directory = "") {
+  const result = [];
+  for (const entry of await readdir(join(root, directory), { withFileTypes: true })) {
+    const path = directory ? `${directory}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) result.push(...(await files(root, path)));
+    else result.push(path);
+  }
+  return result.sort();
+}
+
+test("preparation copies the canonical registry targets and removes stale generated files", async (t) => {
+  const fixtureRoot = await temporaryFixture(t);
+  const registry = JSON.parse(await readFile(join(repoRoot, "apps/docs/registry.json"), "utf8"));
+  const item = registry.items.find((item) => item.name === "eve/self-modification");
+  await put(fixtureRoot, "agent/tools/keep.ts", "authored tool");
+  await put(fixtureRoot, "agent/subagents/self-modification/instructions.md", "stale override");
+
+  await prepareSelfModification({ fixtureRoot });
+  for (const file of item.files) {
+    assert.deepEqual(
+      await readFile(join(fixtureRoot, file.target)),
+      await readFile(join(repoRoot, "apps/docs", file.path)),
+    );
+  }
+  assert.deepEqual(
+    await files(join(fixtureRoot, "agent/subagents/self-modification")),
+    item.files.map((file) => file.target.replace("agent/subagents/self-modification/", "")).sort(),
+  );
+  assert.equal(await readFile(join(fixtureRoot, "agent/tools/keep.ts"), "utf8"), "authored tool");
+
+  await put(fixtureRoot, item.files[0].target, "modified by previous eval");
+  await prepareSelfModification({ fixtureRoot });
+  assert.deepEqual(
+    await readFile(join(fixtureRoot, item.files[0].target)),
+    await readFile(join(repoRoot, "apps/docs", item.files[0].path)),
+  );
+});
+
+test("invalid registry targets fail before replacing the existing scaffold", async (t) => {
+  const root = await temporaryFixture(t);
+  const fixtureRoot = join(root, "fixture");
+  const sourceRepo = join(root, "repo");
+  await put(fixtureRoot, "agent/subagents/self-modification/agent.ts", "existing");
+  await put(sourceRepo, "apps/docs/registry/example.ts", "source");
+  await put(
+    sourceRepo,
+    "apps/docs/registry.json",
+    JSON.stringify({
+      items: [
+        {
+          name: "eve/self-modification",
+          files: [{ path: "registry/example.ts", target: "agent/tools/outside.ts" }],
+        },
+      ],
+    }),
+  );
+  await assert.rejects(prepareSelfModification({ fixtureRoot, repoRoot: sourceRepo }), /outside/);
+  assert.equal(
+    await readFile(join(fixtureRoot, "agent/subagents/self-modification/agent.ts"), "utf8"),
+    "existing",
+  );
+});
+
+test("missing source files fail before replacing the existing scaffold", async (t) => {
+  const root = await temporaryFixture(t);
+  const fixtureRoot = join(root, "fixture");
+  const sourceRepo = join(root, "repo");
+  await put(fixtureRoot, "agent/subagents/self-modification/agent.ts", "existing");
+  await put(
+    sourceRepo,
+    "apps/docs/registry.json",
+    JSON.stringify({
+      items: [
+        {
+          name: "eve/self-modification",
+          files: [
+            { path: "registry/missing.ts", target: "agent/subagents/self-modification/agent.ts" },
+          ],
+        },
+      ],
+    }),
+  );
+  await assert.rejects(prepareSelfModification({ fixtureRoot, repoRoot: sourceRepo }), /ENOENT/);
+  assert.equal(
+    await readFile(join(fixtureRoot, "agent/subagents/self-modification/agent.ts"), "utf8"),
+    "existing",
+  );
+});
@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "outDir": "dist",
+    "rootDir": ".",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "noEmit": true,
+    "types": ["node"]
+  },
+  "include": ["agent/**/*.ts", "evals/**/*.ts"]
+}
@@ -35,6 +35,7 @@
    "@changesets/cli": "3.0.1",
    "@microsoft/api-extractor": "7.59.0",
    "@types/node": "catalog:",
+    "@vercel/connect": "catalog:",
    "gray-matter": "4.0.3",
    "oxfmt": "0.66.0",
    "oxlint": "1.81.0",
@@ -110,6 +110,9 @@ importers:
      '@types/node':
        specifier: 'catalog:'
        version: 24.13.3
+      '@vercel/connect':
+        specifier: 'catalog:'
+        version: 1.0.0(@ai-sdk/mcp@2.0.45(zod@4.5.4))(@auth/core@0.41.2)(@chat-adapter/slack@4.34.0(ai@7.0.93(zod@4.5.4))(bufferutil@4.1.0)(supports-color@10.2.2)(zod@4.5.4))(ai@7.0.93(zod@4.5.4))(eve@packages+eve)
      gray-matter:
        specifier: 4.0.3
        version: 4.0.3
@@ -889,6 +892,34 @@ importers:
        specifier: 'catalog:'
        version: 7.0.2

+  e2e/fixtures/agent-self-modification:
+    dependencies:
+      '@eve-e2e/config':
+        specifier: workspace:*
+        version: link:../e2e-config
+      '@vercel/connect':
+        specifier: 'catalog:'
+        version: 1.0.0(@ai-sdk/mcp@2.0.45(zod@4.5.4))(@auth/core@0.41.2)(@chat-adapter/slack@4.34.0(ai@7.0.93(zod@4.5.4))(bufferutil@4.1.0)(supports-color@10.2.2)(zod@4.5.4))(ai@7.0.93(zod@4.5.4))(eve@packages+eve)
+      '@workflow/world-postgres':
+        specifier: 'catalog:'
+        version: 5.0.0-beta.42(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.4)(sql.js@1.14.1)(supports-color@10.2.2)(typescript@7.0.2)
+      eve:
+        specifier: workspace:*
+        version: link:../../../packages/eve
+      just-bash:
+        specifier: 3.1.0
+        version: 3.1.0(supports-color@10.2.2)
+      zod:
+        specifier: 'catalog:'
+        version: 4.5.4
+    devDependencies:
+      '@types/node':
+        specifier: 'catalog:'
+        version: 24.13.3
+      typescript:
+        specifier: 'catalog:'
+        version: 7.0.2
+
  e2e/fixtures/agent-session-limits:
    dependencies:
      '@eve-e2e/config':
@@ -62,6 +62,7 @@ catalog:
 minimumReleaseAge: 2880 # 2 days

 minimumReleaseAgeExclude:
+  - eve
  - "@kybernesis/arcana"
  - "@upstash/*"
  - "@vercel/*"