From f3cd55a629707f882d33490044f170f7255341bb Mon Sep 17 00:00:00 2001 From: Andrew Barba Date: Thu, 17 Sep 2026 18:03:44 -0400 Subject: [PATCH] refactor(eve): remap model evaluation APIs (#3489) Signed-off-by: Andrew Barba --- .changeset/remap-model-evaluation-apis.md | 5 + docs/agent-config.md | 2 +- docs/guides/evaluate.md | 37 +++--- docs/reference/typescript-api.md | 3 +- e2e/fixtures/agent-evaluate/agent/testing.ts | 4 +- .../agent/tools/evaluate-request.ts | 2 +- .../entrypoints/dynamicTool.ts | 2 +- .../extension-contracts/entrypoints/tool.ts | 2 +- .../reports/dynamicTool/v48.json | 16 +++ packages/eve/package.json | 15 ++- .../evaluate.integration.test.ts | 2 +- .../{experimental/evaluate => ai}/evaluate.ts | 0 .../evaluate => ai}/package.scenario.test.ts | 12 +- .../src/compiler/extension-compatibility.ts | 14 ++- .../host/build-extension.scenario.test.ts | 114 +++++++++--------- .../host/extension-capability-requirements.ts | 8 +- .../auto.integration.test.ts} | 28 ++--- .../evaluate/auto-model.ts => models/auto.ts} | 22 ++-- packages/eve/src/public/ai/index.ts | 1 + .../src/public/experimental/evaluate/index.ts | 2 - packages/eve/src/public/models/index.ts | 1 + .../eve/src/tools/approval/policies.test.ts | 2 +- packages/eve/src/tools/approval/policies.ts | 2 +- research/jev-decision-models.md | 29 ++--- scripts/extension-contracts/configuration.mjs | 3 +- 25 files changed, 181 insertions(+), 147 deletions(-) create mode 100644 .changeset/remap-model-evaluation-apis.md create mode 100644 packages/eve/extension-contracts/reports/dynamicTool/v48.json rename packages/eve/src/{experimental/evaluate => ai}/evaluate.integration.test.ts (98%) rename packages/eve/src/{experimental/evaluate => ai}/evaluate.ts (100%) rename packages/eve/src/{experimental/evaluate => ai}/package.scenario.test.ts (81%) rename packages/eve/src/{experimental/evaluate/auto-model.integration.test.ts => models/auto.integration.test.ts} (88%) rename packages/eve/src/{experimental/evaluate/auto-model.ts => models/auto.ts} (88%) create mode 100644 packages/eve/src/public/ai/index.ts delete mode 100644 packages/eve/src/public/experimental/evaluate/index.ts create mode 100644 packages/eve/src/public/models/index.ts diff --git a/.changeset/remap-model-evaluation-apis.md b/.changeset/remap-model-evaluation-apis.md new file mode 100644 index 000000000..176c650a2 --- /dev/null +++ b/.changeset/remap-model-evaluation-apis.md @@ -0,0 +1,5 @@ +--- +"eve": minor +--- + +Move automatic model selection to `auto` from `eve/models` and standalone evaluation to `evaluate` from `eve/ai`. The former `autoModel` and `eve/experimental/evaluate` imports are no longer available. diff --git a/docs/agent-config.md b/docs/agent-config.md index 152b72c32..abd868faa 100644 --- a/docs/agent-config.md +++ b/docs/agent-config.md @@ -63,7 +63,7 @@ to compaction calls. ### Choose the model dynamically To select a model from the incoming prompt with an AI SDK evaluation model, use -[`autoModel` from `eve/experimental/evaluate`](./guides/evaluate). +[`auto` from `eve/models`](./guides/evaluate). `model` also accepts `defineDynamic({ events })`. Each matching handler must return the concrete model for its scope; a dynamic model has no compiled diff --git a/docs/guides/evaluate.md b/docs/guides/evaluate.md index d7600bf77..182a2c475 100644 --- a/docs/guides/evaluate.md +++ b/docs/guides/evaluate.md @@ -3,28 +3,27 @@ title: Automatic Model Selection description: "Choose agent models automatically or evaluate typed questions in your tools and application code." --- -Use `autoModel` to choose an agent model from an allowlist before inference begins. -It uses the [AI SDK evaluation API](https://ai-sdk.dev/docs/ai-sdk-core/evaluation), +Use `auto` from `eve/models` to choose an agent model from an allowlist before +inference begins. It uses the [AI SDK evaluation API](https://ai-sdk.dev/docs/ai-sdk-core/evaluation), so the evaluator can be a Vercel AI Gateway model ID or an evaluation model from -an installed provider. Use `evaluate` from the same entrypoint to ask typed -questions in your own tools or application code. +an installed provider. Use `evaluate` from `eve/ai` to ask typed questions in +your own tools or application code. -`eve/experimental/evaluate` is experimental. Its API can change between eve -releases, and the AI SDK evaluation model specification can change in patch -releases. +The AI SDK evaluation model specification is experimental and can change in +patch releases. ## Choose from Gateway models -By default, `autoModel` evaluates with `typesafe-ai/jev`. Like other AI SDK +By default, `auto` evaluates with `typesafe-ai/jev`. Like other AI SDK model strings, it uses Vercel AI Gateway unless the application has configured a different global default provider. ```ts title="agent/agent.ts" import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; export default defineAgent({ - model: autoModel({ + model: auto({ options: { "openai/gpt-5.6-sol": "Difficult reasoning and engineering tasks", "openai/gpt-5.6-luna": "Routine tasks where fast completion matters", @@ -38,7 +37,7 @@ does not add a TypeSafe credential or transport layer. During `eve dev`, a Gateway evaluator uses the same connection selected through `/login` as Gateway language models. A configured AI SDK default provider still owns string model resolution during development. The TUI footer displays `dynamic model` when the -agent uses `autoModel`, then adds the resolved model for the current turn, such as +agent uses `auto`, then adds the resolved model for the current turn, such as `dynamic model ยท openai/gpt-5.6-luna`. ## Use a provider directly @@ -53,10 +52,10 @@ pnpm add @ai-sdk/typesafe-ai ```ts title="agent/agent.ts" import { typeSafeAi } from "@ai-sdk/typesafe-ai"; import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; export default defineAgent({ - model: autoModel({ + model: auto({ model: typeSafeAi.evaluationModel("jev-latest"), options: { "openai/gpt-5.6-sol": "Difficult reasoning and engineering tasks", @@ -78,11 +77,11 @@ a provider instance, an alias, or needs a reasoning override. ```ts title="agent/agent.ts" import { anthropic } from "@ai-sdk/anthropic"; import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; export default defineAgent({ reasoning: "medium", - model: autoModel({ + model: auto({ options: { "openai/gpt-5.6-sol": "Hard problems", my_secret_model: { @@ -107,13 +106,13 @@ agent's reasoning setting. ## Evaluate inside a tool Use `evaluate` to ask choice, score, or boolean questions about the state you pass to it. -It defaults to `typesafe-ai/jev` and uses the same authentication as `autoModel`, +It defaults to `typesafe-ai/jev` and uses the same authentication as `auto`, including the Gateway connection selected through `/login` during `eve dev`. Pass `model` to use another evaluation model ID or a provider instance. A configured AI SDK default provider takes precedence over the local Gateway connection. ```ts title="agent/tools/classify-request.ts" -import { evaluate } from "eve/experimental/evaluate"; +import { evaluate } from "eve/ai"; import { defineTool } from "eve/tools"; import { z } from "zod"; @@ -149,7 +148,7 @@ and `providerOptions`. Pass an `abortSignal` to cancel the request. Input and answer validation, retries, and provider errors follow AI SDK semantics. You can also call `evaluate` outside a tool; it does not require an active eve -session. Each call performs its own evaluation. `autoModel` uses this function +session. Each call performs its own evaluation. `auto` uses this function and adds the per-turn routing behavior described below. ## Evaluate tool approvals @@ -179,7 +178,7 @@ options and data handling. ## Runtime behavior -`autoModel` evaluates at the first `step.started` event, after the incoming prompt +`auto` evaluates at the first `step.started` event, after the incoming prompt is available and before the selected language model runs. It reuses that choice for later tool-loop steps in the same turn. A new turn makes a new choice, and child sessions route from their own prompts. diff --git a/docs/reference/typescript-api.md b/docs/reference/typescript-api.md index 45c02011a..090b4d100 100644 --- a/docs/reference/typescript-api.md +++ b/docs/reference/typescript-api.md @@ -136,9 +136,10 @@ import template from "../../prompts/template.txt?raw"; | `eve/sandbox` | `defineSandbox`, backends | | `eve/instrumentation` | `defineInstrumentation`, `isChannel` | | `eve/local-dev` | `getLocalDevCapability`, `LocalDevCapability` | +| [`eve/models`](../guides/evaluate) | Automatic model selection with `auto` | | `eve/models/openai` | `openai`, `chatgpt`, deprecated `experimental_chatgpt` | | `eve/models/anthropic` | `anthropic` | -| [`eve/experimental/evaluate`](../guides/evaluate) | Experimental `autoModel` and standalone `evaluate` | +| [`eve/ai`](../guides/evaluate#evaluate-inside-a-tool) | Standalone `evaluate` | | `eve/evals` | `defineEval`, `defineEvalConfig`, `mockModel`, eval types | | `eve/evals/expect` | `includes`, `equals`, `matches`, `similarity` | | `eve/evals/reporters` | `Braintrust`, `JUnit`, `EvalReporter` | diff --git a/e2e/fixtures/agent-evaluate/agent/testing.ts b/e2e/fixtures/agent-evaluate/agent/testing.ts index 0850c51ce..8d8c16da6 100644 --- a/e2e/fixtures/agent-evaluate/agent/testing.ts +++ b/e2e/fixtures/agent-evaluate/agent/testing.ts @@ -2,7 +2,7 @@ import type { Experimental_EvaluationModel } from "ai"; import { defineDynamic } from "eve"; import { defineState } from "eve/context"; import { mockModel, type MockModelResponder } from "eve/evals"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; export const routing = defineState("evaluate-fixture.routing", () => ({ requests: 0, @@ -56,7 +56,7 @@ export const evaluationModel: Exclude = { /** Run the real router with deterministic evaluation and language models. */ export function fixtureModel(respond: MockModelResponder) { - const model = autoModel({ + const model = auto({ model: evaluationModel, options: { "openai/large": { diff --git a/e2e/fixtures/agent-evaluate/agent/tools/evaluate-request.ts b/e2e/fixtures/agent-evaluate/agent/tools/evaluate-request.ts index 0c5d46951..21f1dc449 100644 --- a/e2e/fixtures/agent-evaluate/agent/tools/evaluate-request.ts +++ b/e2e/fixtures/agent-evaluate/agent/tools/evaluate-request.ts @@ -1,4 +1,4 @@ -import { evaluate } from "eve/experimental/evaluate"; +import { evaluate } from "eve/ai"; import { defineTool } from "eve/tools"; import { evaluationModel } from "../testing"; diff --git a/packages/eve/extension-contracts/entrypoints/dynamicTool.ts b/packages/eve/extension-contracts/entrypoints/dynamicTool.ts index a8ba45e06..151650e93 100644 --- a/packages/eve/extension-contracts/entrypoints/dynamicTool.ts +++ b/packages/eve/extension-contracts/entrypoints/dynamicTool.ts @@ -7,4 +7,4 @@ export { defineDurableSchema, defineDynamic, } from "../../src/public/tools/index.ts"; -export { autoModel } from "../../src/public/experimental/evaluate/index.ts"; +export { auto } from "../../src/public/models/index.ts"; diff --git a/packages/eve/extension-contracts/entrypoints/tool.ts b/packages/eve/extension-contracts/entrypoints/tool.ts index cb7611f78..dca259bc9 100644 --- a/packages/eve/extension-contracts/entrypoints/tool.ts +++ b/packages/eve/extension-contracts/entrypoints/tool.ts @@ -18,4 +18,4 @@ export { type WorkflowToolInput, type WorkflowToolOptions, } from "../../src/public/tools/workflow.ts"; -export { evaluate } from "../../src/public/experimental/evaluate/index.ts"; +export { evaluate } from "../../src/public/ai/index.ts"; diff --git a/packages/eve/extension-contracts/reports/dynamicTool/v48.json b/packages/eve/extension-contracts/reports/dynamicTool/v48.json new file mode 100644 index 000000000..ff814ce4a --- /dev/null +++ b/packages/eve/extension-contracts/reports/dynamicTool/v48.json @@ -0,0 +1,16 @@ +{ + "kind": "eve-extension-capability-contract", + "capability": "dynamicTool", + "epoch": 48, + "sha256": "7b9bf2b87e26c0f9ff49b8ca9e79ce7f87acf61c9cd8271dfd20362a9861c74f", + "exports": [ + "DynamicToolEntry", + "DynamicToolEvents", + "DynamicToolResult", + "DynamicToolSet", + "auto", + "defineDurableCallback", + "defineDurableSchema", + "defineDynamic" + ] +} diff --git a/packages/eve/package.json b/packages/eve/package.json index f0df614f6..af5931cd1 100644 --- a/packages/eve/package.json +++ b/packages/eve/package.json @@ -211,6 +211,16 @@ "eve-source": "./dist/src/public/workflow-modules.d.ts", "types": "./dist/src/public/workflow-modules.d.ts" }, + "./ai": { + "types": "./dist/src/public/ai/index.d.ts", + "import": "./dist/src/public/ai/index.js", + "default": "./dist/src/public/ai/index.js" + }, + "./models": { + "types": "./dist/src/public/models/index.d.ts", + "import": "./dist/src/public/models/index.js", + "default": "./dist/src/public/models/index.js" + }, "./models/anthropic": { "types": "./dist/src/public/models/anthropic/index.d.ts", "import": "./dist/src/public/models/anthropic/index.js", @@ -437,11 +447,6 @@ "types": "./dist/src/self-modification/config.d.ts", "import": "./dist/src/self-modification/config.js", "default": "./dist/src/self-modification/config.js" - }, - "./experimental/evaluate": { - "types": "./dist/src/public/experimental/evaluate/index.d.ts", - "import": "./dist/src/public/experimental/evaluate/index.js", - "default": "./dist/src/public/experimental/evaluate/index.js" } }, "publishConfig": { diff --git a/packages/eve/src/experimental/evaluate/evaluate.integration.test.ts b/packages/eve/src/ai/evaluate.integration.test.ts similarity index 98% rename from packages/eve/src/experimental/evaluate/evaluate.integration.test.ts rename to packages/eve/src/ai/evaluate.integration.test.ts index e0ce1fcf0..445d59c2e 100644 --- a/packages/eve/src/experimental/evaluate/evaluate.integration.test.ts +++ b/packages/eve/src/ai/evaluate.integration.test.ts @@ -1,7 +1,7 @@ import { Experimental_EvaluationMockModelV4 } from "ai/test"; import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; -import { evaluate } from "#public/experimental/evaluate/index.js"; +import { evaluate } from "#public/ai/index.js"; const localEvaluationModel = vi.hoisted(() => vi.fn()); vi.mock("#internal/model-auth/transport.js", async (importOriginal) => ({ diff --git a/packages/eve/src/experimental/evaluate/evaluate.ts b/packages/eve/src/ai/evaluate.ts similarity index 100% rename from packages/eve/src/experimental/evaluate/evaluate.ts rename to packages/eve/src/ai/evaluate.ts diff --git a/packages/eve/src/experimental/evaluate/package.scenario.test.ts b/packages/eve/src/ai/package.scenario.test.ts similarity index 81% rename from packages/eve/src/experimental/evaluate/package.scenario.test.ts rename to packages/eve/src/ai/package.scenario.test.ts index 71b018ebe..149e12984 100644 --- a/packages/eve/src/experimental/evaluate/package.scenario.test.ts +++ b/packages/eve/src/ai/package.scenario.test.ts @@ -10,7 +10,7 @@ const scenarioApp = useScenarioApp(); it("builds an agent and child with Gateway and provider evaluation models from packed eve", async () => { const app = await scenarioApp({ - name: "experimental-evaluate", + name: "ai-models", installDependencies: true, files: { "agent/instructions.md": "Help Alice review the export incident.", @@ -23,11 +23,11 @@ export const evaluationModel = { async doEvaluate() { throw new Error("Build must not evaluate models."); }, } satisfies Exclude;`, "agent/agent.ts": `import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; import { anthropic } from "eve/models/anthropic"; -export default defineAgent({ model: autoModel({ options: { "openai/gpt-5.6-sol": "Investigations", my_secret_model: { model: anthropic("sonnet-5"), reasoning: "low", description: "Routine work" } } }) });`, +export default defineAgent({ model: auto({ options: { "openai/gpt-5.6-sol": "Investigations", my_secret_model: { model: anthropic("sonnet-5"), reasoning: "low", description: "Routine work" } } }) });`, "agent/tools/classify.ts": `import { defineTool } from "eve/tools"; -import { evaluate } from "eve/experimental/evaluate"; +import { evaluate } from "eve/ai"; export default defineTool({ description: "Classify an incident", inputSchema: { type: "object", properties: {}, additionalProperties: false }, @@ -42,10 +42,10 @@ export default defineTool({ });`, "agent/subagents/worker/instructions.md": "Review the assigned evidence.", "agent/subagents/worker/agent.ts": `import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; import { anthropic } from "eve/models/anthropic"; import { evaluationModel } from "../../evaluation"; -export default defineAgent({ description: "Review evidence", model: autoModel({ model: evaluationModel, options: { reviewer: { model: anthropic("sonnet-5"), reasoning: "low", description: "Investigations" } } }) });`, +export default defineAgent({ description: "Review evidence", model: auto({ model: evaluationModel, options: { reviewer: { model: anthropic("sonnet-5"), reasoning: "low", description: "Investigations" } } }) });`, }, }); diff --git a/packages/eve/src/compiler/extension-compatibility.ts b/packages/eve/src/compiler/extension-compatibility.ts index 14d26b42e..5b651f941 100644 --- a/packages/eve/src/compiler/extension-compatibility.ts +++ b/packages/eve/src/compiler/extension-compatibility.ts @@ -24,7 +24,7 @@ const EXTENSION_CAPABILITY_CONTRACTS = { tool: { current: 49, supported: [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 28, 29, 30, 31, 32, 34, 35, 44, 45, 46, 47, 48, 49, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 28, 29, 30, 31, 32, 34, 35, 44, 45, 46, 49, ], dropped: { 14: "TaskExec.delegated was removed; migrate to workflow-backed background tools", @@ -50,13 +50,15 @@ const EXTENSION_CAPABILITY_CONTRACTS = { 41: "workflow no longer accepts agents and its options argument is optional; use workflow() or workflow({ maxSubagents })", 42: "Legacy session history migration was removed; user-role messages require current provenance kinds.", 43: "Legacy session history migration was removed; user-role messages require current provenance kinds.", + 47: "eve/experimental/evaluate was removed; import evaluate from eve/ai", + 48: "eve/experimental/evaluate was removed; import evaluate from eve/ai", }, }, dynamicTool: { - current: 47, + current: 48, supported: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 28, 29, 30, 31, 32, - 33, 41, 42, 43, 44, 45, 46, 47, + 33, 41, 48, ], dropped: { 21: "Message and reasoning append events now expose deltas instead of cumulative snapshots.", @@ -72,6 +74,12 @@ const EXTENSION_CAPABILITY_CONTRACTS = { 38: "workflowMaxSubagents was removed with experimental_workflow; configure generated-program limits with the workflow factory", 39: "Legacy session history migration was removed; user-role messages require current provenance kinds.", 40: "Legacy session history migration was removed; user-role messages require current provenance kinds.", + 42: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", + 43: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", + 44: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", + 45: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", + 46: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", + 47: "autoModel and eve/experimental/evaluate were removed; import auto from eve/models", }, }, channel: { diff --git a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts index f3ddfa027..fa508319f 100644 --- a/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts +++ b/packages/eve/src/internal/nitro/host/build-extension.scenario.test.ts @@ -84,76 +84,79 @@ describe("extension build output", () => { }); it.each([ - ["static re-export", 'export { evaluate as assess } from "eve/experimental/evaluate";'], + ["static re-export", 'export { evaluate as assess } from "eve/ai";'], [ "namespace import", - 'import * as evaluation from "eve/experimental/evaluate"; export const assess = evaluation.evaluate;', + 'import * as evaluation from "eve/ai"; export const assess = evaluation.evaluate;', ], [ "dynamic import", - 'export async function assess(options: Parameters[0]): Promise { await (await import("eve/experimental/evaluate")).evaluate(options); }', + 'export async function assess(options: Parameters[0]): Promise { await (await import("eve/ai")).evaluate(options); }', ], - ])( - "stamps evaluation capabilities for a hook-only extension using a %s", - async (_name, helper) => { - const root = await createExtensionPackage(); - await rm(join(root, "extension", "tools"), { recursive: true }); - await mkdir(join(root, "extension", "hooks")); - await mkdir(join(root, "extension", "lib")); - await writeFile(join(root, "extension", "lib", "evaluation.ts"), helper); - await writeFile( - join(root, "extension", "hooks", "evaluate.ts"), - `import { defineHook } from "eve/hooks"; + ])("stamps the tool capability for a hook-only extension using a %s", async (_name, helper) => { + const root = await createExtensionPackage(); + await rm(join(root, "extension", "tools"), { recursive: true }); + await mkdir(join(root, "extension", "hooks")); + await mkdir(join(root, "extension", "lib")); + await writeFile(join(root, "extension", "lib", "evaluation.ts"), helper); + await writeFile( + join(root, "extension", "hooks", "evaluate.ts"), + `import { defineHook } from "eve/hooks"; import { assess } from "../lib/evaluation"; export default defineHook({ events: { "turn.started": async () => { await assess({ state: { request: "Alice needs a summary." }, questions: { routine: { type: "boolean", instructions: "Is this routine work?" } } }); } } });`, - ); - const config = await tryReadExtensionBuildConfig(root); - const outDir = await buildExtensionPackage(root, config!); - const manifestPath = join(outDir, "extension", "_manifest.json"); - const manifest = parseExtensionCompatibilityManifest( - await readFile(manifestPath, "utf8"), - manifestPath, - ); + ); + const config = await tryReadExtensionBuildConfig(root); + const outDir = await buildExtensionPackage(root, config!); + const manifestPath = join(outDir, "extension", "_manifest.json"); + const manifest = parseExtensionCompatibilityManifest( + await readFile(manifestPath, "utf8"), + manifestPath, + ); - expect(manifest.requires).toEqual({ - extension: EXTENSION_CAPABILITY_VERSIONS.extension, - hook: EXTENSION_CAPABILITY_VERSIONS.hook, - tool: EXTENSION_CAPABILITY_VERSIONS.tool, - dynamicTool: EXTENSION_CAPABILITY_VERSIONS.dynamicTool, - }); - expect(findUnsupportedExtensionCapabilities(manifest)).toEqual([]); - expect( - findUnsupportedExtensionCapabilities(manifest, { - ...EXTENSION_CAPABILITY_SUPPORT, - tool: EXTENSION_CAPABILITY_SUPPORT.tool.filter((version) => version < 47), - }), - ).toEqual([ - expect.objectContaining({ - capability: "tool", - requiredVersion: EXTENSION_CAPABILITY_VERSIONS.tool, - }), - ]); - }, - ); + expect(manifest.requires).toEqual({ + extension: EXTENSION_CAPABILITY_VERSIONS.extension, + hook: EXTENSION_CAPABILITY_VERSIONS.hook, + tool: EXTENSION_CAPABILITY_VERSIONS.tool, + }); + expect(findUnsupportedExtensionCapabilities(manifest)).toEqual([]); + expect( + findUnsupportedExtensionCapabilities(manifest, { + ...EXTENSION_CAPABILITY_SUPPORT, + tool: EXTENSION_CAPABILITY_SUPPORT.tool.filter( + (version) => version !== EXTENSION_CAPABILITY_VERSIONS.tool, + ), + }), + ).toEqual([ + expect.objectContaining({ + capability: "tool", + requiredVersion: EXTENSION_CAPABILITY_VERSIONS.tool, + }), + ]); + }); it.each([ [ - "autoModel", - 'import { autoModel } from "eve/experimental/evaluate"; export const route: ReturnType = autoModel({ options: { "openai/small": "Routine work" } });', - true, + "automatic model selection", + 'import { auto } from "eve/models"; export const route: ReturnType = auto({ options: { "openai/small": "Routine work" } });', + "dynamicTool", ], [ - "type-only import", - 'import type { evaluate } from "eve/experimental/evaluate"; export type Evaluate = typeof evaluate;', - false, + "type-only AI import", + 'import type { evaluate } from "eve/ai"; export type Evaluate = typeof evaluate;', + undefined, ], - ])( - "tracks runtime evaluation imports in a tool-free helper: %s", - async (_name, helper, runtime) => { + [ + "type-only model import", + 'import type { auto } from "eve/models"; export type Auto = typeof auto;', + undefined, + ], + ] as const)( + "tracks runtime AI and model imports in a tool-free helper: %s", + async (_name, helper, capability) => { const root = await createExtensionPackage(); await rm(join(root, "extension", "tools"), { recursive: true }); await mkdir(join(root, "extension", "lib")); @@ -168,12 +171,9 @@ export default defineHook({ events: { "turn.started": async () => { expect(manifest.requires).toEqual({ extension: EXTENSION_CAPABILITY_VERSIONS.extension, - ...(runtime - ? { - tool: EXTENSION_CAPABILITY_VERSIONS.tool, - dynamicTool: EXTENSION_CAPABILITY_VERSIONS.dynamicTool, - } - : {}), + ...(capability === undefined + ? {} + : { [capability]: EXTENSION_CAPABILITY_VERSIONS[capability] }), }); }, ); diff --git a/packages/eve/src/internal/nitro/host/extension-capability-requirements.ts b/packages/eve/src/internal/nitro/host/extension-capability-requirements.ts index adc147125..76d911461 100644 --- a/packages/eve/src/internal/nitro/host/extension-capability-requirements.ts +++ b/packages/eve/src/internal/nitro/host/extension-capability-requirements.ts @@ -90,11 +90,9 @@ export async function deriveExtensionCapabilityRequirements(input: { required.add("config"); } if (usesState) required.add("state"); - // The evaluation entrypoint can be used from any contribution or shared helper. - if (input.runtimeImports.includes("eve/experimental/evaluate")) { - required.add("tool"); - required.add("dynamicTool"); - } + // Runtime imports can use these capabilities outside manifest-declared tools. + if (input.runtimeImports.includes("eve/ai")) required.add("tool"); + if (input.runtimeImports.includes("eve/models")) required.add("dynamicTool"); return Object.fromEntries( (Object.keys(EXTENSION_CAPABILITY_VERSIONS) as ExtensionCapability[]) diff --git a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts b/packages/eve/src/models/auto.integration.test.ts similarity index 88% rename from packages/eve/src/experimental/evaluate/auto-model.integration.test.ts rename to packages/eve/src/models/auto.integration.test.ts index e2ffb6155..dbbbe034b 100644 --- a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts +++ b/packages/eve/src/models/auto.integration.test.ts @@ -6,7 +6,7 @@ import { deserializeContext, serializeContext } from "#context/serialize.js"; import type { DynamicResolveContext } from "#dynamic/definition.js"; import { anthropic } from "#public/models/anthropic/index.js"; -import { autoModel } from "./auto-model.js"; +import { auto } from "./auto.js"; const runtime = vi.hoisted(() => ({ localEvaluationModel: vi.fn(), @@ -61,14 +61,14 @@ beforeEach(() => { runtime.localEvaluationModel.mockReset(); }); -describe("autoModel", () => { +describe("auto", () => { it("defaults to Jev through the AI SDK default provider", async () => { const evaluator = evaluationModel(); const evaluationModelFactory = vi.fn(() => evaluator.model); const previous = Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER"); Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", { evaluationModel: evaluationModelFactory }); try { - const handler = autoModel({ options }).events["step.started"]!; + const handler = auto({ options }).events["step.started"]!; await expect(handler(event(), context())).resolves.toBe("openai/small"); expect(runtime.localEvaluationModel).not.toHaveBeenCalled(); expect(evaluationModelFactory).toHaveBeenCalledWith("typesafe-ai/jev"); @@ -86,7 +86,7 @@ describe("autoModel", () => { const previous = Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER"); Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", { evaluationModel: evaluationModelFactory }); try { - const handler = autoModel({ model: "internal-router", options }).events["step.started"]!; + const handler = auto({ model: "internal-router", options }).events["step.started"]!; await expect(handler(event(), context())).resolves.toBe("openai/small"); expect(evaluationModelFactory).toHaveBeenCalledWith("internal-router"); expect(evaluator.doEvaluate).toHaveBeenCalledOnce(); @@ -102,7 +102,7 @@ describe("autoModel", () => { const evaluator = evaluationModel(); runtime.localEvaluationModel.mockReturnValue(evaluator.model); - const handler = autoModel({ options }).events["step.started"]!; + const handler = auto({ options }).events["step.started"]!; await expect(handler(event(), context())).resolves.toBe("openai/small"); expect(runtime.localEvaluationModel).toHaveBeenCalledWith("typesafe-ai/jev"); @@ -112,7 +112,7 @@ describe("autoModel", () => { it("routes provider language models by alias and preserves reasoning", async () => { const languageModel = anthropic("sonnet-5"); const evaluator = evaluationModel("private"); - const handler = autoModel({ + const handler = auto({ model: evaluator.model, options: { ...options, @@ -145,7 +145,7 @@ describe("autoModel", () => { it("evaluates once per turn and restores the selection from durable context", async () => { const evaluator = evaluationModel(); - const handler = autoModel({ model: evaluator.model, options }).events["step.started"]!; + const handler = auto({ model: evaluator.model, options }).events["step.started"]!; await handler(event(), context()); runtime.state = await deserializeContext(serializeContext(runtime.state!)); @@ -165,7 +165,7 @@ describe("autoModel", () => { throw providerError; }, }); - const failedHandler = autoModel({ model: failed, options }).events["step.started"]!; + const failedHandler = auto({ model: failed, options }).events["step.started"]!; await expect(failedHandler(event(), context())).rejects.toBe(providerError); runtime.state = new ContextContainer(); @@ -176,7 +176,7 @@ describe("autoModel", () => { abortSignal?.addEventListener("abort", () => reject(abortSignal.reason), { once: true }); }), }); - const pendingHandler = autoModel({ model: pendingModel, options }).events["step.started"]!; + const pendingHandler = auto({ model: pendingModel, options }).events["step.started"]!; const pending = pendingHandler(event(), context("Alice needs help.", controller.signal)); const reason = new Error("cancelled"); controller.abort(reason); @@ -185,11 +185,11 @@ describe("autoModel", () => { it("rejects invalid configurations and input", async () => { const evaluator = evaluationModel().model; - expect(() => autoModel({ model: evaluator, options: {} })).toThrow("at least one option"); - expect(() => autoModel({ model: evaluator, options: { broken: "" } })).toThrow(); - expect(() => autoModel({ model: "", options })).toThrow("valid evaluation model"); + expect(() => auto({ model: evaluator, options: {} })).toThrow("at least one option"); + expect(() => auto({ model: evaluator, options: { broken: "" } })).toThrow(); + expect(() => auto({ model: "", options })).toThrow("valid evaluation model"); expect(() => - autoModel({ + auto({ model: evaluator, options: { broken: { model: "openai/large", description: "Difficult", reasoning: "maximum" }, @@ -197,7 +197,7 @@ describe("autoModel", () => { } as never), ).toThrow(); - const handler = autoModel({ model: evaluator, options }).events["step.started"]!; + const handler = auto({ model: evaluator, options }).events["step.started"]!; await expect(handler(event(), context(" "))).rejects.toThrow("requires user text"); }); }); diff --git a/packages/eve/src/experimental/evaluate/auto-model.ts b/packages/eve/src/models/auto.ts similarity index 88% rename from packages/eve/src/experimental/evaluate/auto-model.ts rename to packages/eve/src/models/auto.ts index 489e313b2..c64655842 100644 --- a/packages/eve/src/experimental/evaluate/auto-model.ts +++ b/packages/eve/src/models/auto.ts @@ -16,9 +16,9 @@ import type { PublicAgentStaticModelDefinition, } from "#shared/agent-definition.js"; -import { DEFAULT_EVALUATION_MODEL, evaluate } from "./evaluate.js"; +import { DEFAULT_EVALUATION_MODEL, evaluate } from "#ai/evaluate.js"; -type AutoModelOption = +type AutoOption = | string | { readonly model: PublicAgentStaticModelDefinition; @@ -26,8 +26,8 @@ type AutoModelOption = readonly reasoning?: AgentReasoningDefinition; }; -interface AutoModelConfig< - T extends Readonly> = Readonly>, +interface AutoConfig< + T extends Readonly> = Readonly>, > { /** Evaluation model instance or ID. Defaults to TypeSafe Jev through AI SDK model resolution. */ readonly model?: EvaluationModel; @@ -45,7 +45,7 @@ function turnId(event: unknown): string { typeof event.data.turnId !== "string" || !event.data.turnId ) { - throw new Error("autoModel requires a step.started event with a turn ID."); + throw new Error("auto requires a step.started event with a turn ID."); } return event.data.turnId; } @@ -67,7 +67,7 @@ function routingState(ctx: DynamicResolveContext): Parameters[0 if (!text.trim()) continue; if (text.length + characters > 16_000) { if (messages.length === 0) { - throw new Error("The latest message is too long for autoModel routing."); + throw new Error("The latest message is too long for auto routing."); } break; } @@ -76,14 +76,14 @@ function routingState(ctx: DynamicResolveContext): Parameters[0 } if (!messages.some((message) => message.role === "user")) { - throw new Error("autoModel requires user text to select a model."); + throw new Error("auto requires user text to select a model."); } return { messages }; } /** Select a language model from the current prompt with an AI SDK evaluation model. */ -export function autoModel>>( - config: AutoModelConfig, +export function auto>>( + config: AutoConfig, ): DynamicSentinel { if ( !isRecord(config) || @@ -104,7 +104,7 @@ export function autoModel( options.map(({ key, model, reasoning }) => [ diff --git a/packages/eve/src/public/ai/index.ts b/packages/eve/src/public/ai/index.ts new file mode 100644 index 000000000..a53ad5372 --- /dev/null +++ b/packages/eve/src/public/ai/index.ts @@ -0,0 +1 @@ +export { evaluate } from "#ai/evaluate.js"; diff --git a/packages/eve/src/public/experimental/evaluate/index.ts b/packages/eve/src/public/experimental/evaluate/index.ts deleted file mode 100644 index 0de9d488c..000000000 --- a/packages/eve/src/public/experimental/evaluate/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { autoModel } from "#experimental/evaluate/auto-model.js"; -export { evaluate } from "#experimental/evaluate/evaluate.js"; diff --git a/packages/eve/src/public/models/index.ts b/packages/eve/src/public/models/index.ts new file mode 100644 index 000000000..f0b0f1177 --- /dev/null +++ b/packages/eve/src/public/models/index.ts @@ -0,0 +1 @@ +export { auto } from "#models/auto.js"; diff --git a/packages/eve/src/tools/approval/policies.test.ts b/packages/eve/src/tools/approval/policies.test.ts index 3802600ab..7da0940f3 100644 --- a/packages/eve/src/tools/approval/policies.test.ts +++ b/packages/eve/src/tools/approval/policies.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const evaluate = vi.hoisted(() => vi.fn()); -vi.mock("#experimental/evaluate/evaluate.js", () => ({ evaluate })); +vi.mock("#ai/evaluate.js", () => ({ evaluate })); import type { ApprovalContext } from "#approval/definition.js"; import { always, auto, never, once } from "#tools/approval/policies.js"; diff --git a/packages/eve/src/tools/approval/policies.ts b/packages/eve/src/tools/approval/policies.ts index a38852268..888f61591 100644 --- a/packages/eve/src/tools/approval/policies.ts +++ b/packages/eve/src/tools/approval/policies.ts @@ -1,7 +1,7 @@ import type { Experimental_EvaluationModel as EvaluationModel } from "ai"; import type { ApprovalContext, ApprovalPolicy } from "#approval/definition.js"; -import { evaluate } from "#experimental/evaluate/evaluate.js"; +import { evaluate } from "#ai/evaluate.js"; import { parseJsonValue, type JsonObject } from "#shared/json.js"; import { stampDurableDynamicCallback } from "#tools/durable-callbacks.js"; diff --git a/research/jev-decision-models.md b/research/jev-decision-models.md index 0eb5542fa..6298d1fa9 100644 --- a/research/jev-decision-models.md +++ b/research/jev-decision-models.md @@ -8,30 +8,31 @@ last_updated: "2026-09-17" ## Recommendation -Expose `autoModel` and standalone `evaluate` at `eve/experimental/evaluate`. -Build both on AI SDK's `experimental_evaluate` API. `evaluate` accepts the SDK's -state, typed questions, and request options, with an optional model defaulting to -`typesafe-ai/jev`. It returns the SDK's typed answers and response metadata. +Expose automatic model selection as `auto` from `eve/models` and standalone +`evaluate` from `eve/ai`. Build both on AI SDK's `experimental_evaluate` API. +`evaluate` accepts the SDK's state, typed questions, and request options, with an +optional model defaulting to `typesafe-ai/jev`. It returns the SDK's typed answers +and response metadata. -`autoModel` calls the shared `evaluate` wrapper. Model strings use the configured -AI SDK default provider; without an override, the wrapper resolves eve's local +`auto` calls the shared `evaluate` wrapper. Model strings use the configured AI +SDK default provider; without an override, the wrapper resolves eve's local Gateway connection when available and otherwise leaves Gateway resolution to the SDK. Explicit provider instances retain their own authentication. Tool authors can call `evaluate` with structured state and pass `ctx.abortSignal`. Application code can use it without an active eve session. Standalone calls are -not cached; durable per-turn selection remains specific to `autoModel`. +not cached; durable per-turn selection remains specific to `auto`. -The API is experimental because AI SDK's evaluation model specification is also -experimental and can change in patch releases. The implemented authoring API is -documented in [Automatic Model Selection](../docs/guides/evaluate.md). +AI SDK's evaluation model specification remains experimental and can change in +patch releases. The implemented authoring API is documented in [Automatic Model +Selection](../docs/guides/evaluate.md). ```ts import { defineAgent } from "eve"; -import { autoModel } from "eve/experimental/evaluate"; +import { auto } from "eve/models"; export default defineAgent({ - model: autoModel({ + model: auto({ options: { "openai/gpt-5.6-sol": "Difficult reasoning and engineering tasks", "openai/gpt-5.6-luna": "Routine tasks where fast completion matters", @@ -67,7 +68,7 @@ provider metadata parsing from eve. ```mermaid flowchart LR - P[Incoming prompt] --> A[autoModel] + P[Incoming prompt] --> A[auto] A --> E[AI SDK evaluate] E -->|string ID| G[Default provider / Gateway] E -->|model instance| D[Installed provider] @@ -146,7 +147,7 @@ cannot contain a live provider model instance. The tool loop resolves the step model after projecting the turn input and before language-model inference, so it has the same prompt and can return either a model string or a live instance. -`autoModel` stores only the selected option key and turn ID in a durable +`auto` stores only the selected option key and turn ID in a durable `ContextKey`. A selection is reused for later tool-loop steps in that turn. Provider model objects stay in authored configuration and are resolved again from the key after resume. New turns and child sessions make independent choices. diff --git a/scripts/extension-contracts/configuration.mjs b/scripts/extension-contracts/configuration.mjs index b1de56b71..33f898525 100644 --- a/scripts/extension-contracts/configuration.mjs +++ b/scripts/extension-contracts/configuration.mjs @@ -26,7 +26,8 @@ export const PUBLIC_SURFACES = [ "src/public/tools/index.ts", "src/public/tools/web-search.ts", "src/public/tools/workflow.ts", - "src/public/experimental/evaluate/index.ts", + "src/public/ai/index.ts", + "src/public/models/index.ts", ], capabilities: ["tool", "dynamicTool"], },