mirror of
https://github.com/vercel/eve.git
synced 2026-09-20 05:35:39 +08:00
chore(eve): performance - measure turns and remove root overhead (#2814)
Signed-off-by: Andrew Barba <barba@hey.com>
This commit is contained in:
Andrew Barba
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"eve": patch
|
||||
---
|
||||
|
||||
Skip durable caller bookkeeping steps for root session turns that have no delegated caller.
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"eve": patch
|
||||
---
|
||||
|
||||
Workflow run attributes now persist in parallel with turn-result delivery while remaining joined to the durable step, removing the observability write from the user-visible settlement path.
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
fi
|
||||
# HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
|
||||
# this diff is exactly the PR's changed files.
|
||||
if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
|
||||
if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|scripts/workflow-stress-report\.(mjs|test\.mjs)$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "relevant=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -121,6 +121,10 @@ jobs:
|
||||
pnpm --filter "./e2e/fixtures/*-extension" run --if-present build
|
||||
pnpm --filter dist-extensions update gizmo-extension gadget-extension
|
||||
|
||||
- name: Test Workflow stress reporting
|
||||
if: needs.changes.outputs.relevant == 'true' && matrix.name == 'agent-workflow-stress'
|
||||
run: node --test scripts/workflow-stress-report.test.mjs
|
||||
|
||||
- name: Stage workspace extension fixtures as dist-only
|
||||
if: needs.changes.outputs.relevant == 'true'
|
||||
# Removing author source proves the deployment consumes the same
|
||||
@@ -196,6 +200,28 @@ jobs:
|
||||
--require-phase sandbox.prewarm \
|
||||
--output-format build-time
|
||||
|
||||
- name: Report Workflow stress performance
|
||||
if: success() && needs.changes.outputs.relevant == 'true' && matrix.name == 'agent-workflow-stress'
|
||||
run: |
|
||||
node ./scripts/workflow-stress-report.mjs \
|
||||
--artifacts e2e/fixtures/agent-workflow-stress/.eve/evals \
|
||||
--json .artifacts/workflow-stress-report.json \
|
||||
--markdown .artifacts/workflow-stress-report.md
|
||||
cat .artifacts/workflow-stress-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload Workflow stress performance
|
||||
if: success() && needs.changes.outputs.relevant == 'true' && matrix.name == 'agent-workflow-stress'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: workflow-stress-performance
|
||||
path: |
|
||||
.artifacts/workflow-stress-report.json
|
||||
.artifacts/workflow-stress-report.md
|
||||
e2e/fixtures/agent-workflow-stress/.eve/evals
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload eval artifacts
|
||||
if: (failure()) && needs.changes.outputs.relevant == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -4,6 +4,7 @@ import { equals } from "eve/evals/expect";
|
||||
const SESSION_COUNT = 50;
|
||||
const TURNS_PER_SESSION = 2;
|
||||
const TURN_COUNT = SESSION_COUNT * TURNS_PER_SESSION;
|
||||
const PERFORMANCE_LOG_PREFIX = "EVE_WORKFLOW_STRESS_METRIC=";
|
||||
|
||||
export default defineEval({
|
||||
description: "Workflow stress: 50 durable sessions complete 100 total turns.",
|
||||
@@ -11,26 +12,78 @@ export default defineEval({
|
||||
|
||||
async test(t) {
|
||||
const sessions = Array.from({ length: SESSION_COUNT }, () => t.newSession());
|
||||
const firstBatchStartedAt = performance.now();
|
||||
const firstTurns = await Promise.all(
|
||||
sessions.map((session, index) => session.send(markerFor(index, 1))),
|
||||
sessions.map(async (session, index) => {
|
||||
const startedAt = performance.now();
|
||||
const result = await session.send(markerFor(index, 1));
|
||||
|
||||
return {
|
||||
durationMs: performance.now() - startedAt,
|
||||
result,
|
||||
sessionNumber: index + 1,
|
||||
};
|
||||
}),
|
||||
);
|
||||
const firstBatchDurationMs = performance.now() - firstBatchStartedAt;
|
||||
firstTurns.forEach((turn, index) => {
|
||||
t.log(`workflow run id (${index + 1}/${SESSION_COUNT}): ${turn.sessionId}`);
|
||||
t.log(`workflow run id (${index + 1}/${SESSION_COUNT}): ${turn.result.sessionId}`);
|
||||
});
|
||||
const secondBatchStartedAt = performance.now();
|
||||
const secondTurns = await Promise.all(
|
||||
sessions.map((session, index) => session.send(markerFor(index, 2))),
|
||||
sessions.map(async (session, index) => {
|
||||
const startedAt = performance.now();
|
||||
const result = await session.send(markerFor(index, 2));
|
||||
|
||||
return {
|
||||
durationMs: performance.now() - startedAt,
|
||||
result,
|
||||
sessionNumber: index + 1,
|
||||
};
|
||||
}),
|
||||
);
|
||||
const secondBatchDurationMs = performance.now() - secondBatchStartedAt;
|
||||
|
||||
for (let index = 0; index < SESSION_COUNT; index += 1) {
|
||||
const first = firstTurns[index]!.expectOk();
|
||||
const second = secondTurns[index]!.expectOk();
|
||||
const first = firstTurns[index]!.result.expectOk();
|
||||
const second = secondTurns[index]!.result.expectOk();
|
||||
|
||||
await t.require(first.message, equals(`stress-ack:1:${markerFor(index, 1)}`));
|
||||
await t.require(second.message, equals(`stress-ack:2:${markerFor(index, 2)}`));
|
||||
await t.require(second.sessionId, equals(first.sessionId));
|
||||
}
|
||||
|
||||
await t.require(new Set(firstTurns.map((turn) => turn.sessionId)).size, equals(SESSION_COUNT));
|
||||
await t.require(
|
||||
new Set(firstTurns.map((turn) => turn.result.sessionId)).size,
|
||||
equals(SESSION_COUNT),
|
||||
);
|
||||
|
||||
t.log(
|
||||
`${PERFORMANCE_LOG_PREFIX}${JSON.stringify({
|
||||
batches: [
|
||||
{
|
||||
batchDurationMs: firstBatchDurationMs,
|
||||
samples: firstTurns.map(({ durationMs, sessionNumber }) => ({
|
||||
durationMs,
|
||||
sessionNumber,
|
||||
})),
|
||||
turnNumber: 1,
|
||||
},
|
||||
{
|
||||
batchDurationMs: secondBatchDurationMs,
|
||||
samples: secondTurns.map(({ durationMs, sessionNumber }) => ({
|
||||
durationMs,
|
||||
sessionNumber,
|
||||
})),
|
||||
turnNumber: 2,
|
||||
},
|
||||
],
|
||||
fixture: "agent-workflow-stress",
|
||||
scenario: "concurrent",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
})}`,
|
||||
);
|
||||
|
||||
t.succeeded();
|
||||
t.event("session.started", { count: SESSION_COUNT });
|
||||
|
||||
@@ -2,6 +2,7 @@ import { defineEval } from "eve/evals";
|
||||
import { equals } from "eve/evals/expect";
|
||||
|
||||
const TURN_COUNT = 100;
|
||||
const PERFORMANCE_LOG_PREFIX = "EVE_WORKFLOW_STRESS_METRIC=";
|
||||
|
||||
export default defineEval({
|
||||
description: "Workflow stress: one durable session completes 100 sequential turns.",
|
||||
@@ -9,12 +10,16 @@ export default defineEval({
|
||||
|
||||
async test(t) {
|
||||
let sessionId: string | undefined;
|
||||
const samples: Array<{ durationMs: number; turnNumber: number }> = [];
|
||||
|
||||
for (let turnNumber = 1; turnNumber <= TURN_COUNT; turnNumber += 1) {
|
||||
const marker = `sequential-turn-${String(turnNumber).padStart(3, "0")}`;
|
||||
const startedAt = performance.now();
|
||||
const result = await t.send(marker);
|
||||
const elapsedSeconds = (performance.now() - startedAt) / 1_000;
|
||||
const durationMs = performance.now() - startedAt;
|
||||
const elapsedSeconds = durationMs / 1_000;
|
||||
|
||||
samples.push({ durationMs, turnNumber });
|
||||
|
||||
t.log(
|
||||
`turn ${String(turnNumber).padStart(3, "0")}/${TURN_COUNT} completed in ${elapsedSeconds.toFixed(3)}s`,
|
||||
@@ -31,6 +36,17 @@ export default defineEval({
|
||||
await t.require(turn.message, equals(`stress-ack:${turnNumber}:${marker}`));
|
||||
}
|
||||
|
||||
t.log(
|
||||
`${PERFORMANCE_LOG_PREFIX}${JSON.stringify({
|
||||
fixture: "agent-workflow-stress",
|
||||
scenario: "sequential",
|
||||
schemaVersion: 1,
|
||||
sessionId,
|
||||
samples,
|
||||
unit: "milliseconds",
|
||||
})}`,
|
||||
);
|
||||
|
||||
t.succeeded();
|
||||
t.event("session.started", { count: 1 });
|
||||
t.event("turn.started", { count: TURN_COUNT });
|
||||
|
||||
@@ -390,9 +390,10 @@ describe("createNodeHarnessTools", () => {
|
||||
});
|
||||
|
||||
describe("createExecutionNodeStep", () => {
|
||||
it("builds a usable harness step for the root node", async () => {
|
||||
it("keeps the instrumentation drain on the root harness-step critical path", async () => {
|
||||
setupMockAgentForToolExecution("regular-tool", { question: "Run the tool." });
|
||||
const forceFlush = vi.fn(async () => undefined);
|
||||
const flush = Promise.withResolvers<void>();
|
||||
const forceFlush = vi.fn(() => flush.promise);
|
||||
const runtime: InstrumentationRuntime = {
|
||||
forceFlush,
|
||||
hooks: createInstrumentationHooks([]),
|
||||
@@ -454,25 +455,35 @@ describe("createExecutionNodeStep", () => {
|
||||
turn: { id: "root-turn", sequence: 0 },
|
||||
});
|
||||
|
||||
const result = await contextStorage.run(ctx, () =>
|
||||
step(
|
||||
createSession({
|
||||
continuationToken: "test-root",
|
||||
sessionId: "sess-root",
|
||||
turnAgent: rootNode.turnAgent,
|
||||
}),
|
||||
{
|
||||
message: "Run the tool.",
|
||||
},
|
||||
),
|
||||
);
|
||||
let stepSettled = false;
|
||||
const resultPromise = contextStorage
|
||||
.run(ctx, () =>
|
||||
step(
|
||||
createSession({
|
||||
continuationToken: "test-root",
|
||||
sessionId: "sess-root",
|
||||
turnAgent: rootNode.turnAgent,
|
||||
}),
|
||||
{
|
||||
message: "Run the tool.",
|
||||
},
|
||||
),
|
||||
)
|
||||
.finally(() => {
|
||||
stepSettled = true;
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(forceFlush).toHaveBeenCalledOnce());
|
||||
expect(stepSettled).toBe(false);
|
||||
flush.resolve();
|
||||
const result = await resultPromise;
|
||||
|
||||
expect(result.next).toEqual({ done: true, output: "tool-output" });
|
||||
expect(resolveRuntimeModelReference).toHaveBeenCalledWith(
|
||||
rootNode.turnAgent.model,
|
||||
modelResolutionScope,
|
||||
);
|
||||
expect(forceFlush).toHaveBeenCalledOnce();
|
||||
expect(stepSettled).toBe(true);
|
||||
});
|
||||
|
||||
it("records visible subagent tools as pending runtime actions", async () => {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import type { TurnCaller } from "#channel/types.js";
|
||||
import {
|
||||
notifyDelegatedParentStep,
|
||||
notifyTurnCallerStep,
|
||||
} from "#execution/delegated-parent-notification.js";
|
||||
import {
|
||||
createDelegatedSubagentErrorResult,
|
||||
createDelegatedSubagentSuccessResult,
|
||||
} from "#execution/delegated-parent-result.js";
|
||||
import type { DurableSessionState } from "#execution/durable-session-store.js";
|
||||
import type { NextDriverAction } from "#execution/next-driver-action.js";
|
||||
import { fireSessionCallbackStep } from "#execution/session-callback-step.js";
|
||||
import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";
|
||||
import { terminateChildSessionsStep } from "#execution/terminate-child-sessions-step.js";
|
||||
import type { RunMode } from "#shared/run-mode.js";
|
||||
import type { TokenUsage } from "#shared/token-usage.js";
|
||||
|
||||
export async function finalizeExpiredSession(input: {
|
||||
readonly caller: TurnCaller | undefined;
|
||||
readonly driverWritable: WritableStream<Uint8Array>;
|
||||
readonly mode: RunMode;
|
||||
readonly serializedContext: Record<string, unknown>;
|
||||
readonly sessionState: DurableSessionState;
|
||||
}): Promise<{ readonly output: unknown }> {
|
||||
await terminateChildSessionsStep({
|
||||
serializedContext: input.serializedContext,
|
||||
sessionState: input.sessionState,
|
||||
});
|
||||
await emitTerminalSessionCompletionStep({
|
||||
parentWritable: input.driverWritable,
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
|
||||
if (input.mode === "task") {
|
||||
await fireSessionCallbackStep({
|
||||
output: "",
|
||||
serializedContext: input.serializedContext,
|
||||
status: "completed",
|
||||
});
|
||||
await notifyDelegatedParentStep({
|
||||
result: createDelegatedSubagentSuccessResult(input.serializedContext, ""),
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
} else if (input.caller !== undefined) {
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.caller,
|
||||
lifecycle: "terminal",
|
||||
sessionId: input.sessionState.sessionId,
|
||||
settled: { output: "" },
|
||||
});
|
||||
}
|
||||
return { output: "" };
|
||||
}
|
||||
|
||||
export async function finalizeDone(input: {
|
||||
readonly action: NextDriverAction & { readonly kind: "done" };
|
||||
readonly caller: TurnCaller | undefined;
|
||||
readonly mode: RunMode;
|
||||
}): Promise<{ readonly output: unknown }> {
|
||||
const { output, serializedContext } = input.action;
|
||||
const failed = input.action.isError === true;
|
||||
|
||||
await terminateChildSessionsStep({
|
||||
serializedContext,
|
||||
sessionState: input.action.sessionState,
|
||||
});
|
||||
if (input.mode === "task") {
|
||||
await fireSessionCallbackStep({
|
||||
error: failed ? output : undefined,
|
||||
output: failed ? undefined : output,
|
||||
serializedContext,
|
||||
status: failed ? "failed" : "completed",
|
||||
usage: input.action.usage,
|
||||
});
|
||||
await notifyDelegatedParentStep({
|
||||
result: failed
|
||||
? createDelegatedSubagentErrorResult(serializedContext, output)
|
||||
: createDelegatedSubagentSuccessResult(serializedContext, output),
|
||||
serializedContext,
|
||||
usage: input.action.usage,
|
||||
});
|
||||
} else {
|
||||
const settled: {
|
||||
isError?: boolean;
|
||||
output: unknown;
|
||||
usage?: TokenUsage;
|
||||
} = { output, usage: input.action.usageDelta };
|
||||
if (failed) {
|
||||
settled.isError = true;
|
||||
}
|
||||
if (input.caller !== undefined) {
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.caller,
|
||||
lifecycle: "terminal",
|
||||
sessionId: input.action.sessionState.sessionId,
|
||||
settled,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { output };
|
||||
}
|
||||
@@ -699,6 +699,7 @@ describe("workflowEntry integration", () => {
|
||||
]);
|
||||
|
||||
const stream = captureTurnEvents(run);
|
||||
let completed = false;
|
||||
const hook = await waitForHook(
|
||||
{ runId: run.runId },
|
||||
{
|
||||
@@ -744,9 +745,17 @@ describe("workflowEntry integration", () => {
|
||||
event.data.message?.includes("follow up") === true,
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
await workflowRuntime.dispatchSession({
|
||||
command: { kind: "reset", reason: "Test step inventory" },
|
||||
sessionId: run.runId,
|
||||
});
|
||||
await expect(run.returnValue).resolves.toEqual({ output: "" });
|
||||
completed = true;
|
||||
expect(await listCallerStepNames(run.runId)).toEqual([]);
|
||||
} finally {
|
||||
stream.dispose();
|
||||
await run.cancel();
|
||||
if (!completed) await run.cancel();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -978,6 +987,13 @@ describe("workflowEntry integration", () => {
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(await listCallerStepNames(child.runId)).toEqual([
|
||||
"bindTurnCallerContextStep",
|
||||
"bindTurnCallerContextStep",
|
||||
"notifyTurnCallerStep",
|
||||
"notifyTurnCallerStep",
|
||||
"resolveInitialTurnCallerStep",
|
||||
]);
|
||||
} finally {
|
||||
stream.dispose();
|
||||
await child.cancel();
|
||||
@@ -1304,6 +1320,25 @@ describe("workflowEntry integration", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const CALLER_STEP_NAMES = new Set([
|
||||
"bindTurnCallerContextStep",
|
||||
"notifyTurnCallerStep",
|
||||
"resolveInitialTurnCallerStep",
|
||||
]);
|
||||
|
||||
async function listCallerStepNames(runId: string): Promise<string[]> {
|
||||
const world = await getWorld();
|
||||
const steps = await world.steps.list({
|
||||
pagination: { limit: 1_000 },
|
||||
resolveData: "none",
|
||||
runId,
|
||||
});
|
||||
return steps.data
|
||||
.map((step) => step.stepName.split("//").at(-1) ?? "")
|
||||
.filter((name) => CALLER_STEP_NAMES.has(name))
|
||||
.sort();
|
||||
}
|
||||
|
||||
interface CapturedEventStream {
|
||||
dispose(): void;
|
||||
nextUntil(
|
||||
|
||||
@@ -6,7 +6,9 @@ import type { HookPayload } from "#channel/types.js";
|
||||
import { ChannelRequestIdKey, SubagentDepthKey } from "#context/keys.js";
|
||||
import { createSessionStep } from "#execution/create-session-step.js";
|
||||
import {
|
||||
bindTurnCallerContextStep,
|
||||
notifyDelegatedParentStep,
|
||||
notifyTaskTurnStartedStep,
|
||||
notifyTurnCallerStep,
|
||||
resolveInitialTurnCallerStep,
|
||||
} from "#execution/delegated-parent-notification.js";
|
||||
@@ -213,6 +215,80 @@ describe("workflowEntry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("omits caller resolution, binding, and settlement steps for a root turn", async () => {
|
||||
const sessionState = createBaseSessionState();
|
||||
vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
|
||||
installHookMocks({
|
||||
deliveryHooks: [{ token: "http:test", values: [] }],
|
||||
turnControls: [
|
||||
turnResult({ action: "park", sessionState, settled: { output: "root answer" } }),
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
workflowEntry({
|
||||
input: { message: "hello" },
|
||||
serializedContext: createSerializedContext(),
|
||||
}),
|
||||
).resolves.toEqual({ output: "" });
|
||||
|
||||
expect(resolveInitialTurnCallerStep).not.toHaveBeenCalled();
|
||||
expect(bindTurnCallerContextStep).not.toHaveBeenCalled();
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains caller steps and task start notification for a callback task turn", async () => {
|
||||
const sessionState = createBaseSessionState();
|
||||
const caller = {
|
||||
callId: "call-task",
|
||||
replyTo: {
|
||||
kind: "callback" as const,
|
||||
token: "task:task-1:inbox",
|
||||
url: "https://parent.example.com/callback",
|
||||
},
|
||||
subagentName: "researcher",
|
||||
taskId: "task-1",
|
||||
};
|
||||
vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
|
||||
vi.mocked(resolveInitialTurnCallerStep).mockResolvedValueOnce(caller);
|
||||
installHookMocks({
|
||||
deliveryHooks: [{ token: "http:test", values: [] }],
|
||||
turnControls: [
|
||||
turnResult({ action: "park", sessionState, settled: { output: "task answer" } }),
|
||||
],
|
||||
});
|
||||
const serializedContext = createSerializedContext({
|
||||
"eve.sessionCallback": {
|
||||
callId: "call-task",
|
||||
subagentName: "researcher",
|
||||
taskId: "task-1",
|
||||
token: "task:task-1:inbox",
|
||||
url: "https://parent.example.com/callback",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(workflowEntry({ input: { message: "task" }, serializedContext })).resolves.toEqual(
|
||||
{ output: "" },
|
||||
);
|
||||
|
||||
expect(resolveInitialTurnCallerStep).toHaveBeenCalledExactlyOnceWith({ serializedContext });
|
||||
expect(notifyTaskTurnStartedStep).toHaveBeenCalledExactlyOnceWith({
|
||||
caller,
|
||||
childSessionId: "wrun_test_123",
|
||||
childTurnId: "turn_0",
|
||||
});
|
||||
expect(bindTurnCallerContextStep).toHaveBeenCalledExactlyOnceWith({
|
||||
caller,
|
||||
serializedContext: expect.objectContaining({ "eve.sessionCallback": expect.any(Object) }),
|
||||
});
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledExactlyOnceWith({
|
||||
caller,
|
||||
lifecycle: "parked",
|
||||
sessionId: "wrun_test_123",
|
||||
settled: { output: "task answer" },
|
||||
});
|
||||
});
|
||||
|
||||
it("finalizes children when the durable command inbox closes", async () => {
|
||||
const sessionState = createBaseSessionState();
|
||||
const serializedContext = {
|
||||
@@ -399,12 +475,7 @@ describe("workflowEntry", () => {
|
||||
Number.POSITIVE_INFINITY,
|
||||
);
|
||||
expect(fireSessionCallbackStep).not.toHaveBeenCalled();
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledWith({
|
||||
caller: undefined,
|
||||
lifecycle: "terminal",
|
||||
sessionId: "wrun_test_123",
|
||||
settled: { output: "" },
|
||||
});
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("dispatches a compact control without converting it into a delivery", async () => {
|
||||
@@ -653,13 +724,9 @@ describe("workflowEntry", () => {
|
||||
expect(terminateChildSessionsStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-resolve a caller the loop already settled and cleared", async () => {
|
||||
// A crash after a settled reply cleared the cell must notify no one:
|
||||
// `caller: undefined` with the resolution flag set means "nothing left
|
||||
// to notify", not "never resolved".
|
||||
it("does not resolve or notify a caller when a root turn crashes", async () => {
|
||||
const sessionState = createBaseSessionState();
|
||||
vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
|
||||
vi.mocked(resolveInitialTurnCallerStep).mockResolvedValueOnce(undefined);
|
||||
installHookMocks({
|
||||
turnControls: [
|
||||
{
|
||||
@@ -676,10 +743,8 @@ describe("workflowEntry", () => {
|
||||
}),
|
||||
).rejects.toMatchObject({ name: "EveWorkflowFailure" });
|
||||
|
||||
expect(resolveInitialTurnCallerStep).toHaveBeenCalledOnce();
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ caller: undefined }),
|
||||
);
|
||||
expect(resolveInitialTurnCallerStep).not.toHaveBeenCalled();
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("notifies the latest delegated exchange when a resumed turn fails terminally", async () => {
|
||||
@@ -839,7 +904,17 @@ describe("workflowEntry", () => {
|
||||
await expect(
|
||||
workflowEntry({
|
||||
input: { message: "delegate" },
|
||||
serializedContext: createSerializedContext(),
|
||||
serializedContext: createSerializedContext({
|
||||
"eve.channel": {
|
||||
kind: "subagent",
|
||||
state: {
|
||||
callId: "call-1",
|
||||
parentContinuationToken: "parent-turn",
|
||||
parentSessionId: "parent-session",
|
||||
subagentName: "researcher",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).resolves.toEqual({ output: "" });
|
||||
|
||||
@@ -1078,12 +1153,7 @@ describe("workflowEntry", () => {
|
||||
serializedContext: { settled: true },
|
||||
sessionState: settledState,
|
||||
});
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledExactlyOnceWith({
|
||||
caller: undefined,
|
||||
lifecycle: "terminal",
|
||||
sessionId: "wrun_test_123",
|
||||
settled: { output: "ok" },
|
||||
});
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not settle an ordinary park as cancelled", async () => {
|
||||
@@ -1103,7 +1173,7 @@ describe("workflowEntry", () => {
|
||||
expect(settleCancelledTurnStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("routes every settled conversation turn without classifying the session", async () => {
|
||||
it("does not schedule caller notification for a settled root turn", async () => {
|
||||
const sessionState = createBaseSessionState();
|
||||
vi.mocked(createSessionStep).mockResolvedValue(createSessionStepResultForMock(sessionState));
|
||||
installHookMocks({
|
||||
@@ -1125,12 +1195,7 @@ describe("workflowEntry", () => {
|
||||
}),
|
||||
).resolves.toEqual({ output: "" });
|
||||
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledWith({
|
||||
caller: undefined,
|
||||
lifecycle: "parked",
|
||||
sessionId: "wrun_test_123",
|
||||
settled: { output: "hello" },
|
||||
});
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-send a settled parent turn when a delivery routes to a child", async () => {
|
||||
@@ -1165,7 +1230,7 @@ describe("workflowEntry", () => {
|
||||
}),
|
||||
).resolves.toEqual({ output: "" });
|
||||
|
||||
expect(notifyTurnCallerStep).toHaveBeenCalledTimes(1);
|
||||
expect(notifyTurnCallerStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("adopts retired proxy state before the next parked driver turn", async () => {
|
||||
|
||||
@@ -19,18 +19,15 @@ import {
|
||||
notifyTurnCallerStep,
|
||||
resolveInitialTurnCallerStep,
|
||||
} from "#execution/delegated-parent-notification.js";
|
||||
import {
|
||||
createDelegatedSubagentErrorResult,
|
||||
createDelegatedSubagentSuccessResult,
|
||||
} from "#execution/delegated-parent-result.js";
|
||||
import { createDelegatedSubagentErrorResult } from "#execution/delegated-parent-result.js";
|
||||
import type { DurableSessionState } from "#execution/durable-session-store.js";
|
||||
import type { NextDriverAction } from "#execution/next-driver-action.js";
|
||||
import { nextTurnDelivery, type NextTurnInstruction } from "#execution/parked-delivery-wait.js";
|
||||
import { SessionStateCursor } from "#execution/session-state-cursor.js";
|
||||
import { cancelDescendantTurnsStep } from "#execution/cancel-descendant-turns-step.js";
|
||||
import { dispatchAndAwaitTurn } from "#execution/turn-dispatch.js";
|
||||
import type { TurnDriverAction } from "#execution/turn-control-receiver.js";
|
||||
import { normalizeSerializableError } from "#execution/workflow-errors.js";
|
||||
import { finalizeDone, finalizeExpiredSession } from "#execution/workflow-entry-finalization.js";
|
||||
import { createSessionStep } from "#execution/create-session-step.js";
|
||||
import { settleCancelledTurnStep } from "#execution/settle-cancelled-turn-step.js";
|
||||
import { emitTerminalSessionFailureStep } from "#execution/terminal-session-failure-step.js";
|
||||
@@ -40,14 +37,14 @@ import { createSessionCommandInbox } from "#execution/session-command-inbox.js";
|
||||
import { activeTurnId } from "#harness/active-turn-id.js";
|
||||
import { sessionCommandHookToken } from "#execution/session-command-token.js";
|
||||
import { DEFAULT_SESSION_TIMEOUT_MS } from "#execution/session-timeout.js";
|
||||
import { emitTerminalSessionCompletionStep } from "#execution/terminal-session-completion-step.js";
|
||||
import { createSessionTimeoutControl } from "#execution/session-timeout-control.js";
|
||||
import { terminateChildSessionsStep } from "#execution/terminate-child-sessions-step.js";
|
||||
import { readSerializedSubagentDepth } from "#harness/subagent-depth.js";
|
||||
import type { DynamicSubagentAgentConfig } from "#runtime/subagents/dynamic-agent-config.js";
|
||||
import type { TokenUsage } from "#shared/token-usage.js";
|
||||
import { isTaskOwnedSerializedContext } from "#execution/tasks/child/instructions.js";
|
||||
import { attachClientContext, readClientContext } from "#internal/client-context.js";
|
||||
import { CHANNEL_CONTEXT_KEY_NAME, SESSION_CALLBACK_CONTEXT_KEY_NAME } from "#context/key-names.js";
|
||||
import { SUBAGENT_ADAPTER_KIND } from "#execution/subagent-adapter-state.js";
|
||||
|
||||
const SAFE_OUTER_WORKFLOW_FAILURE_MESSAGE =
|
||||
"Agent workflow failed. Inspect the private session trace for details.";
|
||||
@@ -102,13 +99,14 @@ type DriverLoopOutcome =
|
||||
interface CrashCleanupState {
|
||||
// The caller whose awaited reply is still unsettled, so the catch can
|
||||
// reject it with the error instead of leaving it parked forever.
|
||||
// Populated for every session; only conversation-mode paths read it.
|
||||
// Root sessions leave it undefined; only conversation-mode paths read it.
|
||||
caller: TurnCaller | undefined;
|
||||
// Whether `resolveInitialTurnCallerStep` has run. `caller: undefined` is
|
||||
// ambiguous on its own: it also means "resolved and later cleared because
|
||||
// its reply settled". This flag lets the crash path tell that apart from
|
||||
// "crashed before the caller was ever resolved", where a delegated caller
|
||||
// may still be parked on this session's reply.
|
||||
// Whether initial lineage has been classified, either by proving it is a
|
||||
// root session or by resolving its caller. `caller: undefined` is ambiguous
|
||||
// on its own: it also means "resolved and later cleared because its reply
|
||||
// settled". This flag lets the crash path tell that apart from "crashed
|
||||
// before lineage was classified", where a delegated caller may still be
|
||||
// parked on this session's reply.
|
||||
callerResolved: boolean;
|
||||
// The latest snapshot the driver has received, so the catch can
|
||||
// terminate children adopted after turn 1. Honest staleness window: the
|
||||
@@ -127,7 +125,7 @@ interface CrashCleanupState {
|
||||
*
|
||||
* Owns the stable command inbox, its channel alias, and the session lifecycle; each turn-owned
|
||||
* turn resolves its own runtime actions in-line and reports back only
|
||||
* `done`/`park` via the closed-contract {@link NextDriverAction}. The
|
||||
* `done`/`park` via the closed `NextDriverAction` contract. The
|
||||
* only session-shape flag the driver reads (besides identity) is
|
||||
* `hasProxyInputRequests`, the documented short-circuit for hook-payload
|
||||
* routing to any descendant still active when the parent parks.
|
||||
@@ -179,12 +177,11 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
|
||||
taskOwned: isTaskOwnedSerializedContext(input.serializedContext),
|
||||
});
|
||||
crashCleanupState.lastSessionState = sessionState;
|
||||
// Resolved for every session so the cell's population never depends
|
||||
// on session shape: the step returns undefined for root sessions,
|
||||
// and every reader is mode-gated.
|
||||
crashCleanupState.caller = await resolveInitialTurnCallerStep({
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
if (!hasGuaranteedRootLineage(input.serializedContext)) {
|
||||
crashCleanupState.caller = await resolveInitialTurnCallerStep({
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
}
|
||||
crashCleanupState.callerResolved = true;
|
||||
|
||||
const outcome = await runDriverLoop({
|
||||
@@ -263,12 +260,15 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
} else {
|
||||
await notifyTurnCallerStep({
|
||||
caller: await resolveCallerForCrash(crashCleanupState, input.serializedContext),
|
||||
lifecycle: "terminal",
|
||||
sessionId,
|
||||
settled: { isError: true, output: error },
|
||||
});
|
||||
const caller = await resolveCallerForCrash(crashCleanupState, input.serializedContext);
|
||||
if (caller !== undefined) {
|
||||
await notifyTurnCallerStep({
|
||||
caller,
|
||||
lifecycle: "terminal",
|
||||
sessionId,
|
||||
settled: { isError: true, output: error },
|
||||
});
|
||||
}
|
||||
}
|
||||
throw createSafeOuterWorkflowError();
|
||||
}
|
||||
@@ -278,11 +278,11 @@ export async function workflowEntry(input: WorkflowEntryInput): Promise<Workflow
|
||||
* Caller to reject from the crash path. Normally the resolved cell value —
|
||||
* including `undefined` after a settled reply cleared it, when there is
|
||||
* nothing left to notify. When the crash happened before
|
||||
* `resolveInitialTurnCallerStep` ever ran (e.g. `createSessionStep` threw),
|
||||
* the cell is empty even though a delegated caller may be parked on this
|
||||
* session's reply, so the caller is re-resolved from the serialized context
|
||||
* — which needs nothing from the failed steps. Best-effort: when resolution
|
||||
* fails again there is no reachable caller to notify.
|
||||
* initial lineage was classified (e.g. `createSessionStep` threw), the cell is
|
||||
* empty even though a delegated caller may be parked on this session's reply,
|
||||
* so the caller is resolved from the serialized context — which needs nothing
|
||||
* from the failed steps. Best-effort: when resolution fails there is no
|
||||
* reachable caller to notify.
|
||||
*/
|
||||
async function resolveCallerForCrash(
|
||||
state: CrashCleanupState,
|
||||
@@ -304,6 +304,18 @@ function createSafeOuterWorkflowError(): Error {
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Root lineage has no turn caller to resolve, bind, or notify. */
|
||||
function hasGuaranteedRootLineage(serializedContext: Record<string, unknown>): boolean {
|
||||
if (serializedContext[SESSION_CALLBACK_CONTEXT_KEY_NAME] !== undefined) return false;
|
||||
if (serializedContext["eve.parentSession"] !== undefined) return false;
|
||||
if (serializedContext["eve.subagentDepth"] !== undefined) return false;
|
||||
|
||||
const channel = serializedContext[CHANNEL_CONTEXT_KEY_NAME];
|
||||
if (channel === null || typeof channel !== "object") return false;
|
||||
const kind = Reflect.get(channel, "kind");
|
||||
return typeof kind === "string" && kind !== SUBAGENT_ADAPTER_KIND;
|
||||
}
|
||||
|
||||
async function runDriverLoop(input: {
|
||||
readonly capabilities?: SessionCapabilities;
|
||||
readonly driverWritable: WritableStream<Uint8Array>;
|
||||
@@ -430,10 +442,13 @@ async function runDriverLoop(input: {
|
||||
childTurnId: activeTurnId(stateCursor.sessionState.emissionState),
|
||||
});
|
||||
}
|
||||
const serializedContext = await bindTurnCallerContextStep({
|
||||
caller,
|
||||
serializedContext: stateCursor.serializedContext,
|
||||
});
|
||||
const serializedContext =
|
||||
caller === undefined
|
||||
? stateCursor.serializedContext
|
||||
: await bindTurnCallerContextStep({
|
||||
caller,
|
||||
serializedContext: stateCursor.serializedContext,
|
||||
});
|
||||
const turn = await dispatchAndAwaitTurn({
|
||||
bufferedDeliveries,
|
||||
bufferedSessionControls,
|
||||
@@ -520,12 +535,14 @@ async function runDriverLoop(input: {
|
||||
// the full StepResult so no state-key fallback exists anymore.
|
||||
const settled = action.settled;
|
||||
if (action.cancelled !== true && settled !== undefined) {
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.crashCleanupState.caller,
|
||||
lifecycle: "parked",
|
||||
sessionId: stateCursor.sessionState.sessionId,
|
||||
settled,
|
||||
});
|
||||
if (input.crashCleanupState.caller !== undefined) {
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.crashCleanupState.caller,
|
||||
lifecycle: "parked",
|
||||
sessionId: stateCursor.sessionState.sessionId,
|
||||
settled,
|
||||
});
|
||||
}
|
||||
input.crashCleanupState.caller = undefined;
|
||||
} else if (action.cancelled === true) {
|
||||
input.crashCleanupState.caller = undefined;
|
||||
@@ -610,86 +627,3 @@ async function runDriverLoop(input: {
|
||||
await commandInbox.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
async function finalizeExpiredSession(input: {
|
||||
readonly caller: TurnCaller | undefined;
|
||||
readonly driverWritable: WritableStream<Uint8Array>;
|
||||
readonly mode: RunMode;
|
||||
readonly serializedContext: Record<string, unknown>;
|
||||
readonly sessionState: DurableSessionState;
|
||||
}): Promise<WorkflowEntryResult> {
|
||||
await terminateChildSessionsStep({
|
||||
serializedContext: input.serializedContext,
|
||||
sessionState: input.sessionState,
|
||||
});
|
||||
await emitTerminalSessionCompletionStep({
|
||||
parentWritable: input.driverWritable,
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
|
||||
if (input.mode === "task") {
|
||||
await fireSessionCallbackStep({
|
||||
output: "",
|
||||
serializedContext: input.serializedContext,
|
||||
status: "completed",
|
||||
});
|
||||
await notifyDelegatedParentStep({
|
||||
result: createDelegatedSubagentSuccessResult(input.serializedContext, ""),
|
||||
serializedContext: input.serializedContext,
|
||||
});
|
||||
} else {
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.caller,
|
||||
lifecycle: "terminal",
|
||||
sessionId: input.sessionState.sessionId,
|
||||
settled: { output: "" },
|
||||
});
|
||||
}
|
||||
return { output: "" };
|
||||
}
|
||||
|
||||
async function finalizeDone(input: {
|
||||
readonly action: NextDriverAction & { readonly kind: "done" };
|
||||
readonly caller: TurnCaller | undefined;
|
||||
readonly mode: RunMode;
|
||||
}): Promise<WorkflowEntryResult> {
|
||||
const { output, serializedContext } = input.action;
|
||||
const failed = input.action.isError === true;
|
||||
|
||||
await terminateChildSessionsStep({
|
||||
serializedContext,
|
||||
sessionState: input.action.sessionState,
|
||||
});
|
||||
if (input.mode === "task") {
|
||||
await fireSessionCallbackStep({
|
||||
error: failed ? output : undefined,
|
||||
output: failed ? undefined : output,
|
||||
serializedContext,
|
||||
status: failed ? "failed" : "completed",
|
||||
usage: input.action.usage,
|
||||
});
|
||||
await notifyDelegatedParentStep({
|
||||
result: failed
|
||||
? createDelegatedSubagentErrorResult(serializedContext, output)
|
||||
: createDelegatedSubagentSuccessResult(serializedContext, output),
|
||||
serializedContext,
|
||||
usage: input.action.usage,
|
||||
});
|
||||
} else {
|
||||
const settled: {
|
||||
isError?: boolean;
|
||||
output: unknown;
|
||||
usage?: TokenUsage;
|
||||
} = { output, usage: input.action.usageDelta };
|
||||
if (failed) {
|
||||
settled.isError = true;
|
||||
}
|
||||
await notifyTurnCallerStep({
|
||||
caller: input.caller,
|
||||
lifecycle: "terminal",
|
||||
sessionId: input.action.sessionState.sessionId,
|
||||
settled,
|
||||
});
|
||||
}
|
||||
return { output };
|
||||
}
|
||||
|
||||
@@ -113,6 +113,7 @@ vi.mock("ai", () => ({
|
||||
const {
|
||||
mockCreateAiSdkHookBridge,
|
||||
mockGetRegisteredTelemetryIntegrations,
|
||||
mockSetEveAttributes,
|
||||
registeredAuthorIntegration,
|
||||
registeredOtelIntegration,
|
||||
} = vi.hoisted(() => ({
|
||||
@@ -120,6 +121,9 @@ const {
|
||||
mockGetRegisteredTelemetryIntegrations: vi.fn(
|
||||
(_options?: { readonly sanitizeEveOtelErrors?: boolean }): unknown[] => [],
|
||||
),
|
||||
mockSetEveAttributes: vi.fn<(attrs: Record<string, unknown>) => Promise<void>>(
|
||||
async () => undefined,
|
||||
),
|
||||
registeredAuthorIntegration: { onStart: vi.fn() },
|
||||
registeredOtelIntegration: { onStart: vi.fn() },
|
||||
}));
|
||||
@@ -134,6 +138,10 @@ vi.mock("#instrumentation/ai-sdk-telemetry.js", () => ({
|
||||
mockGetRegisteredTelemetryIntegrations(options),
|
||||
}));
|
||||
|
||||
vi.mock("#runtime/attributes/emit.js", () => ({
|
||||
setEveAttributes: (attrs: Record<string, unknown>) => mockSetEveAttributes(attrs),
|
||||
}));
|
||||
|
||||
let declaredAudience: ChannelAudience = "unknown";
|
||||
let declaredDecision: InstrumentationDecision | undefined;
|
||||
let declaredInstrumentation: SessionInstrumentation | undefined;
|
||||
@@ -241,6 +249,7 @@ afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
declareTelemetry(undefined);
|
||||
mockGetRegisteredTelemetryIntegrations.mockReturnValue([]);
|
||||
mockSetEveAttributes.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
function createTestSession(overrides?: Partial<HarnessSession>): HarnessSession {
|
||||
@@ -1990,6 +1999,61 @@ describe("createToolLoopHarness", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("emits the terminal result while Workflow attributes persist, then joins", async () => {
|
||||
setupMockAgent({
|
||||
finishReason: "stop",
|
||||
response: { messages: [{ content: "Hello!", role: "assistant" }] },
|
||||
text: "Hello!",
|
||||
toolCalls: [],
|
||||
toolResults: [],
|
||||
});
|
||||
const attributeWrite = Promise.withResolvers<void>();
|
||||
mockSetEveAttributes.mockReturnValueOnce(attributeWrite.promise);
|
||||
const { emit, events } = createEventCollector();
|
||||
const result = createToolLoopHarness(createTestConfig("conversation", emit))(
|
||||
createTestSession(),
|
||||
{ message: "Hi" },
|
||||
);
|
||||
|
||||
try {
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
expect(events.at(-1)?.type).toBe("session.waiting");
|
||||
},
|
||||
{ timeout: 1_000 },
|
||||
);
|
||||
|
||||
let settled = false;
|
||||
void result.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
} finally {
|
||||
attributeWrite.resolve();
|
||||
await result;
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an unexpected Workflow attribute rejection out of the turn result", async () => {
|
||||
setupMockAgent({
|
||||
finishReason: "stop",
|
||||
response: { messages: [{ content: "Hello!", role: "assistant" }] },
|
||||
text: "Hello!",
|
||||
toolCalls: [],
|
||||
toolResults: [],
|
||||
});
|
||||
mockSetEveAttributes.mockRejectedValueOnce(new Error("attribute write rejected"));
|
||||
const { emit, events } = createEventCollector();
|
||||
|
||||
await expect(
|
||||
createToolLoopHarness(createTestConfig("conversation", emit))(createTestSession(), {
|
||||
message: "Hi",
|
||||
}),
|
||||
).resolves.toMatchObject({ next: null });
|
||||
expect(events.at(-1)?.type).toBe("session.waiting");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
details: {
|
||||
|
||||
@@ -1747,7 +1747,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
|
||||
} catch {
|
||||
modelTag = undefined;
|
||||
}
|
||||
await setEveAttributes({
|
||||
const attributeWrite = setEveAttributes({
|
||||
"$eve.model": modelTag,
|
||||
"$eve.input_tokens": nextTurnUsage.inputTokens,
|
||||
"$eve.output_tokens": nextTurnUsage.outputTokens,
|
||||
@@ -1755,23 +1755,40 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
|
||||
"$eve.cache_write_tokens": nextTurnUsage.cacheWriteTokens,
|
||||
"$eve.cost_usd": nextTurnUsage.sawCost ? nextTurnUsage.costUsd : undefined,
|
||||
"$eve.tool_count": config.tools.size,
|
||||
}).catch((error: unknown) => {
|
||||
// `setEveAttributes` owns this best-effort boundary. Keep the overlap
|
||||
// defensive if a future implementation accidentally lets a rejection
|
||||
// escape before result handling reaches the join below.
|
||||
logError(log, "Workflow attribute write failed unexpectedly", error, {
|
||||
sessionId: session.sessionId,
|
||||
turnId: emissionState.turnId,
|
||||
});
|
||||
});
|
||||
|
||||
// --- Handle result ------------------------------------------------------
|
||||
|
||||
return handleStepResult({
|
||||
config,
|
||||
emit,
|
||||
emissionState,
|
||||
delegatedCaller: taskUpdatesEnabled,
|
||||
durableModelPromptMessageCount:
|
||||
ephemeralContextMessages.length === 0 ? projectedMessages.length : undefined,
|
||||
promptMessages: messages,
|
||||
result,
|
||||
runStep,
|
||||
session,
|
||||
runtimeActionTools: modelCallRuntimeActionTools,
|
||||
});
|
||||
// Overlap the best-effort attribute write with result processing, including
|
||||
// the terminal stream epilogue, but keep it inside this step's lifetime so
|
||||
// cumulative writes cannot reorder or race the workflow's terminal state.
|
||||
let stepResult: StepResult;
|
||||
try {
|
||||
stepResult = await handleStepResult({
|
||||
config,
|
||||
emit,
|
||||
emissionState,
|
||||
delegatedCaller: taskUpdatesEnabled,
|
||||
durableModelPromptMessageCount:
|
||||
ephemeralContextMessages.length === 0 ? projectedMessages.length : undefined,
|
||||
promptMessages: messages,
|
||||
result,
|
||||
runStep,
|
||||
session,
|
||||
runtimeActionTools: modelCallRuntimeActionTools,
|
||||
});
|
||||
} finally {
|
||||
await attributeWrite;
|
||||
}
|
||||
return stepResult;
|
||||
}
|
||||
|
||||
return runStep;
|
||||
|
||||
@@ -0,0 +1,891 @@
|
||||
---
|
||||
issue: https://github.com/vercel/eve/issues/876
|
||||
status: proposed
|
||||
last_updated: "2026-09-01"
|
||||
---
|
||||
|
||||
# Turn performance and Workflow overhead
|
||||
|
||||
## Summary
|
||||
|
||||
eve's hosted turn latency is a product problem, not a model-speed problem. The deterministic
|
||||
Vercel Workflow stress fixture currently takes about 3.1 seconds per sequential turn on the
|
||||
existing Workflow SDK, while production reports show 3.5–7 seconds before a model step starts.
|
||||
Local profiling reaches the same directional conclusion: a warm mock `turnStep` takes about
|
||||
50 ms, but the observed follow-up path takes about 353 ms, with 86% of that time outside the
|
||||
model/tool step.
|
||||
|
||||
The fixed cost comes from the durable topology. An ordinary root turn starts a child Workflow
|
||||
run, crosses five explicit step boundaries for full settlement, creates private hooks, writes
|
||||
8–9 stream chunks, resumes the session driver twice, and carries the full session snapshot through
|
||||
multiple persisted values. Workflow already group-commits adjacent stream chunks at the World
|
||||
boundary, so chunk count is not equivalent to persistence-request count. Two of the explicit
|
||||
steps perform no work for a root session. Every additional model/tool cycle schedules another
|
||||
`turnStep` and performs another Workflow attributes write.
|
||||
|
||||
This proposal makes performance measurable before changing the topology, then tests three levels
|
||||
of improvement:
|
||||
|
||||
1. remove unnecessary steps and observability writes from the critical path;
|
||||
2. reduce per-turn child-run, hook, stream-write, and state-transfer overhead;
|
||||
3. prototype a bounded run-per-turn architecture that preserves durable session semantics without
|
||||
replaying one ever-growing driver run.
|
||||
|
||||
The first tracking increment lands with this plan: the existing stress fixture emits raw
|
||||
machine-readable samples for sequential and concurrent turns, CI publishes a GitHub job summary,
|
||||
and successful runs retain the report and eval artifacts for 30 days. Those measurements are
|
||||
informational until paired base/head trials establish the hosted noise floor.
|
||||
|
||||
The same increment includes one conservative critical-path change. Best-effort Workflow
|
||||
attributes now start in parallel with result settlement, allowing the user-visible terminal event
|
||||
to persist first, but the durable step still joins the attribute write before it exits. This
|
||||
preserves cumulative-write ordering and identical local and hosted lifetime behavior.
|
||||
|
||||
The independent hosted trials now identify one mergeable result and one architectural target.
|
||||
Removing guaranteed root-only caller steps reduces sequential mean by 33.1% in isolation; the
|
||||
current PR combination reduces mean by 34.0%, p50 by 33.9%, and p95 by 36.6%. A benchmark-only
|
||||
inline-turn prototype reduces p50 by 63.5%, and its combination with the root optimization
|
||||
reproduces 0.799–0.828-second p50s. That fast path cannot ship because it loses live-deployment,
|
||||
cancellation, and runtime-wait semantics, but it proves that eve's parent/child topology—not the
|
||||
hosted platform—is the largest remaining fixed cost.
|
||||
|
||||
The narrow background-work pass found no additional hot-path await that can safely move to
|
||||
`ctx.waitUntil` today. Request-route work already uses Nitro's lifetime primitive; Workflow steps
|
||||
have no supported equivalent. Stream writes are already group-committed, the stress fixture's
|
||||
instrumentation flush is a no-op, and detaching attributes, hook operations, or terminal cleanup
|
||||
would weaken persistence ordering, retry safety, or cancellation. The useful follow-up is a small
|
||||
set of explicit Workflow primitives, not an eve fire-and-forget shim.
|
||||
|
||||
## Observed baseline
|
||||
|
||||
The current stress fixture uses a synchronous deterministic mock model, so its latency is almost
|
||||
entirely eve and Workflow overhead. Recent unchanged/main-equivalent Vercel runs cluster at:
|
||||
|
||||
| Metric | Observed range |
|
||||
| ------------------------------------------------------ | -----------------: |
|
||||
| Mean, 100 sequential turns | 3.08–3.19 s |
|
||||
| p50 | 3.02–3.14 s |
|
||||
| p95 | 3.82–3.94 s |
|
||||
| Mean, turns 1–10 | 2.42–2.75 s |
|
||||
| Mean, turns 91–100 | 3.58–3.75 s |
|
||||
| Sequential turn-order slope in two representative runs | +13.6–14.6 ms/turn |
|
||||
|
||||
Representative GitHub runs are
|
||||
[`33449386170`](https://github.com/vercel/eve/actions/runs/33449386170) and
|
||||
[`33445215572`](https://github.com/vercel/eve/actions/runs/33445215572).
|
||||
|
||||
The turn-order slope is correlated with history depth but does not identify its cause: history,
|
||||
event-log growth, queue drift, changing hosted load, and warming all move with sequential test
|
||||
order. It makes the trend visible; only an interleaved benchmark over independently pre-seeded
|
||||
history depths can call the resulting coefficient a history-depth slope.
|
||||
|
||||
The Workflow dependency update in
|
||||
[PR #2611](https://github.com/vercel/eve/pull/2611) produced two runs near 4.91 seconds mean,
|
||||
5.05–5.20 seconds p50, and 6.44–6.52 seconds p95. That is a useful demonstration that this fixture
|
||||
can expose a material regression, but it is correlation rather than causal proof: the runs were
|
||||
not an interleaved base/head experiment, and the upgrade changed several aligned Workflow
|
||||
packages. The beta.47 change that persists `hook_received` before publishing a wake is a leading
|
||||
hypothesis because eve resumes two hot-path hooks per turn; it needs an isolated SDK A/B.
|
||||
[The Workflow change](https://github.com/vercel/workflow/pull/3841) explicitly adds one
|
||||
producer-side event-write round trip but expects end-to-end time-to-resume to remain approximately
|
||||
neutral because the consumer no longer performs the corresponding ensure write.
|
||||
|
||||
Customer evidence is consistent with the stress fixture:
|
||||
|
||||
- [Issue #876](https://github.com/vercel/eve/issues/876) records about 3.5 seconds from a warm
|
||||
channel webhook to model-step start, followed by only about 0.4 seconds of model work. Later
|
||||
reports observe 5–7 second dispatch/orchestration gaps.
|
||||
- [Issue #1476](https://github.com/vercel/eve/issues/1476) reports 1.1–5.5 second new-session create
|
||||
latency. `createSession()` currently waits for the workflow to own its stable command hook
|
||||
before returning the already-known run id.
|
||||
|
||||
These sources measure different boundaries and must not be averaged together. The stress
|
||||
fixture's `t.send()` resolves at the streamed `session.waiting` event, before the child step and
|
||||
parent driver finish their whole tail. Back-to-back turns can therefore charge the previous
|
||||
turn's tail to the next request. It is a valid user-visible response/throughput indicator, but not
|
||||
an isolated measure of every phase.
|
||||
|
||||
### Local attribution
|
||||
|
||||
Three runs of the focused workflow-entry integration test on the local file-backed Workflow world
|
||||
produced these warm medians:
|
||||
|
||||
| Phase | Median |
|
||||
| ----------------------------------------- | -----: |
|
||||
| Root bind start → `session.waiting` write | 353 ms |
|
||||
| `turnStep` | 50 ms |
|
||||
| Time outside `turnStep` | 303 ms |
|
||||
| Root caller-context bind step | 44 ms |
|
||||
| Child dispatch step | 48 ms |
|
||||
| Child start → `turnStep` start | 139 ms |
|
||||
| Terminal control-send step | 50 ms |
|
||||
| Root caller-notification step | 46 ms |
|
||||
|
||||
The run is repeatable with:
|
||||
|
||||
```sh
|
||||
pnpm --filter eve build:js
|
||||
pnpm --filter eve exec vitest run \
|
||||
--config vitest.integration.config.ts \
|
||||
src/execution/workflow-entry.integration.test.ts \
|
||||
-t "parks in conversation mode and resumes via runtime delivery" \
|
||||
--reporter=dot
|
||||
```
|
||||
|
||||
The phase timestamps, persisted input/output sizes, and stream-chunk counts were read from
|
||||
`packages/eve/.eve/.workflow-data/vitest-1/{runs,steps,streams}` after each of three clean runs.
|
||||
This is directional attribution, not the success gate. Phase 1 promotes the extraction into a
|
||||
committed reporter before an optimization relies on it.
|
||||
|
||||
The local world is not a hosted latency predictor, but the attribution is decisive: fixed durable
|
||||
boundaries dominate even without network or model work. A two-turn run persisted 17 stream chunks.
|
||||
The dispatch input also grew from 1,103 bytes to 1,579 bytes after one turn because the durable
|
||||
session snapshot embeds full history.
|
||||
|
||||
A one-off codec spot check put devalue and compression CPU orders of magnitude below the measured
|
||||
durable boundaries. It was not a committed benchmark and is not used as proof. The paired
|
||||
benchmark should retain a reproducible codec/payload-size microcase; until then, transport,
|
||||
storage, and replay of repeated snapshots remain in scope while codec micro-optimization does not.
|
||||
|
||||
## Current critical path
|
||||
|
||||
```text
|
||||
client delivery
|
||||
└─ resume stable session hook
|
||||
└─ replay long-lived session driver
|
||||
├─ bind caller step (no-op for root sessions)
|
||||
├─ dispatch step
|
||||
│ └─ start a latest-deployment child turn workflow
|
||||
│ ├─ claim private inbox hook
|
||||
│ ├─ claim cancellation hook
|
||||
│ ├─ turnStep × model/tool cycles
|
||||
│ │ ├─ rebuild context, bundle, harness, and tools
|
||||
│ │ ├─ enqueue each ordered protocol stream chunk
|
||||
│ │ └─ overlap attributes with result handling, then join
|
||||
│ ├─ terminal control-send step
|
||||
│ └─ dispose private hooks
|
||||
├─ resume and replay session driver
|
||||
├─ adopt full returned state
|
||||
├─ notify caller step (no-op for root sessions)
|
||||
└─ rekey/park on the stable inbox
|
||||
```
|
||||
|
||||
The child workflow exists for important reasons: a long-lived session driver stays pinned to its
|
||||
originating deployment, while each child turn can run the latest deployment; the child also owns
|
||||
turn cancellation and waits that span tool/subagent activity. An optimization cannot simply
|
||||
delete it without replacing those semantics.
|
||||
|
||||
There are three independent performance dimensions:
|
||||
|
||||
```text
|
||||
turn latency = fixed ingress/turn orchestration
|
||||
+ model-step count × incremental durable-step overhead
|
||||
+ history depth × replay/state-transfer growth
|
||||
+ actual model, tool, hook, and provider work
|
||||
```
|
||||
|
||||
A single 100-turn mean hides whether a change affects the fixed intercept, per-step slope, or
|
||||
history slope. The benchmark must report all three.
|
||||
|
||||
## Measurement system
|
||||
|
||||
### Tracking added now
|
||||
|
||||
The Vercel stress fixture now records raw millisecond samples in a versioned JSON log record:
|
||||
|
||||
- sequential: every turn number and duration;
|
||||
- concurrent: every session's first- and second-turn duration plus batch makespan;
|
||||
- report: count, mean, p50, p90, p95, min, max, cold/warm buckets, and sequential
|
||||
turn-order slope, explicitly labeled as time-order confounded.
|
||||
|
||||
`scripts/workflow-stress-report.mjs` reads the normal eve eval artifacts and emits JSON and
|
||||
Markdown. The existing PR Vercel e2e workflow appends the Markdown to the job summary and uploads
|
||||
both reports plus raw eval artifacts on successful stress runs. This establishes a history and
|
||||
makes a regression visible without pretending that one hosted sample is a reliable gate.
|
||||
|
||||
### Paired runtime benchmark
|
||||
|
||||
Add a dedicated `apps/runtime-benchmarks` driver and
|
||||
`.github/workflows/runtime-performance.yml` rather than turning the correctness e2e suite into a
|
||||
statistical harness. For a relevant same-repository PR, the workflow should:
|
||||
|
||||
1. build the merge base and head independently;
|
||||
2. deploy both to immutable preview URLs in the same project and region;
|
||||
3. warm both subjects, then alternate request order `base/head`, `head/base`;
|
||||
4. retain every raw sample and both deployment/run identities;
|
||||
5. publish a paired comparison in the job summary and a sticky PR comment using authenticated
|
||||
`gh`;
|
||||
6. upload JSON and Markdown on every outcome.
|
||||
|
||||
The initial cases are:
|
||||
|
||||
| Case | Purpose |
|
||||
| -------------------------------------------- | --------------------------------------------------------------- |
|
||||
| New session | Split request acceptance, hook readiness, and first model start |
|
||||
| Warm one-step turn | Measure the fixed turn intercept |
|
||||
| 1, 2, 4, 8 deterministic tool cycles | Fit incremental durable-step cost |
|
||||
| Pre-seeded history depths 1, 10, 25, 50, 100 | Fit replay/state-growth slope |
|
||||
| 20–50 independent sessions | Measure p95, batch makespan, and turns/second under load |
|
||||
|
||||
Use five unmeasured warmups, 20 paired samples for one-step cases, and at least 12 paired blocks for
|
||||
step scaling. Never discard outliers. Report p50/p90/p95, median absolute deviation, paired delta
|
||||
and ratio, a deterministic bootstrap 95% confidence interval, fixed-turn intercept,
|
||||
incremental-step slope, and history-depth slope.
|
||||
|
||||
History-depth subjects must be independent pre-seeded sessions, requested in randomized or
|
||||
balanced interleaved order. Advancing one session from turn 1 through 100 remains a useful
|
||||
throughput test, but cannot separate history growth from elapsed test time.
|
||||
|
||||
The artifact schema must include the base/head SHAs, eve and Workflow package versions, world,
|
||||
region, deployment URLs, run ids, request order, history depth, step count, event/stream-write
|
||||
count, serialized input/output bytes, and raw client and server phase durations. Client monotonic
|
||||
timings and server event timestamps stay separate; subtracting clocks from different machines
|
||||
would manufacture precision.
|
||||
|
||||
Run an A/A trial and at least 30 successful main batches before enforcing hosted timing budgets.
|
||||
During calibration, a possible warning floor is a paired regression above both 10–15% and
|
||||
75–100 ms. The eventual gate should fail only when a confirmatory batch agrees and the 95%
|
||||
confidence interval clears both the relative and absolute budgets. Absolute cold-start latency and
|
||||
one noisy p95 sample must not block a PR.
|
||||
|
||||
A nightly main run should repeat the suite three times, retain raw GitHub artifacts, and publish
|
||||
the aggregates to the existing observability backend for trend and SDK-release correlation.
|
||||
Workflow Agent Runs traces are valuable for phase attribution and outliers, but should not be the
|
||||
primary benchmark dependency; CI needs a project-scoped token to inspect those runs.
|
||||
|
||||
### Phase attribution
|
||||
|
||||
Add opt-in benchmark instrumentation for these boundaries:
|
||||
|
||||
- HTTP receipt → hook resume persisted/published;
|
||||
- driver wake/replay → bind → child dispatch;
|
||||
- child created → child started → hook ownership → first `turnStep`;
|
||||
- each `turnStep`: scheduling gap, actual model/tool work, attributes write, protocol writes;
|
||||
- `session.waiting` emitted → step completed → terminal control delivered;
|
||||
- parent resumed → state adopted → caller settled → driver parked;
|
||||
- event-log entry count/bytes and durable session/context bytes at each boundary.
|
||||
|
||||
The deterministic benchmark should disable external telemetry exporters. A separate diagnostic
|
||||
pass can measure real OTLP/provider flush cost so instrumentation overhead is not hidden inside
|
||||
framework overhead.
|
||||
|
||||
### Low-risk await audit
|
||||
|
||||
`ctx.waitUntil` is not one uniform primitive in this stack. Channel routes collect promises and
|
||||
forward a failure-observing aggregate to Nitro's request `waitUntil`, so their response can return
|
||||
first. Schedule handlers expose the same authoring name but await all registered work before the
|
||||
task completes. Workflow bodies and steps expose no public `ctx.waitUntil`; Workflow's executor
|
||||
uses a private host `waitUntil` only for its own tracked stream operations.
|
||||
|
||||
Detaching work from a Workflow step therefore moves it outside Workflow durability. The step can
|
||||
complete and a successor can run first; a retry can duplicate or reorder the work; a crash,
|
||||
timeout, or deployment can lose it; and its failure cannot retry the step. A rejected background
|
||||
promise can also become an unhandled rejection unless it is converted to a fulfilled, logged
|
||||
result. Only work that tolerates all of those outcomes is eligible.
|
||||
|
||||
The hot-path audit produced this disposition:
|
||||
|
||||
| Awaited work | Critical property | Disposition |
|
||||
| ---------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
|
||||
| Per-attempt `$eve.*` attribute write | Best-effort metadata, but cumulative writes must stay ordered and precede terminal run state | Overlap with result handling now; keep the join inside the step |
|
||||
| Instrumentation provider `flush()` | Public idle-drain guarantee; authored providers may buffer state | Measure, then split exporter drain from authored idle drain before backgrounding anything |
|
||||
| Activity projection callback | Best-effort presentation only; already invoked with `void` | Retain lifetime and batch for reliability; there is no await left to remove |
|
||||
| Stable and authorization hook claims | Both are required before the session can serve work, but neither depends on the other | Start together and await both; preserve partial-failure cleanup |
|
||||
| Continuation ownership and stable-hook readiness | Both gate a create result, but can proceed independently after the Workflow starts | Start together and await both; preserve ownership-conflict behavior |
|
||||
| Protocol stream write/close | Client event order and durable response boundary | Keep awaited; Workflow already batches and background-flushes its tracked stream operations |
|
||||
| Adapter delivery, memory, hooks, dynamic extensions, connections | Return values, context mutation, authored failure semantics, or next-step input | Keep awaited |
|
||||
| Hook resume/claim/dispose, caller notification, task acknowledgement, cancellation and child cleanup | Ownership, wake ordering, and terminal settlement | Keep awaited |
|
||||
| Instrumentation event handlers and trace preparation | Provider ordering, durable context state, and trace parenting | Keep awaited |
|
||||
| Channel route background tasks | Post-ack webhook work | Already uses the appropriate request `waitUntil` path |
|
||||
|
||||
The first code change starts `setEveAttributes()` and `handleStepResult()` together, including the
|
||||
terminal `session.waiting` epilogue, then joins the attribute promise before returning from
|
||||
`turnStep`. This can hide the attribute round trip behind work already required to settle the
|
||||
result without allowing older cumulative counters to overwrite newer ones or racing
|
||||
`run_completed`. A focused unit test proves the overlap and join ordering; it does not prove a
|
||||
hosted latency win, which still requires the paired benchmark.
|
||||
|
||||
The next low-risk trials are:
|
||||
|
||||
1. measure attribute-write and instrumentation-flush duration and presence per step;
|
||||
2. parallelize independent stable/authorization hook claims and, on operation-id creates,
|
||||
continuation-ownership/stable-hook readiness checks;
|
||||
3. if the attribute write remains material, coalesce one final cumulative batch into the existing
|
||||
terminal control step and overlap it with the control resume;
|
||||
4. split internal OTel export draining from authored provider idle flushing, then retain only the
|
||||
exporter drain in the host lifetime;
|
||||
5. batch activity projections once per step and retain that callback for reliability.
|
||||
|
||||
A direct host `waitUntil` comparison belongs in a fault-injection experiment, not the low-risk
|
||||
shipping queue. It must measure latency, `$eve.*` tag retention, crash loss, retries, and
|
||||
cross-step ordering, and it must not ship until Workflow exposes a supported step-lifetime API.
|
||||
|
||||
Do not import Workflow's private `waitUntil` helper. Either use a supported public Workflow
|
||||
step-lifetime API when one exists or keep the awaited join. A host-only fallback that silently
|
||||
no-ops outside Vercel would make correctness environment-dependent.
|
||||
|
||||
## Experiments
|
||||
|
||||
Experiments are ordered by information value and expected risk. Each change gets a paired hosted
|
||||
run, the narrowest correctness tests, cancellation/replay tests where relevant, and the existing
|
||||
stress e2e before it can ship.
|
||||
|
||||
### Hosted experiment ledger
|
||||
|
||||
All deltas below use the exact benchmark-base run as the control. The stress model is synchronous,
|
||||
so these measurements isolate eve and Workflow overhead rather than provider latency. Each row is
|
||||
an independently pushed branch unless explicitly labeled as a combination. Raw JSON and Markdown
|
||||
reports are attached to the linked GitHub Actions runs.
|
||||
|
||||
| Experiment | Run | Sequential mean | Sequential p50 | Sequential p95 | Concurrent second p50 | Turn-order slope | Decision |
|
||||
| ---------------------------------------------- | ----------------------------------------------------------------------- | --------------: | -------------: | -------------: | --------------------: | ---------------: | ------------------------------- |
|
||||
| Exact benchmark control | [`33468124247`](https://github.com/vercel/eve/actions/runs/33468124247) | 3.042 s | 3.036 s | 3.692 s | 1.867 s | +13.31 ms/turn | Reference |
|
||||
| Parallel readiness hooks | [`33468104562`](https://github.com/vercel/eve/actions/runs/33468104562) | 3.018 s | 2.897 s | 3.753 s | 1.867 s | +13.80 ms/turn | Neutral on follow-up turns |
|
||||
| Overlap retired control cleanup | [`33470436515`](https://github.com/vercel/eve/actions/runs/33470436515) | 3.084 s | 2.977 s | 3.604 s | 1.929 s | +13.97 ms/turn | Neutral; do not merge |
|
||||
| Workflow SDK beta.47 | [`33468117677`](https://github.com/vercel/eve/actions/runs/33468117677) | 4.460 s | 4.847 s | 6.422 s | 1.863 s | +7.40 ms/turn | Reject; material regression |
|
||||
| Remove root no-op steps | [`33468608676`](https://github.com/vercel/eve/actions/runs/33468608676) | 2.036 s | 2.053 s | 2.367 s | 1.584 s | +5.17 ms/turn | Ship |
|
||||
| Root no-ops + attribute overlap | [`33469404605`](https://github.com/vercel/eve/actions/runs/33469404605) | 2.009 s | 2.008 s | 2.341 s | 1.607 s | +5.27 ms/turn | Current PR candidate |
|
||||
| Inline ordinary root turn | [`33468225787`](https://github.com/vercel/eve/actions/runs/33468225787) | 1.170 s | 1.110 s | 1.515 s | 0.946 s | +0.44 ms/turn | Ceiling only; semantics missing |
|
||||
| Root no-ops + inline root turn, confirmation 1 | [`33469355029`](https://github.com/vercel/eve/actions/runs/33469355029) | 0.814 s | 0.799 s | 1.063 s | 0.743 s | +2.63 ms/turn | Architectural floor |
|
||||
| Root no-ops + inline root turn, confirmation 2 | [`33469627035`](https://github.com/vercel/eve/actions/runs/33469627035) | 0.858 s | 0.828 s | 1.218 s | 0.738 s | +3.35 ms/turn | Architectural floor reproduced |
|
||||
|
||||
The current PR candidate improves the exact control by 34.0% on sequential mean, 33.9% on p50,
|
||||
36.6% on p95, and 13.9% on concurrent second-turn p50. The independently confirmed root no-op
|
||||
change accounts for almost all of that result; attribute overlap is directionally small compared
|
||||
with hosted noise. The two combined-floor runs reproduce a 71.8–73.2% mean reduction and a
|
||||
72.7–73.7% p50 reduction, proving that sub-second warm turns are possible if eve replaces the
|
||||
parent/child topology without losing its semantics.
|
||||
|
||||
### 1. Isolate Workflow SDK and resume cost
|
||||
|
||||
Run the same eve SHA against the current Workflow package set and beta.47, with beta.46 included
|
||||
if its public hook API is compatible. Interleave at least five deployed runs per package set. Add
|
||||
a small Workflow-only benchmark for one persisted `resumeHook` and an eve-shaped two-resume child
|
||||
round trip.
|
||||
|
||||
This decides whether the recent approximately 1.8-second fixed regression belongs in eve, the
|
||||
Workflow SDK/world, or their interaction. If write-before-wake is the cost, work with Workflow on
|
||||
a transactional persist-and-wake primitive; eve must not restore a lossy wake ordering.
|
||||
|
||||
#### Hosted result
|
||||
|
||||
The isolated `barba/perf-exp-workflow-sdk-latest` branch changed only the aligned Workflow
|
||||
packages from the benchmark base to beta.47. Its stress job
|
||||
[`33468117677`](https://github.com/vercel/eve/actions/runs/33468117677) completed and regressed
|
||||
sequential mean from 3.042 to 4.460 seconds (+46.6%), p50 from 3.036 to 4.847 seconds (+59.7%),
|
||||
and p95 from 3.692 to 6.422 seconds (+73.9%). Concurrent second-turn p50 was effectively neutral
|
||||
at 1.863 versus 1.867 seconds, so the package set changed the sequential path rather than applying
|
||||
a uniform hosted-load penalty.
|
||||
|
||||
Do not upgrade eve on this result. The two approximately 4.91-second runs from PR #2611 point in
|
||||
the same direction, but the experiment does not assign causality to one Workflow change because
|
||||
the compatible core, API, Vercel World, and Nitro packages moved together. A Workflow-only
|
||||
`resumeHook`/child-round-trip microbenchmark is still required before attributing the regression
|
||||
to write-before-wake or another persistence change.
|
||||
|
||||
### 2. Remove guaranteed root no-op steps
|
||||
|
||||
Skip `bindTurnCallerContextStep`, `notifyTurnCallerStep`, and the initial caller-resolution step
|
||||
when serialized lineage proves there is no delegated caller. Root sessions pay two no-op steps on
|
||||
every settled turn today, about 90 ms combined even in the local world. Subagent, callback, task,
|
||||
and crash-cleanup behavior remains on the existing path.
|
||||
|
||||
This is the lowest-risk structural change. Prove exact step-count reduction and no change to root,
|
||||
subagent, task, failure, and cancellation results.
|
||||
|
||||
#### Hosted result
|
||||
|
||||
Two Vercel stress runs of the isolated change reproduced a material improvement against a
|
||||
contemporaneous instrumentation-only control. The first experiment SHA, `522e869`, is runtime
|
||||
identical to the final `863daf1` SHA; the amend changed comments only. All three stress jobs
|
||||
completed successfully and uploaded their raw JSON reports:
|
||||
|
||||
- [Control run `33468124247`](https://github.com/vercel/eve/actions/runs/33468124247), SHA
|
||||
`57b477d`, with [artifact `9785707208`](https://github.com/vercel/eve/actions/runs/33468124247/artifacts/9785707208).
|
||||
- [Experiment run `33468070932`](https://github.com/vercel/eve/actions/runs/33468070932), SHA
|
||||
`522e869`, with [artifact `9785657364`](https://github.com/vercel/eve/actions/runs/33468070932/artifacts/9785657364).
|
||||
- [Confirmatory experiment run `33468608676`](https://github.com/vercel/eve/actions/runs/33468608676),
|
||||
final SHA `863daf1`, with
|
||||
[artifact `9785832093`](https://github.com/vercel/eve/actions/runs/33468608676/artifacts/9785832093).
|
||||
|
||||
The control and confirmatory workflows' aggregate conclusions are failures because their separate
|
||||
`fixture-tasks` jobs failed. Their `agent-workflow-stress` jobs and artifact uploads succeeded.
|
||||
|
||||
| Raw artifact metric | Control | Experiment 1, delta | Experiment 2, delta |
|
||||
| --------------------------- | -------------: | ----------------------------: | ---------------------------: |
|
||||
| Sequential mean | 3.042 s | 2.184 s, −0.858 s (−28.2%) | 2.036 s, −1.006 s (−33.1%) |
|
||||
| Sequential p50 | 3.036 s | 2.057 s, −0.979 s (−32.3%) | 2.053 s, −0.983 s (−32.4%) |
|
||||
| Sequential p90 | 3.584 s | 2.455 s, −1.129 s (−31.5%) | 2.288 s, −1.295 s (−36.1%) |
|
||||
| Sequential p95 | 3.692 s | 2.527 s, −1.165 s (−31.6%) | 2.367 s, −1.326 s (−35.9%) |
|
||||
| First 10 warm mean | 2.396 s | 1.806 s, −0.591 s (−24.6%) | 1.722 s, −0.674 s (−28.1%) |
|
||||
| Last 10 mean | 3.607 s | 2.362 s, −1.244 s (−34.5%) | 2.209 s, −1.398 s (−38.8%) |
|
||||
| Sequential turn-order slope | +13.31 ms/turn | −0.08 ms/turn, −13.39 ms/turn | +5.17 ms/turn, −8.14 ms/turn |
|
||||
| Concurrent first-turn p50 | 5.679 s | 4.320 s, −1.359 s (−23.9%) | 4.394 s, −1.285 s (−22.6%) |
|
||||
| Concurrent second-turn p50 | 1.867 s | 1.510 s, −0.357 s (−19.1%) | 1.584 s, −0.283 s (−15.1%) |
|
||||
|
||||
Exact Workflow-world integration coverage confirms the mechanism: two root turns omit one caller
|
||||
resolution, two caller binds, and two caller notifications, while the delegated two-turn path
|
||||
retains that `1 + 2 + 2` step inventory and its results.
|
||||
|
||||
The result is replicated but not yet statistically paired. These workflows ran close together,
|
||||
not as interleaved base/head blocks, and isolated maxima remained noisy: experiment 1 recorded a
|
||||
15.687-second sequential maximum and a 3.025-second concurrent second-turn batch, while experiment
|
||||
2 recorded a 6.373-second concurrent first-turn batch against the control's 6.282 seconds. The
|
||||
p50 and p95 improved in both experiment runs, but the dedicated paired benchmark remains the proof
|
||||
gate for shipping.
|
||||
|
||||
#### Retired control-cleanup result: neutral
|
||||
|
||||
The isolated `barba/perf-exp-terminal-overlap` branch started turn N's deferred control-hook
|
||||
disposal as soon as turn N+1 settled, overlapped it with independent parent settlement, and joined
|
||||
both before the driver parked or returned. It retained cleanup-first error precedence and did not
|
||||
overlap the active child's cancellation-hook disposal with terminal publication. Twenty-six
|
||||
focused Workflow/cancellation integrations passed.
|
||||
|
||||
Hosted run [`33470436515`](https://github.com/vercel/eve/actions/runs/33470436515), with raw report
|
||||
[artifact `9786485510`](https://github.com/vercel/eve/actions/runs/33470436515/artifacts/9786485510),
|
||||
did not show a material improvement against the exact control:
|
||||
|
||||
| Metric | Control | Overlap branch | Delta |
|
||||
| --------------------------- | ------------: | -------------: | ----: |
|
||||
| Sequential mean | 3.042 s | 3.084 s | +1.4% |
|
||||
| Sequential p50 | 3.036 s | 2.977 s | −1.9% |
|
||||
| Sequential p95 | 3.692 s | 3.604 s | −2.4% |
|
||||
| Concurrent second-turn p50 | 1.867 s | 1.929 s | +3.3% |
|
||||
| Sequential turn-order slope | 13.31 ms/turn | 13.97 ms/turn | +4.9% |
|
||||
|
||||
The mixed signs and low-single-digit changes are hosted noise, while concurrent second-turn p95
|
||||
also produced an isolated 7.958-second outlier. Do not add another stateful cleanup abstraction for
|
||||
this result. The branch conflicts in the same hot loop with the inline/root combination, and root
|
||||
caller-step removal leaves even less independent settlement work available to hide cleanup behind.
|
||||
Keep the existing durable ordering and focus on eliminating boundaries rather than rearranging
|
||||
this one.
|
||||
|
||||
### 3. Reduce observability-only work in each step
|
||||
|
||||
`setEveAttributes` is awaited after every model attempt. Inside a Workflow step the SDK writes an
|
||||
attribute event to the world, so best-effort error handling does not make it free. The first
|
||||
low-risk change overlaps that write with result handling while still joining it before step exit.
|
||||
A/B the stress and 1/2/4/8-step cases against the serialized implementation. If the remaining
|
||||
cost is material, compare disabling the write, a host-retained write with measured tag retention,
|
||||
and one cumulative write attached to the terminal control step.
|
||||
|
||||
The stream-write audit does not currently justify an eve runtime change. eve already coalesces
|
||||
adjacent text, reasoning, tool-input, and tool-partial events in its bounded ordered emitter.
|
||||
Workflow `@workflow/core@5.0.0-beta.43` then acknowledges `writer.write()` when a chunk enters its
|
||||
bounded buffer and group-commits buffered chunks with `world.streams.writeMulti`. The Vercel World
|
||||
implementation (`@workflow/world-vercel@5.0.0-beta.39`) preserves each chunk boundary inside that
|
||||
single request. `writer.close()` drains pending writes before closing, and the step executor adopts
|
||||
the same drain barrier when eve releases the writer lock to park.
|
||||
|
||||
A controlled probe against the installed Workflow stream implementation used eight ordered chunks
|
||||
and a mocked World with 20–40 ms write latency:
|
||||
|
||||
| Write pattern | World data writes | Close writes | Consequence |
|
||||
| ----------------------------------- | -------------------------- | -----------: | ---------------------------------- |
|
||||
| Immediate, default flush interval | 1 `write` + 1 `writeMulti` | 1 | No leading delay |
|
||||
| 30 ms apart, default flush interval | 8 `write` | 1 | No adjacent chunks to group-commit |
|
||||
| Immediate, 5 ms flush interval | 1 `writeMulti` | 1 | At least 5 ms leading delay |
|
||||
|
||||
This is protocol-level evidence, not a hosted latency result. It disproves the assumption that
|
||||
eight awaited eve writes necessarily create eight persistence round trips. Making the global
|
||||
flush interval positive could collapse a burst to one request, but would add fixed delay to the
|
||||
first text, tool, and terminal chunk of every idle stream. That violates this experiment's
|
||||
immediate-streaming constraint.
|
||||
|
||||
Packing several eve events into one Workflow chunk is also incorrect with the current protocol.
|
||||
Workflow reconnects by chunk index, while eve clients increment that cursor once per decoded
|
||||
event. A disconnect after the first event in a packed chunk would resume at the next chunk and
|
||||
skip the remainder. Promise concurrency would not reduce backend writes and could move lifecycle
|
||||
handlers ahead of durable event order.
|
||||
|
||||
The narrow useful upstream primitive is a `writeMany(chunks)` or scoped `cork()`/`uncork()` that
|
||||
invokes `writeMulti` while retaining distinct chunk indexes. Before requesting it, hosted traces
|
||||
should count `workflow.stream.flush` operations, chunks per flush, buffer dwell, and chunk RTT to
|
||||
show that lifecycle bursts actually miss the existing in-flight group commit. The current
|
||||
ordering-barrier, sink-failure/drain, reconnect, rewind, and terminal-`session.waiting` tests are
|
||||
the correctness baseline for any later prototype.
|
||||
|
||||
#### Narrow coalescing result: blocked on a Workflow primitive
|
||||
|
||||
A prototype on benchmark base `57b477dc1` carried cumulative metrics out of each successful
|
||||
`turnStep`, retained only the latest totals, and wrote them before `resumeHook` in the existing
|
||||
terminal control step. The happy path can collapse repeated writes, but the current Workflow APIs
|
||||
do not preserve the required failure and retry semantics:
|
||||
|
||||
- Step-body `setAttributes` appends an unguarded, out-of-band `attr_set`. If terminal hook delivery
|
||||
fails after that write and the step retries, it appends another event. Last-write-wins keeps the
|
||||
displayed counters correct, but the promised one-write invariant and its performance cost do
|
||||
not survive retries.
|
||||
- The newest metrics exist after the model returns but before harness post-processing finishes.
|
||||
Deferring them until a successful `StepResult` loses that attempt's counters when a stream,
|
||||
hook, memory, or dynamic-extension callback fails in the same step. The current write happens
|
||||
before this failure boundary.
|
||||
- Workflow-body `setAttributes` is replay-correlated, but it commits through a suspension and an
|
||||
additional replay. It is neither background work nor part of the terminal control step, and
|
||||
makes observability progress part of workflow progress.
|
||||
|
||||
Do not ship this coalescing change on the current SDK. The useful upstream shape is either an
|
||||
idempotent attribute update keyed by a stable operation id, or a terminal primitive that commits
|
||||
attributes and hook delivery atomically. Workflow exposes no step- or run-scoped `waitUntil`
|
||||
today. Such a primitive would help only if Workflow durably joins it before teardown, isolates
|
||||
best-effort failures from the run, and deduplicates its side effect across step retries;
|
||||
fire-and-forget alone is insufficient.
|
||||
|
||||
### 4. Reduce child startup and control handshakes
|
||||
|
||||
Measure child-created → first-step-start in production before choosing a design. Then prototype,
|
||||
in increasing order of risk:
|
||||
|
||||
1. combine or reuse the child inbox and cancellation/control hook with `(turnId, sequence)`
|
||||
deduplication;
|
||||
2. move terminal control delivery into an existing step without publishing state before its
|
||||
durable checkpoint;
|
||||
3. execute ordinary no-wait turns directly from the driver or a persistent executor, retaining
|
||||
the child path only when cancellation/runtime waits require it;
|
||||
4. ask Workflow for a latest-deployment step/child-completion primitive that avoids a second run
|
||||
and polling join.
|
||||
|
||||
The narrow inbox/cancellation-hook pass found no semantics-preserving lazy claim. The inbox claim
|
||||
is the duplicate child-run fence: deferring it until a runtime-action wait would let duplicate
|
||||
starts execute the model, tools, and side effects concurrently. The cancellation claim must be
|
||||
ready before the first `turnStep`, because an otherwise ordinary one-step turn can be steered or
|
||||
cancelled while its model or tool is running.
|
||||
|
||||
One hook can demultiplex tagged cancel and runtime-action payloads within a single deployment, but
|
||||
it cannot replace the two current tokens safely across deployment versions. Session drivers stay
|
||||
pinned while child turns route to latest: old drivers resume `{control}:cancel`, and the shared
|
||||
duplicate-run fence must remain `{control}:inbox` so old and new child retries still contend for
|
||||
the same owner. A per-turn readiness handshake adds another durable control step, while
|
||||
`HookOptions.metadata` makes every inbox resume hydrate the run encryption key; the current
|
||||
Workflow SDK explicitly takes that slower path for any metadata-bearing hook. Either choice can
|
||||
cost more than the claim it removes, especially on subagent/runtime-action turns.
|
||||
|
||||
The required Workflow primitive is one hook entity with atomically registered token aliases (or
|
||||
an equivalent public, no-key-lookup protocol capability). Aliasing both `{control}:inbox` and
|
||||
`{control}:cancel` to one durable iterator would preserve old-driver cancellation, cross-version
|
||||
duplicate fencing, stale-message isolation, and one-claim startup. Until that exists, keep the two
|
||||
hooks; there is no safe hosted A/B whose faster result would represent shippable behavior. The
|
||||
research-only branch `barba/perf-exp-turn-hook-coalesce` records the rejected prototype at
|
||||
`3343fc7`; no runtime changes remain on that branch.
|
||||
|
||||
The third option is only viable if it retains latest-deployment routing. Running all future turns
|
||||
inside the pinned driver would improve latency by silently disabling live upgrades, which is not
|
||||
an acceptable trade.
|
||||
|
||||
#### Child return-value result: failed under sustained turns
|
||||
|
||||
Branch `barba/perf-exp-child-return` replaced the terminal `resumeHook` with a negotiated child
|
||||
workflow return value. New drivers started a durable step that awaited `Run.returnValue` while
|
||||
continuing to service nonterminal control messages; older pinned turn workflows retained the
|
||||
existing terminal-control protocol. The design preserved latest-deployment child dispatch and
|
||||
the turn workflow's cancellation/runtime-wait ownership, so it isolated the completion channel
|
||||
rather than deleting the child boundary.
|
||||
|
||||
The focused two-turn integration path passed, but the installed
|
||||
`@workflow/core@5.0.0-beta.43` implements `Run.returnValue` by polling run state every second from
|
||||
a `"use step"` getter and warns that the wait occupies a queue worker. The hosted stress run
|
||||
[`33469001202`](https://github.com/vercel/eve/actions/runs/33469001202) then failed the sequential
|
||||
case after only two completed turns: turns 1 and 2 took 2.536 and 2.379 seconds, turn 3 never
|
||||
settled, and the eval aborted at 600.002 seconds. The concurrent two-turn case completed in 9.651
|
||||
seconds, but no performance report was emitted because the sequential gate timed out. The raw
|
||||
JUnit evidence is retained in
|
||||
[artifact `9786105820`](https://github.com/vercel/eve/actions/runs/33469001202/artifacts/9786105820).
|
||||
|
||||
Do not ship the polling join. The sustained-turn stall is consistent with the SDK's documented
|
||||
worker-capacity hazard, and replacing one terminal resume with a polling step is not a latency
|
||||
optimization even before that failure. A viable upstream primitive must let a workflow subscribe
|
||||
to child completion without polling or reserving a worker, replay the terminal value/error
|
||||
deterministically, and race safely with hook messages. The beta.47 SDK adds a long-poll path in
|
||||
worlds that support it, but its isolated full-package A/B regressed this fixture materially, so
|
||||
that package update is not evidence that this design is safe or faster.
|
||||
|
||||
### 5. Bound replay and state growth
|
||||
|
||||
Measure event-log entries/bytes and state payloads at every target history depth. Full history is
|
||||
currently embedded in each session snapshot, passed into the child, returned by `turnStep`, sent
|
||||
through terminal control, and retained in the long-lived driver's event log. The hosted
|
||||
turn-order trend and growing local payloads make this a strong hypothesis, not a proven cause.
|
||||
|
||||
Prototype two approaches independently:
|
||||
|
||||
- append-only history with a revision/cursor and periodic bounded snapshots, so a turn transfers
|
||||
only its delta while a step can hydrate the current revision once;
|
||||
- a successor-run chain, Workflow's documented `continueAsNew` analogue, where each bounded run
|
||||
handles one turn and hands state/inbox ownership to a latest-deployment successor.
|
||||
|
||||
The external-state option adds a read/write round trip and wins only if it costs less than repeated
|
||||
snapshot transport/replay. The successor-run option is the most promising structural design
|
||||
because it can replace both the replay-growing driver and per-turn child, but it needs an atomic
|
||||
handoff protocol.
|
||||
|
||||
### 6. Move new-session readiness off the caller path
|
||||
|
||||
The workflow run id exists as soon as `start()` resolves, yet `createSession()` waits until the
|
||||
workflow owns its command hook. Test two compatible improvements:
|
||||
|
||||
- claim the stable and authorization hooks before session hydration/caller resolution, allowing
|
||||
their commit to overlap `createSessionStep`;
|
||||
- return an accepted session id immediately and define explicit `starting` behavior for send,
|
||||
cancel, reset, and stream attachment until hook ownership is ready.
|
||||
|
||||
The second option improves API acceptance latency but does not by itself start the model sooner.
|
||||
The first can improve both. The benchmark reports acceptance, readiness, and first-model timing
|
||||
separately so the result cannot be presented as a turn-speed improvement when it only moves the
|
||||
wait.
|
||||
|
||||
#### Parallel readiness result
|
||||
|
||||
The isolated `barba/perf-exp-parallel-hooks` branch started independent stable/authorization
|
||||
claims together and overlapped continuation-ownership validation with stable-hook readiness while
|
||||
retaining partial-failure cleanup. Focused correctness, type, invariant, and build checks passed.
|
||||
Its hosted stress run
|
||||
[`33468104562`](https://github.com/vercel/eve/actions/runs/33468104562) was neutral on the measured
|
||||
follow-up path: mean changed from 3.042 to 3.018 seconds (−0.8%), p50 to 2.897 seconds (−4.6%),
|
||||
p95 to 3.753 seconds (+1.6%), and concurrent second-turn p50 remained 1.867 seconds. The first
|
||||
cold turn was also unchanged at 2.614 versus 2.615 seconds.
|
||||
|
||||
This does not justify a turn-performance claim. The changed awaits primarily affect session
|
||||
creation/readiness, while the current stress fixture measures follow-up turns after setup. Keep
|
||||
the branch as a candidate for the dedicated acceptance/readiness benchmark, not as part of the
|
||||
turn-latency shipping change.
|
||||
|
||||
### 7. Budget extension and per-step work
|
||||
|
||||
After the fixed topology is addressed, fit the incremental cost of tool cycles and authored
|
||||
extensions. Time adapter delivery, memory lifecycle, stream hooks, dynamic model/connections/
|
||||
subagents/tools/skills/instructions, tool-wrapper construction, and instrumentation flushes.
|
||||
Expose slow-provider diagnostics. Separate internal network exporter drain from the authored
|
||||
provider `flush()` contract, move only the exporter drain to a host-retained boundary, and keep
|
||||
authored flush awaited at actual park/done/error transitions.
|
||||
|
||||
Do not combine side-effectful tool cycles into one replayable step unless each tool invocation
|
||||
keeps an independent durable idempotency checkpoint. Faster retries that repeat external effects
|
||||
are a correctness regression.
|
||||
|
||||
#### Instrumentation flush audit
|
||||
|
||||
The low-risk audit found a real critical-path await, but no safe detach primitive available to
|
||||
eve step code today. `createExecutionNodeStep()` awaits `instrumentation.flush()` in `finally`
|
||||
before `turnStep` can derive its next action. The resulting boundaries are:
|
||||
|
||||
| Path | Awaited drains | Logical boundary |
|
||||
| ---------------------------------------------------------------------- | -------------: | ------------------------------------------------- |
|
||||
| Ordinary harness result, including `action: "continue"` | 1 | Model/tool step; session may continue immediately |
|
||||
| Harness result that becomes `park`, `done`, or runtime-action dispatch | 1 | Idle, terminal, or durable dispatch boundary |
|
||||
| Adapter consumes a delivery without entering the harness | 1 | Idle boundary |
|
||||
| Adapter failure before the harness | 1 | Error boundary |
|
||||
| Harness failure | 2 | Harness `finally`, then delivery-failure cleanup |
|
||||
| Cancellation thrown by the harness | 2 | Harness `finally`, then cancellation epilogue |
|
||||
|
||||
For the provider-directory layout, each drain starts the OpenTelemetry runtime flush and every
|
||||
authored provider `flush()` concurrently, awaits all of them, logs individual failures, and never
|
||||
fails the user step. Awaiting the call also prevents flushes from successive iterations of one
|
||||
session from overlapping. Detaching the entire operation would therefore change
|
||||
authored-provider ordering as well as exporter timing.
|
||||
|
||||
This path does not explain the existing Vercel stress baseline. That fixture uses the legacy
|
||||
single-file `instrumentation.ts` layout. Its eve runtime installs an async no-op `forceFlush`;
|
||||
the optional Datadog `registerOTel()` call made during setup is outside that runtime. The current
|
||||
per-step await in the stress fixture therefore drains no network exporter. A regression probe now
|
||||
holds a fake drain open and proves structurally that a harness step cannot settle until the drain
|
||||
does, without using timing-sensitive assertions.
|
||||
|
||||
There is no public, cross-world lifetime primitive that a transformed Workflow step can use to
|
||||
move the real provider-directory exporter drain off its response path. Nitro's public
|
||||
`event.waitUntil()` exists only at the route boundary, while the step body receives no `H3Event`.
|
||||
Workflow's public exports expose no step-scoped equivalent. Its runtime has a private helper that
|
||||
loads Vercel Functions' request-scoped `waitUntil`, but importing that private module would couple
|
||||
eve to an unsupported implementation detail. Calling Vercel Functions directly is insufficient
|
||||
for eve's portable runtime: outside a Vercel request context it silently registers nothing and
|
||||
does not report whether the promise gained a lifetime owner, so an await fallback cannot be
|
||||
selected reliably.
|
||||
|
||||
The required primitive is a public step-scoped operation such as
|
||||
`waitUntil(promise): "registered" | "unsupported"` that guarantees the current invocation remains
|
||||
alive in hosted, local, and self-hosted worlds, or explicitly reports that eve must await. Once it
|
||||
exists, the narrow experiment should keep authored provider flushes awaited, serialize internal
|
||||
exporter drains, register only the non-rejecting internal exporter promise in the step lifetime,
|
||||
and retain full awaited drains for shutdown. A dedicated provider-directory fixture with a gated
|
||||
exporter must prove both that step settlement no longer includes exporter latency and that the
|
||||
export completes before invocation teardown. The existing stress fixture should show no expected
|
||||
delta from this experiment; it needs a separate exporter diagnostic rather than being presented as
|
||||
proof of the change.
|
||||
|
||||
### Inline-turn ceiling result
|
||||
|
||||
The benchmark-only branch in
|
||||
[PR #2824](https://github.com/vercel/eve/pull/2824) executed an ordinary root turn in the session
|
||||
driver instead of starting a child run. The isolated hosted run
|
||||
[`33468225787`](https://github.com/vercel/eve/actions/runs/33468225787) reduced sequential mean by
|
||||
61.6% (3.042 to 1.170 seconds), p50 by 63.5% (3.036 to 1.110 seconds), p95 by 59.0% (3.692 to
|
||||
1.515 seconds), and concurrent second-turn p50 by 49.3% (1.867 to 0.946 seconds). The turn-order
|
||||
slope fell from 13.31 to 0.44 ms/turn.
|
||||
|
||||
Combining that prototype with root no-op removal crossed the product target twice. Runs
|
||||
[`33469355029`](https://github.com/vercel/eve/actions/runs/33469355029) and
|
||||
[`33469627035`](https://github.com/vercel/eve/actions/runs/33469627035) recorded sequential p50s
|
||||
of 0.799 and 0.828 seconds and p95s of 1.063 and 1.218 seconds. Both stress jobs passed. This is
|
||||
the strongest evidence in the investigation: the existing platform can sustain sub-second warm
|
||||
turns when eve removes the parent/child round trip.
|
||||
|
||||
The prototype is deliberately non-mergeable. It pins future ordinary turns to the driver's
|
||||
deployment and lacks the child's mid-turn cancellation, runtime wait, sleep, and background-work
|
||||
ownership. Those are product semantics, not optional overhead. The measurements establish an
|
||||
architectural ceiling and justify a successor/latest-deployment primitive; they do not justify
|
||||
shipping the inline fast path.
|
||||
|
||||
## Structural direction
|
||||
|
||||
The preferred long-term prototype is a chain of bounded, latest-deployment turn runs behind a
|
||||
stable eve session identity:
|
||||
|
||||
```text
|
||||
stable session address
|
||||
└─ current owner run N
|
||||
├─ accept exactly one sequenced delivery
|
||||
├─ execute and stream the turn
|
||||
├─ checkpoint state revision N
|
||||
├─ start latest-deployment owner run N+1
|
||||
└─ atomically hand off the stable inbox, then exit
|
||||
```
|
||||
|
||||
This would make replay bounded and remove the parent-driver/child-control round trip. It is an
|
||||
experiment, not a settled implementation. The handoff must preserve all of these invariants:
|
||||
|
||||
- one active turn per session and FIFO delivery under concurrent sends;
|
||||
- stable public session id and continuation aliases independent of Workflow run ids;
|
||||
- no delivery loss or duplicate model/tool execution across crash/retry/handoff;
|
||||
- latest-deployment routing and versioned state migration;
|
||||
- ordered, resumable event streaming across successor runs;
|
||||
- turn cancellation, session cancel/reset/timeout, authorization and input waits;
|
||||
- subagent/task caller settlement and descendant routing;
|
||||
- safe at-least-once terminal notifications and stale-message rejection.
|
||||
|
||||
If Workflow cannot provide atomic hook ownership transfer, an eve-owned sequenced inbox/CAS may be
|
||||
necessary. A direct ingress fast path that starts a turn before driver replay is another possible
|
||||
prototype, but it has the same serialization and fencing problem and should follow, not precede,
|
||||
the successor-run experiment.
|
||||
|
||||
### Successor-run feasibility result
|
||||
|
||||
The executable prototype on `barba/perf-exp-successor-turns` narrows the design space, but it is
|
||||
not a production candidate. The Workflow APIs provide two required pieces today:
|
||||
|
||||
- explicit recursive `start(..., { deploymentId: "latest" })` creates a bounded successor on the
|
||||
current deployment; and
|
||||
- a `WritableStream` can be passed across runs, so successor output can remain on the original
|
||||
session stream.
|
||||
|
||||
They do not provide atomic ownership transfer for the stable command hook. Hook tokens have one
|
||||
active owner. Disposing the old hook and claiming it in a successor are separate durable writes.
|
||||
The integration proof deliberately gates the successor between those writes and observes
|
||||
`HookNotFoundError` from both `getHookByToken()` and `resumeHook()`. A direct run-to-run handoff
|
||||
would therefore reject a command in that interval and cannot replace the current driver.
|
||||
|
||||
The same test contains the narrowest lossless workaround supported by the current API:
|
||||
|
||||
```text
|
||||
stable public token
|
||||
└─ minimal FIFO sequencer run
|
||||
├─ active executor N receives commands
|
||||
├─ executor N asks the sequencer to seal generation N
|
||||
├─ sequencer forwards a seal, then durably holds later commands
|
||||
├─ executor N drains every pre-seal command
|
||||
├─ executor N starts latest-deployment executor N+1 with state + shared stream
|
||||
└─ sequencer activates N+1 and forwards held commands in FIFO order
|
||||
```
|
||||
|
||||
Six burst deliveries produced six ordered stream records through six owning executor runs. Every
|
||||
nonterminal executor persisted exactly five steps regardless of history depth; the terminal
|
||||
executor persisted three. The deterministic executor token also fenced a losing duplicate start.
|
||||
This proves that one-turn executor histories and stream continuity are possible with current
|
||||
Workflow primitives.
|
||||
|
||||
It also shows why the workaround should not ship as the performance fix:
|
||||
|
||||
- the sequencer is still a long-lived relay whose hook/event log grows with every command;
|
||||
- every ingress crosses a sequencer replay plus a forwarding step before the executor wakes;
|
||||
- seal, successor start, readiness, and activation add five durable steps per nonterminal turn;
|
||||
- the full conversation snapshot still grows once per successor input even though it is no longer
|
||||
copied repeatedly into one driver's event log; and
|
||||
- the prototype intentionally omits production cancellation, timeout, authorization, HITL,
|
||||
subagent/task, terminal-caller, and failure-recovery paths.
|
||||
|
||||
Consequently there is no hosted stress result for this branch: wiring the prototype into the eve
|
||||
runtime would knowingly add overhead and leave required semantics incomplete. The focused test is
|
||||
the proof artifact, not a benchmark substitute. Commit `a4c7ae2` retains that artifact; two
|
||||
focused integration cases, full typecheck, lint, and invariant checks pass.
|
||||
|
||||
The enabling Workflow primitive is an atomic, replay-idempotent successor handoff. Given a stable
|
||||
token, expected owner/generation, handoff id, latest-deployment workflow reference, checkpoint,
|
||||
and existing stream, one commit must:
|
||||
|
||||
1. fence the old owner at an exact inbox sequence;
|
||||
2. start and register the successor on the latest deployment;
|
||||
3. preserve or transfer every payload before the fence and route every later payload to the
|
||||
successor;
|
||||
4. keep `resumeHook(stableToken, payload)` continuously addressable, never transiently not found;
|
||||
5. return the same successor on replay of the handoff id; and
|
||||
6. preserve the stable eve session id and original resumable stream.
|
||||
|
||||
An eve-owned durable inbox with monotonic sequence numbers and compare-and-swap ownership could
|
||||
provide equivalent semantics, but then eve owns a new storage protocol, retry/fencing rules, and a
|
||||
stream directory. Prototype that only if Workflow cannot expose the atomic operation. Even with
|
||||
either handoff, append-only history revisions or delta snapshots remain a separate requirement to
|
||||
remove the growing successor-input payload.
|
||||
|
||||
## Decisions and next work
|
||||
|
||||
Ship the measurement/reporting increment, root no-op removal, and joined attribute overlap in the
|
||||
current PR. The root change is independently reproduced, clears the 30% target, reduces exact
|
||||
durable step count, and preserves the delegated path. Keep reporting informational until paired
|
||||
base/head calibration is complete.
|
||||
|
||||
Do not ship the SDK beta.47 update, polling child return, terminal attribute coalescing, stream
|
||||
packing, hook coalescing, retired-cleanup overlap, inline root turn, or sequencer successor. The
|
||||
first six are negative, unsafe, or neutral on measured evidence; the final two prove the latency
|
||||
ceiling but omit required semantics.
|
||||
|
||||
Proceed in this order:
|
||||
|
||||
1. land the dedicated paired benchmark and phase counters so future PRs compare immutable base and
|
||||
head deployments rather than unrelated hosted samples;
|
||||
2. take the Workflow team a minimal resume/child round-trip reproducer plus the beta.43/beta.47
|
||||
artifact pair;
|
||||
3. specify the atomic successor/hook-ownership transfer primitive, since removing the parent/child
|
||||
topology has the only reproduced path to sub-second p50 without relying on model changes;
|
||||
4. prototype append-only history revisions or delta snapshots behind that bounded executor;
|
||||
5. request non-polling child completion, hook token aliases, idempotent attribute writes,
|
||||
cursor-preserving `writeMany`, and a public step-lifetime `waitUntil` as independent primitives;
|
||||
6. retain the current child topology until the successor prototype passes FIFO, retry, crash,
|
||||
cancellation, authorization, HITL, task/subagent, latest-deployment, and stream-resume suites.
|
||||
|
||||
## Proof of success
|
||||
|
||||
An optimization is proven only by a paired base/head hosted benchmark with raw artifacts. Its
|
||||
bootstrap 95% confidence interval must show an improvement, its p95 must not regress materially,
|
||||
and all durable correctness suites must pass. For the first structural release, target:
|
||||
|
||||
- at least 30% and 750 ms lower warm one-step p50 on the deterministic hosted benchmark;
|
||||
- no more than a 10% p95 regression in any benchmark case;
|
||||
- at least 50% lower fixed-turn intercept in the step-scaling fit;
|
||||
- last-ten versus first-ten warm (turns 2–11) latency growth below both 10% and 200 ms at depth
|
||||
100;
|
||||
- no increase in exact durable step/hook/stream-write counts unless phase data proves a net win;
|
||||
- unchanged event order, model/tool outputs, replay, cancellation, latest-deployment, and
|
||||
concurrent-delivery semantics.
|
||||
|
||||
The product north star is sub-second framework-controlled time from a warm accepted delivery to
|
||||
model-step start at p50, with p95 below 1.5 seconds. Phase data may show a platform floor that eve
|
||||
cannot remove alone; in that case the report must isolate that floor and the plan moves the
|
||||
corresponding primitive into the Workflow workstream rather than weakening the measurement.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Counting a faster model/provider as an eve performance improvement.
|
||||
- Masking pre-model delay with UI animation or synthetic streaming.
|
||||
- Optimizing generic map construction, metadata parsing, or compression before phase data makes
|
||||
it material.
|
||||
- Trading away durability, retry safety, event ordering, cancellation, or live-deployment routing.
|
||||
- Adding legacy fallback paths for superseded performance architectures.
|
||||
@@ -0,0 +1,333 @@
|
||||
import { readdir, readFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, relative, resolve, sep } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const PERFORMANCE_LOG_PREFIX = "EVE_WORKFLOW_STRESS_METRIC=";
|
||||
const REQUIRED_SCENARIOS = ["concurrent", "sequential"];
|
||||
|
||||
export async function collectWorkflowStressMetrics(artifactsRoot) {
|
||||
const artifactPaths = (await findJsonFiles(artifactsRoot)).sort();
|
||||
const metricsByRun = new Map();
|
||||
|
||||
for (const artifactPath of artifactPaths) {
|
||||
const artifact = JSON.parse(await readFile(artifactPath, "utf8"));
|
||||
const logs = artifact?.result?.logs;
|
||||
|
||||
if (!Array.isArray(logs)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const log of logs) {
|
||||
if (typeof log !== "string") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const markerIndex = log.indexOf(PERFORMANCE_LOG_PREFIX);
|
||||
|
||||
if (markerIndex === -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const metric = JSON.parse(log.slice(markerIndex + PERFORMANCE_LOG_PREFIX.length));
|
||||
validateMetric(metric, artifactPath);
|
||||
const runDirectory = findEvalRunDirectory(artifactsRoot, artifactPath);
|
||||
const metricsByScenario = metricsByRun.get(runDirectory) ?? new Map();
|
||||
metricsByScenario.set(metric.scenario, { artifactPath, metric });
|
||||
metricsByRun.set(runDirectory, metricsByScenario);
|
||||
}
|
||||
}
|
||||
|
||||
const latestRun = [...metricsByRun.entries()]
|
||||
.sort(([left], [right]) => left.localeCompare(right, "en"))
|
||||
.at(-1);
|
||||
|
||||
if (latestRun === undefined) {
|
||||
throw new Error(`Missing Workflow stress metrics under ${artifactsRoot}`);
|
||||
}
|
||||
|
||||
const [runDirectory, metricsByScenario] = latestRun;
|
||||
|
||||
for (const scenario of REQUIRED_SCENARIOS) {
|
||||
if (!metricsByScenario.has(scenario)) {
|
||||
throw new Error(
|
||||
`Latest Workflow stress metric run ${runDirectory} is missing the ${scenario} scenario`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
runDirectory,
|
||||
...Object.fromEntries(
|
||||
[...metricsByScenario.entries()].map(([scenario, entry]) => [scenario, entry]),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkflowStressReport(metrics, metadata = {}) {
|
||||
const sequentialSamples = metrics.sequential.metric.samples;
|
||||
const firstConcurrentBatch = metrics.concurrent.metric.batches.find(
|
||||
(batch) => batch.turnNumber === 1,
|
||||
);
|
||||
const secondConcurrentBatch = metrics.concurrent.metric.batches.find(
|
||||
(batch) => batch.turnNumber === 2,
|
||||
);
|
||||
|
||||
if (firstConcurrentBatch === undefined || secondConcurrentBatch === undefined) {
|
||||
throw new Error("Concurrent Workflow stress metric must include turn 1 and turn 2 batches");
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
metadata,
|
||||
schemaVersion: 1,
|
||||
scenarios: {
|
||||
concurrent: {
|
||||
firstTurns: summarizeSamples(firstConcurrentBatch.samples),
|
||||
firstTurnsBatchDurationMs: firstConcurrentBatch.batchDurationMs,
|
||||
secondTurns: summarizeSamples(secondConcurrentBatch.samples),
|
||||
secondTurnsBatchDurationMs: secondConcurrentBatch.batchDurationMs,
|
||||
},
|
||||
sequential: {
|
||||
allTurns: summarizeSamples(sequentialSamples),
|
||||
coldTurn: summarizeSamples(sequentialSamples.slice(0, 1)),
|
||||
firstTenWarmTurns: summarizeSamples(sequentialSamples.slice(1, 11)),
|
||||
sequentialTurnOrderSlopeMsPerTurn: calculateLinearSlope(
|
||||
sequentialSamples.slice(1).map((sample) => [sample.turnNumber, sample.durationMs]),
|
||||
),
|
||||
lastTenTurns: summarizeSamples(sequentialSamples.slice(-10)),
|
||||
warmTurns: summarizeSamples(sequentialSamples.slice(1)),
|
||||
},
|
||||
},
|
||||
sources: {
|
||||
concurrent: metrics.concurrent.artifactPath,
|
||||
runDirectory: metrics.runDirectory,
|
||||
sequential: metrics.sequential.artifactPath,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function renderWorkflowStressMarkdown(report) {
|
||||
const sequential = report.scenarios.sequential;
|
||||
const concurrent = report.scenarios.concurrent;
|
||||
const rows = [
|
||||
["Sequential, all turns", sequential.allTurns],
|
||||
["Sequential, first/cold turn", sequential.coldTurn],
|
||||
["Sequential, warm turns 2–100", sequential.warmTurns],
|
||||
["Sequential, first 10 warm turns (2–11)", sequential.firstTenWarmTurns],
|
||||
["Sequential, turns 91–100", sequential.lastTenTurns],
|
||||
["Concurrent, first turn", concurrent.firstTurns],
|
||||
["Concurrent, second turn", concurrent.secondTurns],
|
||||
];
|
||||
const lines = [
|
||||
"## Workflow stress performance",
|
||||
"",
|
||||
"> Informational hosted measurement. Compare paired base/head runs before attributing a change; this report is not a performance gate.",
|
||||
"",
|
||||
"| Scenario | Samples | Mean | p50 | p90 | p95 | Min | Max |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
...rows.map(([label, summary]) =>
|
||||
[
|
||||
`| ${label}`,
|
||||
summary.count,
|
||||
formatDuration(summary.meanMs),
|
||||
formatDuration(summary.p50Ms),
|
||||
formatDuration(summary.p90Ms),
|
||||
formatDuration(summary.p95Ms),
|
||||
formatDuration(summary.minMs),
|
||||
`${formatDuration(summary.maxMs)} |`,
|
||||
].join(" | "),
|
||||
),
|
||||
"",
|
||||
`Sequential warm turn-order slope: **${sequential.sequentialTurnOrderSlopeMsPerTurn.toFixed(2)} ms/turn** (history-correlated, but also time-order confounded).`,
|
||||
`Concurrent batch wall time: **${formatDuration(concurrent.firstTurnsBatchDurationMs)}** first turns, **${formatDuration(concurrent.secondTurnsBatchDurationMs)}** second turns.`,
|
||||
];
|
||||
|
||||
const metadata = Object.entries(report.metadata).filter(([, value]) => value !== undefined);
|
||||
|
||||
if (metadata.length > 0) {
|
||||
lines.push(
|
||||
"",
|
||||
`Run: ${metadata.map(([key, value]) => `${key}=\`${String(value)}\``).join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function summarizeSamples(samples) {
|
||||
const values = samples.map((sample) => sample.durationMs).sort((left, right) => left - right);
|
||||
|
||||
if (values.length === 0) {
|
||||
throw new Error("Cannot summarize an empty Workflow stress sample set");
|
||||
}
|
||||
|
||||
return {
|
||||
count: values.length,
|
||||
maxMs: values.at(-1),
|
||||
meanMs: values.reduce((total, value) => total + value, 0) / values.length,
|
||||
minMs: values[0],
|
||||
p50Ms: percentile(values, 0.5),
|
||||
p90Ms: percentile(values, 0.9),
|
||||
p95Ms: percentile(values, 0.95),
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(sortedValues, probability) {
|
||||
const position = (sortedValues.length - 1) * probability;
|
||||
const lowerIndex = Math.floor(position);
|
||||
const upperIndex = Math.ceil(position);
|
||||
const lowerValue = sortedValues[lowerIndex];
|
||||
const upperValue = sortedValues[upperIndex];
|
||||
|
||||
return lowerValue + (upperValue - lowerValue) * (position - lowerIndex);
|
||||
}
|
||||
|
||||
function calculateLinearSlope(points) {
|
||||
const xMean = points.reduce((total, [x]) => total + x, 0) / points.length;
|
||||
const yMean = points.reduce((total, [, y]) => total + y, 0) / points.length;
|
||||
let numerator = 0;
|
||||
let denominator = 0;
|
||||
|
||||
for (const [x, y] of points) {
|
||||
numerator += (x - xMean) * (y - yMean);
|
||||
denominator += (x - xMean) ** 2;
|
||||
}
|
||||
|
||||
return denominator === 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
|
||||
function formatDuration(milliseconds) {
|
||||
return `${(milliseconds / 1_000).toFixed(3)}s`;
|
||||
}
|
||||
|
||||
function validateMetric(metric, artifactPath) {
|
||||
if (
|
||||
metric?.schemaVersion !== 1 ||
|
||||
metric.fixture !== "agent-workflow-stress" ||
|
||||
!REQUIRED_SCENARIOS.includes(metric.scenario) ||
|
||||
metric.unit !== "milliseconds"
|
||||
) {
|
||||
throw new Error(`Invalid Workflow stress metric in ${artifactPath}`);
|
||||
}
|
||||
|
||||
if (metric.scenario === "sequential") {
|
||||
validateSamples(metric.samples, artifactPath, "turnNumber");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Array.isArray(metric.batches) || metric.batches.length === 0) {
|
||||
throw new Error(`Invalid concurrent Workflow stress batches in ${artifactPath}`);
|
||||
}
|
||||
|
||||
for (const batch of metric.batches) {
|
||||
if (!Number.isInteger(batch?.turnNumber) || !isFiniteNonnegative(batch?.batchDurationMs)) {
|
||||
throw new Error(`Invalid concurrent Workflow stress batch in ${artifactPath}`);
|
||||
}
|
||||
|
||||
validateSamples(batch.samples, artifactPath, "sessionNumber");
|
||||
}
|
||||
}
|
||||
|
||||
function validateSamples(samples, artifactPath, ordinalKey) {
|
||||
if (
|
||||
!Array.isArray(samples) ||
|
||||
samples.length === 0 ||
|
||||
samples.some(
|
||||
(sample) =>
|
||||
!isFiniteNonnegative(sample?.durationMs) || !Number.isInteger(sample?.[ordinalKey]),
|
||||
)
|
||||
) {
|
||||
throw new Error(`Invalid Workflow stress samples in ${artifactPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isFiniteNonnegative(value) {
|
||||
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
||||
}
|
||||
|
||||
async function findJsonFiles(root) {
|
||||
const entries = await readdir(root, { withFileTypes: true });
|
||||
const paths = await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
const path = resolve(root, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
return findJsonFiles(path);
|
||||
}
|
||||
|
||||
return entry.isFile() && entry.name.endsWith(".json") ? [path] : [];
|
||||
}),
|
||||
);
|
||||
|
||||
return paths.flat();
|
||||
}
|
||||
|
||||
function findEvalRunDirectory(artifactsRoot, artifactPath) {
|
||||
const root = resolve(artifactsRoot);
|
||||
const segments = relative(root, artifactPath).split(sep);
|
||||
const evalsIndex = segments.indexOf("evals");
|
||||
|
||||
if (evalsIndex === -1) {
|
||||
throw new Error(`Workflow stress metric is outside an eval run directory: ${artifactPath}`);
|
||||
}
|
||||
|
||||
return resolve(root, ...segments.slice(0, evalsIndex));
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const options = {};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const name = argv[index];
|
||||
const value = argv[index + 1];
|
||||
|
||||
if (!["--artifacts", "--json", "--markdown"].includes(name) || value === undefined) {
|
||||
throw new Error(
|
||||
"Usage: workflow-stress-report.mjs --artifacts <dir> [--json <path>] [--markdown <path>]",
|
||||
);
|
||||
}
|
||||
|
||||
options[name.slice(2)] = value;
|
||||
index += 1;
|
||||
}
|
||||
|
||||
if (options.artifacts === undefined) {
|
||||
throw new Error("--artifacts is required");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
async function writeOutput(path, value) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, value);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArguments(process.argv.slice(2));
|
||||
const metrics = await collectWorkflowStressMetrics(options.artifacts);
|
||||
const report = createWorkflowStressReport(metrics, {
|
||||
attempt: process.env.GITHUB_RUN_ATTEMPT,
|
||||
model: process.env.EVE_E2E_MODEL,
|
||||
runId: process.env.GITHUB_RUN_ID,
|
||||
sha: process.env.GITHUB_SHA,
|
||||
});
|
||||
const markdown = renderWorkflowStressMarkdown(report);
|
||||
|
||||
if (options.json !== undefined) {
|
||||
await writeOutput(options.json, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
if (options.markdown !== undefined) {
|
||||
await writeOutput(options.markdown, markdown);
|
||||
}
|
||||
|
||||
process.stdout.write(markdown);
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
import.meta.url === pathToFileURL(resolve(process.argv[1])).href
|
||||
) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
collectWorkflowStressMetrics,
|
||||
createWorkflowStressReport,
|
||||
renderWorkflowStressMarkdown,
|
||||
} from "./workflow-stress-report.mjs";
|
||||
|
||||
const PREFIX = "EVE_WORKFLOW_STRESS_METRIC=";
|
||||
|
||||
test("builds a report from eval artifact metrics", async (t) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "eve-workflow-stress-report-"));
|
||||
t.after(() => rm(root, { force: true, recursive: true }));
|
||||
const artifactDirectory = join(root, "2026-08-31", "evals");
|
||||
await mkdir(artifactDirectory, { recursive: true });
|
||||
|
||||
await writeArtifact(artifactDirectory, "sequential.json", {
|
||||
fixture: "agent-workflow-stress",
|
||||
samples: Array.from({ length: 100 }, (_, index) => ({
|
||||
durationMs: 1_000 + index * 10,
|
||||
turnNumber: index + 1,
|
||||
})),
|
||||
scenario: "sequential",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
});
|
||||
await writeArtifact(artifactDirectory, "concurrent.json", {
|
||||
batches: [
|
||||
{
|
||||
batchDurationMs: 1_200,
|
||||
samples: [
|
||||
{ durationMs: 1_000, sessionNumber: 1 },
|
||||
{ durationMs: 1_200, sessionNumber: 2 },
|
||||
],
|
||||
turnNumber: 1,
|
||||
},
|
||||
{
|
||||
batchDurationMs: 900,
|
||||
samples: [
|
||||
{ durationMs: 800, sessionNumber: 1 },
|
||||
{ durationMs: 900, sessionNumber: 2 },
|
||||
],
|
||||
turnNumber: 2,
|
||||
},
|
||||
],
|
||||
fixture: "agent-workflow-stress",
|
||||
scenario: "concurrent",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
});
|
||||
|
||||
const metrics = await collectWorkflowStressMetrics(root);
|
||||
const report = createWorkflowStressReport(metrics, { sha: "abc123" });
|
||||
const markdown = renderWorkflowStressMarkdown(report);
|
||||
|
||||
assert.equal(report.scenarios.sequential.allTurns.count, 100);
|
||||
assert.equal(report.scenarios.sequential.allTurns.meanMs, 1_495);
|
||||
assert.equal(report.scenarios.sequential.sequentialTurnOrderSlopeMsPerTurn, 10);
|
||||
assert.equal(report.scenarios.concurrent.firstTurns.p50Ms, 1_100);
|
||||
assert.match(markdown, /Sequential warm turn-order slope: \*\*10\.00 ms\/turn\*\*/);
|
||||
assert.match(markdown, /sha=`abc123`/);
|
||||
});
|
||||
|
||||
test("rejects artifacts without both stress scenarios", async (t) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "eve-workflow-stress-report-"));
|
||||
t.after(() => rm(root, { force: true, recursive: true }));
|
||||
await writeArtifact(join(root, "2026-08-31", "evals"), "sequential.json", {
|
||||
fixture: "agent-workflow-stress",
|
||||
samples: [{ durationMs: 1_000, turnNumber: 1 }],
|
||||
scenario: "sequential",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
});
|
||||
|
||||
await assert.rejects(collectWorkflowStressMetrics(root), /missing the concurrent scenario/);
|
||||
});
|
||||
|
||||
test("does not combine scenarios from different eval runs", async (t) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "eve-workflow-stress-report-"));
|
||||
t.after(() => rm(root, { force: true, recursive: true }));
|
||||
const olderRun = join(root, "2026-08-30", "evals");
|
||||
const latestRun = join(root, "2026-08-31", "evals");
|
||||
|
||||
await writeArtifact(olderRun, "sequential.json", sequentialMetric());
|
||||
await writeArtifact(olderRun, "concurrent.json", concurrentMetric());
|
||||
await writeArtifact(latestRun, "sequential.json", sequentialMetric());
|
||||
|
||||
await assert.rejects(
|
||||
collectWorkflowStressMetrics(root),
|
||||
/2026-08-31.*missing the concurrent scenario/,
|
||||
);
|
||||
});
|
||||
|
||||
async function writeArtifact(directory, name, metric) {
|
||||
await mkdir(directory, { recursive: true });
|
||||
await writeFile(
|
||||
join(directory, name),
|
||||
JSON.stringify({ result: { logs: [`${PREFIX}${JSON.stringify(metric)}`] } }),
|
||||
);
|
||||
}
|
||||
|
||||
function sequentialMetric() {
|
||||
return {
|
||||
fixture: "agent-workflow-stress",
|
||||
samples: [{ durationMs: 1_000, turnNumber: 1 }],
|
||||
scenario: "sequential",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
};
|
||||
}
|
||||
|
||||
function concurrentMetric() {
|
||||
return {
|
||||
batches: [
|
||||
{
|
||||
batchDurationMs: 1_000,
|
||||
samples: [{ durationMs: 1_000, sessionNumber: 1 }],
|
||||
turnNumber: 1,
|
||||
},
|
||||
{
|
||||
batchDurationMs: 900,
|
||||
samples: [{ durationMs: 900, sessionNumber: 1 }],
|
||||
turnNumber: 2,
|
||||
},
|
||||
],
|
||||
fixture: "agent-workflow-stress",
|
||||
scenario: "concurrent",
|
||||
schemaVersion: 1,
|
||||
unit: "milliseconds",
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user