mirror of
https://github.com/vercel/workflow.git
synced 2026-09-14 19:59:43 +08:00
[core] Pin correlation-id draw order to event-log order (#3700)
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
---
|
||||
'workflow': minor
|
||||
'@workflow/core': minor
|
||||
---
|
||||
|
||||
Pin correlation-ID draw order to event-log order (Node.js VM engine), so two concurrent replays of the same run assign the same IDs even when one loaded a shorter event-log prefix. Set `WORKFLOW_LOG_ORDER_DRAWS=0` to opt back into arrival-order delivery resolution.
|
||||
@@ -49,3 +49,7 @@ event-log-race-repro-results.json
|
||||
event-log-race-repro-summary.md
|
||||
event-log-race-repro-previous-comment.md
|
||||
event-log-race-repro-server.log
|
||||
|
||||
# Per-run e2e diagnostics sidecars written to the repo root by the harness
|
||||
# (writeDiagnosticsSidecar in packages/core/e2e/utils.ts)
|
||||
e2e-diagnostics-*.json
|
||||
|
||||
@@ -140,6 +140,16 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL
|
||||
- Delay before a re-invocation caused by a rejected event creation.
|
||||
- Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up — so the delay gives the other writers a moment to quiesce.
|
||||
|
||||
### `WORKFLOW_LOG_ORDER_DRAWS`
|
||||
|
||||
- Default: enabled
|
||||
- Experimental. Pins correlation-ID draw order to event-log order: a branch-deciding delivery (a step result, hook payload, or wait completion) resolves to the workflow only after every earlier-in-log delivery's continuation has fully quiesced, and never ahead of a lower-slot delivery that is committed to happening.
|
||||
- Without it, a delivery that resolves while an earlier delivery's continuation is still a few microtask hops from its next step/hook/wait call can overtake it on the run's shared correlation-ID sequence. Draw order — and therefore correlation IDs — then depends on how much of the event log a replay had loaded, and two concurrent replays holding different-length prefixes can bind one ID to two different entities, failing the run with `CORRUPTED_EVENT_LOG`.
|
||||
- Costs one event-loop turn (roughly 15-20 microseconds via `setImmediate`) per branch-deciding delivery during replay, more when continuations genuinely overlap. Measured on a 100-step sequential replay: about 2ms added end to end.
|
||||
- Only applies to the default Node.js VM engine. `WORKFLOW_VM=quickjs` has its own event feed and correlation-ID sequence and is unaffected by this setting.
|
||||
- Correlation IDs of runs created before the setting changed are not affected on platforms where a run keeps replaying on the deployment it started on. Elsewhere, only change it while no runs are in flight.
|
||||
- Set `0` to opt back into arrival-order delivery resolution. Only the literal value `0` opts out; `false` or `off` leave it enabled.
|
||||
|
||||
## Inline execution
|
||||
|
||||
### `WORKFLOW_V2_TIMEOUT_MS`
|
||||
|
||||
@@ -80,6 +80,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
});
|
||||
const ulid = monotonicFactory(() => context.globalThis.Math.random());
|
||||
const workflowStartedAt = context.globalThis.Date.now();
|
||||
// Real-session parity: the log-order-draws quiescence fixpoint keys its
|
||||
// progress metric on `mintCount`; without it the loop degrades to a single
|
||||
// turn and this suite would only exercise a degraded variant.
|
||||
let mintCount = 0;
|
||||
const promiseQueueHolder = { current: Promise.resolve() };
|
||||
const ctxRef: { current?: WorkflowOrchestratorContext } = {};
|
||||
const ctx: WorkflowOrchestratorContext = {
|
||||
@@ -99,7 +103,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
getPromiseQueue: () => promiseQueueHolder.current,
|
||||
}),
|
||||
invocationsQueue: new Map(),
|
||||
generateUlid: () => ulid(workflowStartedAt),
|
||||
generateUlid: () => {
|
||||
mintCount += 1;
|
||||
return ulid(workflowStartedAt);
|
||||
},
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * context.globalThis.Math.random())
|
||||
),
|
||||
@@ -831,3 +841,88 @@ describe('suspension timing against parked step deliveries', () => {
|
||||
expectSuspensionSnapshotSteps(error, ['followUp']);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── step result above a wait parked behind an unclaimed payload ────────────
|
||||
//
|
||||
// The log-order-draws turnstile (`quiesceEarlierCascades`) refuses to resolve
|
||||
// a delivery while a LOWER-index ARMED barrier is still registered. An armed
|
||||
// wait can itself be parked behind an unclaimed buffered hook payload — a
|
||||
// chain only the idle-gated safety net can move (lowest-first retirement).
|
||||
// This test pins the termination argument for that shape: the spinning step
|
||||
// delivery must not count as a parked committed delivery (`resolvesOnItsOwn`
|
||||
// excludes it — it gates on the parked wait), so `canRetireAbandonedBarriers`
|
||||
// stays reachable, the net retires the payload, the wait delivers, and the
|
||||
// turnstile opens. A regression that makes the turnstile wait on parked
|
||||
// chains directly, or counts the spinner as self-resolving, deadlocks this
|
||||
// replay instead of suspending it.
|
||||
describe('log-order draws turnstile above a parked chain', () => {
|
||||
const scenario = async () => {
|
||||
const resumeAt = new Date(FIXED_TIMESTAMP + 5_000);
|
||||
const ops: Promise<unknown>[] = [];
|
||||
const [payload, stepAResult] = await Promise.all([
|
||||
dehydrateStepReturnValue({ poke: 1 }, 'wrun_test', undefined, ops),
|
||||
dehydrateStepReturnValue('a', 'wrun_test', undefined, ops),
|
||||
]);
|
||||
|
||||
const events: Event[] = [
|
||||
event('evnt_0', 'hook_created', `hook_${ULIDS[0]}`, {
|
||||
token: 'parked-token',
|
||||
isWebhook: false,
|
||||
}),
|
||||
event('evnt_1', 'wait_created', `wait_${ULIDS[1]}`, { resumeAt }),
|
||||
event('evnt_2', 'step_created', `step_${ULIDS[2]}`, {
|
||||
stepName: 'stepA',
|
||||
}),
|
||||
event('evnt_3', 'step_started', `step_${ULIDS[2]}`, {
|
||||
stepName: 'stepA',
|
||||
}),
|
||||
event('evnt_4', 'hook_received', `hook_${ULIDS[0]}`, { payload }),
|
||||
event('evnt_5', 'wait_completed', `wait_${ULIDS[1]}`, { resumeAt }),
|
||||
event('evnt_6', 'step_completed', `step_${ULIDS[2]}`, {
|
||||
stepName: 'stepA',
|
||||
result: stepAResult,
|
||||
}),
|
||||
event('evnt_7', 'step_created', `step_${ULIDS[3]}`, {
|
||||
stepName: 'afterBoth',
|
||||
}),
|
||||
];
|
||||
|
||||
const ctx = setupWorkflowContext(events);
|
||||
const useStep = createUseStep(ctx);
|
||||
const sleep = createSleep(ctx);
|
||||
const createHook = createCreateHook(ctx);
|
||||
|
||||
const error = await replay(ctx, async () => {
|
||||
const stepA = useStep('stepA');
|
||||
const afterBoth = useStep('afterBoth');
|
||||
// Fire-and-forget hook: its payload (evnt_4) is consumed but never
|
||||
// claimed, so its barrier stays unarmed and parks the wait behind it.
|
||||
createHook({ token: 'parked-token' });
|
||||
await Promise.all([sleep('5s'), stepA()]);
|
||||
await afterBoth();
|
||||
});
|
||||
|
||||
expectSuspendedWithPendingSteps(ctx, error, ['afterBoth']);
|
||||
};
|
||||
|
||||
it('terminates and suspends with log-order draws on', async () => {
|
||||
// Pin the flag rather than inherit the ambient environment: a suite-wide
|
||||
// WORKFLOW_LOG_ORDER_DRAWS=0 sweep would otherwise silently run the off
|
||||
// path twice and this test would prove nothing about the turnstile.
|
||||
vi.stubEnv('WORKFLOW_LOG_ORDER_DRAWS', '1');
|
||||
try {
|
||||
await scenario();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it('terminates and suspends with log-order draws off', async () => {
|
||||
vi.stubEnv('WORKFLOW_LOG_ORDER_DRAWS', '0');
|
||||
try {
|
||||
await scenario();
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -50,6 +50,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
});
|
||||
const ulid = monotonicFactory(() => context.globalThis.Math.random());
|
||||
const workflowStartedAt = context.globalThis.Date.now();
|
||||
// Real-session parity: the log-order-draws quiescence fixpoint keys its
|
||||
// progress metric on `mintCount`; without it the loop degrades to a single
|
||||
// turn and this suite would only exercise a degraded variant.
|
||||
let mintCount = 0;
|
||||
const promiseQueueHolder = { current: Promise.resolve() };
|
||||
const ctxRef: { current?: WorkflowOrchestratorContext } = {};
|
||||
const ctx: WorkflowOrchestratorContext = {
|
||||
@@ -71,7 +75,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
getPromiseQueue: () => promiseQueueHolder.current,
|
||||
}),
|
||||
invocationsQueue: new Map(),
|
||||
generateUlid: () => ulid(workflowStartedAt),
|
||||
generateUlid: () => {
|
||||
mintCount += 1;
|
||||
return ulid(workflowStartedAt);
|
||||
},
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * context.globalThis.Math.random())
|
||||
),
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
import type { Event, WorkflowRun } from '@workflow/world';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { WorkflowSuspension } from './global.js';
|
||||
import {
|
||||
dehydrateStepReturnValue,
|
||||
dehydrateWorkflowArguments,
|
||||
} from './serialization.js';
|
||||
import { runWorkflow } from './workflow.js';
|
||||
|
||||
/**
|
||||
* Offline regression coverage for `WORKFLOW_LOG_ORDER_DRAWS=1`: correlation-id
|
||||
* draw order pinned to event-log order, so draw bindings are stable under
|
||||
* dense-prefix extension and concurrent replays of different-length prefixes
|
||||
* mint compatible ids.
|
||||
*
|
||||
* The shape is the 2026-08-20 production corruption (five runs on
|
||||
* 5.0.0-beta.43): a fan-out where each branch launches a step, then races a
|
||||
* hook against a watchdog sleep. A replay whose dense prefix ends just before
|
||||
* a sibling branch's launch completion sees that branch parked at its `await`
|
||||
* minting nothing, so the woken branch's finalize takes the ordinal a fresher
|
||||
* replay gives the sibling's wait. With draws pinned to log order, the
|
||||
* finalize is minted inside the cascade of the delivery that enabled it, and
|
||||
* extending the log can only append draws, never renumber them.
|
||||
*/
|
||||
|
||||
const RUN_ID = 'wrun_log_order_draws';
|
||||
|
||||
async function makeRun(): Promise<WorkflowRun> {
|
||||
const ops: Promise<unknown>[] = [];
|
||||
const input = await dehydrateWorkflowArguments([], RUN_ID, undefined, ops);
|
||||
await Promise.all(ops);
|
||||
return {
|
||||
runId: RUN_ID,
|
||||
workflowName: 'workflow',
|
||||
status: 'running',
|
||||
input,
|
||||
createdAt: new Date('2024-01-01T00:00:00.000Z'),
|
||||
updatedAt: new Date('2024-01-01T00:00:00.000Z'),
|
||||
startedAt: new Date('2024-01-01T00:00:00.000Z'),
|
||||
deploymentId: 'test-deployment',
|
||||
};
|
||||
}
|
||||
|
||||
const TRANSFORM = `;globalThis.__private_workflows = new Map();
|
||||
globalThis.__private_workflows.set("workflow", workflow);`;
|
||||
|
||||
/**
|
||||
* The 2026-08-20 production shape (five corrupted runs on 5.0.0-beta.43): a
|
||||
* fan-out where each branch launches a step, then races a hook against a
|
||||
* watchdog sleep, and finalizes when the hook wins. A replay whose dense
|
||||
* prefix ends just before a sibling branch's launch completion sees that
|
||||
* branch parked at its `await` — the branch mints nothing, not even its
|
||||
* watchdog wait — so the woken branch's `finalizeTask` draws the ordinal a
|
||||
* fresher replay gives the sibling's wait. One correlation id then names both
|
||||
* a step and a wait, and every later replay fails with an unconsumable
|
||||
* `step_created` (CORRUPTED_EVENT_LOG).
|
||||
*
|
||||
* The two branches' pre-race hops differ on purpose: the woken branch reaches
|
||||
* its mint through the `Promise.race` resolution (two hops after delivery)
|
||||
* while the unblocked sibling mints its wait one hop after its own delivery,
|
||||
* which is how the sibling overtakes it on the shared counter.
|
||||
*/
|
||||
const BLOCKED_BRANCH_CODE = `
|
||||
const useStep = globalThis[Symbol.for("WORKFLOW_USE_STEP")];
|
||||
const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")];
|
||||
const sleep = globalThis[Symbol.for("WORKFLOW_SLEEP")];
|
||||
const launchTask = useStep("launchTask");
|
||||
const finalizeTask = useStep("finalizeTask");
|
||||
const WATCHDOG = "watchdog";
|
||||
async function workflow() {
|
||||
await Promise.all([0, 1, 2].map(async (task) => {
|
||||
const hook = createHook({ token: "task-done-" + task });
|
||||
await launchTask(task);
|
||||
const winner = await Promise.race([
|
||||
hook,
|
||||
sleep("1h").then(() => WATCHDOG),
|
||||
]);
|
||||
if (winner !== WATCHDOG) {
|
||||
await finalizeTask(task);
|
||||
}
|
||||
}));
|
||||
}${TRANSFORM}`;
|
||||
|
||||
/** Replays the blocked-branch workflow and returns every pending entity. */
|
||||
async function blockedBranchEntities(
|
||||
events: Event[]
|
||||
): Promise<{ type: string; correlationId: string; stepName?: string }[]> {
|
||||
const run = await makeRun();
|
||||
try {
|
||||
await runWorkflow(BLOCKED_BRANCH_CODE, run, events, undefined);
|
||||
} catch (error) {
|
||||
const suspension = error as WorkflowSuspension;
|
||||
if (suspension.name !== 'WorkflowSuspension') {
|
||||
throw error;
|
||||
}
|
||||
return suspension.steps.map((item) => ({
|
||||
type: item.type,
|
||||
correlationId: item.correlationId,
|
||||
stepName: (item as { stepName?: string }).stepName,
|
||||
}));
|
||||
}
|
||||
throw new Error('expected the replay to suspend');
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the two dense prefixes of the production log. The shorter one ends
|
||||
* before the second branch's launch completion; the longer one appends it.
|
||||
*/
|
||||
async function blockedBranchPrefixes(): Promise<{
|
||||
shorter: Event[];
|
||||
longer: Event[];
|
||||
}> {
|
||||
const initial = await blockedBranchEntities([]);
|
||||
const hooks = initial.filter((item) => item.type === 'hook');
|
||||
const launches = initial.filter((item) => item.stepName === 'launchTask');
|
||||
expect(hooks).toHaveLength(3);
|
||||
expect(launches).toHaveLength(3);
|
||||
|
||||
let at = 0;
|
||||
const stamp = () => new Date(Date.parse('2024-01-01T00:00:00.000Z') + ++at);
|
||||
const event = (
|
||||
eventType: Event['eventType'],
|
||||
correlationId: string,
|
||||
eventData: object
|
||||
): Event => ({
|
||||
eventId: `event-${at + 1}`,
|
||||
runId: RUN_ID,
|
||||
eventType,
|
||||
correlationId,
|
||||
eventData: eventData as Event['eventData'],
|
||||
createdAt: stamp(),
|
||||
});
|
||||
|
||||
const ops: Promise<unknown>[] = [];
|
||||
const launchResult = await dehydrateStepReturnValue(
|
||||
'launched',
|
||||
RUN_ID,
|
||||
undefined,
|
||||
ops
|
||||
);
|
||||
const hookPayload = await dehydrateStepReturnValue(
|
||||
{ done: true },
|
||||
RUN_ID,
|
||||
undefined,
|
||||
ops
|
||||
);
|
||||
await Promise.all(ops);
|
||||
|
||||
// Mirrors the production slot order: every branch's hook and launch created,
|
||||
// the first two launches completed, both of their hooks received, and the
|
||||
// third branch's launch completion as the extension event.
|
||||
const shorter: Event[] = [
|
||||
...hooks.map((hook, task) =>
|
||||
event('hook_created', hook.correlationId, {
|
||||
token: `task-done-${task}`,
|
||||
isWebhook: false,
|
||||
})
|
||||
),
|
||||
...launches.map((launch) =>
|
||||
event('step_created', launch.correlationId, { stepName: 'launchTask' })
|
||||
),
|
||||
event('step_completed', launches[0]!.correlationId, {
|
||||
stepName: 'launchTask',
|
||||
result: launchResult,
|
||||
}),
|
||||
event('step_completed', launches[1]!.correlationId, {
|
||||
stepName: 'launchTask',
|
||||
result: launchResult,
|
||||
}),
|
||||
event('hook_received', hooks[0]!.correlationId, {
|
||||
payload: hookPayload,
|
||||
}),
|
||||
event('hook_received', hooks[1]!.correlationId, {
|
||||
payload: hookPayload,
|
||||
}),
|
||||
];
|
||||
const longer: Event[] = [
|
||||
...shorter,
|
||||
event('step_completed', launches[2]!.correlationId, {
|
||||
stepName: 'launchTask',
|
||||
result: launchResult,
|
||||
}),
|
||||
];
|
||||
return { shorter, longer };
|
||||
}
|
||||
|
||||
function suspensionBindings(
|
||||
items: { type: string; correlationId: string; stepName?: string }[]
|
||||
) {
|
||||
return items
|
||||
.map((item) => `${item.correlationId}=${item.type}:${item.stepName ?? ''}`)
|
||||
.sort();
|
||||
}
|
||||
|
||||
describe('arrival-order draws (opt-out, WORKFLOW_LOG_ORDER_DRAWS=0)', () => {
|
||||
const original = process.env.WORKFLOW_LOG_ORDER_DRAWS;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.WORKFLOW_LOG_ORDER_DRAWS = '0';
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (original === undefined) {
|
||||
delete process.env.WORKFLOW_LOG_ORDER_DRAWS;
|
||||
} else {
|
||||
process.env.WORKFLOW_LOG_ORDER_DRAWS = original;
|
||||
}
|
||||
});
|
||||
|
||||
// Deliberate CONTROL: asserts the arrival-order BUG still reproduces with
|
||||
// the flag off. If this stops failing-to-bind (i.e. `rebound` comes back
|
||||
// empty), the control is obsolete, not broken — most likely because
|
||||
// positional rebinding was fixed independently of draw scheduling, e.g.
|
||||
// call-site-addressed correlation ids (vercel/workflow#3179) landing, which
|
||||
// removes the rebinding in BOTH modes. Delete this block then; do not chase
|
||||
// it as a regression.
|
||||
it('rebinds an ordinal from a step to a wait under extension (the control)', async () => {
|
||||
const { shorter, longer } = await blockedBranchPrefixes();
|
||||
const stale = await blockedBranchEntities(shorter);
|
||||
const fresh = await blockedBranchEntities(longer);
|
||||
const staleFinalizes = stale
|
||||
.filter((item) => item.stepName === 'finalizeTask')
|
||||
.map((item) => item.correlationId);
|
||||
const freshIds = new Set(fresh.map((item) => item.correlationId));
|
||||
const rebound = staleFinalizes.filter((id) => !freshIds.has(id));
|
||||
expect(rebound.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('log-order draws (the default)', () => {
|
||||
const original = process.env.WORKFLOW_LOG_ORDER_DRAWS;
|
||||
|
||||
beforeEach(() => {
|
||||
delete process.env.WORKFLOW_LOG_ORDER_DRAWS;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (original !== undefined) {
|
||||
process.env.WORKFLOW_LOG_ORDER_DRAWS = original;
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps every binding of the shorter prefix under extension', async () => {
|
||||
const { shorter, longer } = await blockedBranchPrefixes();
|
||||
const stale = await blockedBranchEntities(shorter);
|
||||
const fresh = await blockedBranchEntities(longer);
|
||||
// Every entity the shorter replay would create must exist, under the SAME
|
||||
// correlation id and kind, in the longer replay: extension appends draws,
|
||||
// never renumbers them.
|
||||
// The corruption signature is one correlation id bound to two different
|
||||
// entities across the two replays. Compare bindings on shared ids: an id
|
||||
// present in both pending sets must name the same entity. Ids only in the
|
||||
// shorter set are entities the extension consumed (the sibling's launch);
|
||||
// ids only in the longer set are the extension's appended draws.
|
||||
const byId = (
|
||||
items: { type: string; correlationId: string; stepName?: string }[]
|
||||
) =>
|
||||
new Map(
|
||||
items.map((item) => [
|
||||
item.correlationId,
|
||||
`${item.type}:${item.stepName ?? ''}`,
|
||||
])
|
||||
);
|
||||
const staleById = byId(stale);
|
||||
const freshById = byId(fresh);
|
||||
const rebound: string[] = [];
|
||||
for (const [id, binding] of staleById) {
|
||||
const extended = freshById.get(id);
|
||||
if (extended !== undefined && extended !== binding) {
|
||||
rebound.push(`${id}: ${binding} -> ${extended}`);
|
||||
}
|
||||
}
|
||||
expect(rebound).toEqual([]);
|
||||
// And specifically: the woken branches' finalize steps keep their ids.
|
||||
const staleFinalizes = stale
|
||||
.filter((item) => item.stepName === 'finalizeTask')
|
||||
.map((item) => item.correlationId)
|
||||
.sort();
|
||||
const freshFinalizes = fresh
|
||||
.filter((item) => item.stepName === 'finalizeTask')
|
||||
.map((item) => item.correlationId)
|
||||
.sort();
|
||||
expect(freshFinalizes).toEqual(staleFinalizes);
|
||||
});
|
||||
|
||||
it('is stable across every dense prefix of the log', async () => {
|
||||
// The pairwise test above targets the production window; this sweeps all
|
||||
// of them: no shared correlation id may change entity between any two
|
||||
// consecutive dense prefixes.
|
||||
const { longer } = await blockedBranchPrefixes();
|
||||
let previous: Map<string, string> | undefined;
|
||||
for (let length = 1; length <= longer.length; length++) {
|
||||
const entities = await blockedBranchEntities(longer.slice(0, length));
|
||||
const current = new Map(
|
||||
entities.map((item) => [
|
||||
item.correlationId,
|
||||
`${item.type}:${item.stepName ?? ''}`,
|
||||
])
|
||||
);
|
||||
if (previous) {
|
||||
for (const [id, binding] of previous) {
|
||||
const extended = current.get(id);
|
||||
expect(
|
||||
extended === undefined || extended === binding,
|
||||
`prefix ${length}: ${id} rebound ${binding} -> ${extended}`
|
||||
).toBe(true);
|
||||
}
|
||||
}
|
||||
previous = current;
|
||||
}
|
||||
});
|
||||
|
||||
it('is deterministic per prefix', async () => {
|
||||
const { longer } = await blockedBranchPrefixes();
|
||||
const a = suspensionBindings(await blockedBranchEntities(longer));
|
||||
const b = suspensionBindings(await blockedBranchEntities(longer));
|
||||
expect(b).toEqual(a);
|
||||
});
|
||||
});
|
||||
@@ -158,6 +158,14 @@ export interface WorkflowOrchestratorContext {
|
||||
* whole run and both replays of a run must draw in the same order.
|
||||
*/
|
||||
generateUlid: () => string;
|
||||
/**
|
||||
* Monotone count of correlation-id draws this replay has made. Progress
|
||||
* metric for {@link quiesceEarlierCascades}: a macrotask turn in which it
|
||||
* does not move (and no hydration is in flight) means every woken branch has
|
||||
* run as far as it can without another delivery. Optional so lightweight
|
||||
* test contexts degrade to the single-yield behavior.
|
||||
*/
|
||||
readonly mintCount?: number;
|
||||
generateNanoid: () => string;
|
||||
/**
|
||||
* Sequential promise queue that ensures all event-driven promise resolutions
|
||||
@@ -290,6 +298,11 @@ const DEFER_BEHIND: Record<DeliveryKind, readonly DeliveryKind[]> = {
|
||||
* Those two MUST agree exactly, and the doc block on
|
||||
* {@link awaitEarlierDeliveries} stakes deadlock-freedom on it, so the
|
||||
* condition lives here rather than being spelled out twice.
|
||||
*
|
||||
* One deliberate exception: the log-order-draws turnstile in
|
||||
* {@link quiesceEarlierCascades} waits on ANY lower armed entry, a strictly
|
||||
* wider relation than this one. Why that width cannot deadlock the
|
||||
* safety-net dispenser is argued at the turnstile itself.
|
||||
*/
|
||||
function gatesOn(
|
||||
kind: DeliveryKind,
|
||||
@@ -431,17 +444,127 @@ function computeResolvesOnItsOwn(
|
||||
* loaded. storm-log-replay.test.ts replays a production log corrupted exactly
|
||||
* that way.)
|
||||
*/
|
||||
/**
|
||||
* Whether correlation-id draw order is pinned to event-log order. Default ON:
|
||||
* only the literal string `WORKFLOW_LOG_ORDER_DRAWS=0` opts out — `=false`,
|
||||
* `=off`, and every other value keep it enabled. Read per call so tests can
|
||||
* flip it.
|
||||
*
|
||||
* Off, a delivery that had to defer yields ONE macrotask after its
|
||||
* predecessors resolve — enough for short consumers, but a woken branch whose
|
||||
* path to its next draw crosses more hops (a `Promise.race` resolution, a
|
||||
* user-level semaphore, an async-iterator read) can still be overtaken by a
|
||||
* later-in-log delivery's shorter cascade, so the run's draw order — and
|
||||
* therefore its correlation ids — depends on how much log this replay loaded.
|
||||
* On, the yield becomes a fixpoint: the delivery resolves only once every
|
||||
* earlier cascade has quiesced, making the draw sequence a pure function of
|
||||
* the dense log, stable under prefix extension, and concurrent writers'
|
||||
* duplicate creates identical (deduped) instead of colliding.
|
||||
*/
|
||||
function isLogOrderDrawsEnabled(): boolean {
|
||||
return process.env.WORKFLOW_LOG_ORDER_DRAWS !== '0';
|
||||
}
|
||||
|
||||
/**
|
||||
* One quiescence turn: lets the entire pending microtask queue drain, then
|
||||
* yields to the event loop once. `setImmediate` (check phase) is used where
|
||||
* available because Node clamps `setTimeout(0)` to ~1ms while `setImmediate`
|
||||
* costs ~20µs — and every branch-deciding delivery pays this turn at least
|
||||
* once, so on a sequential replay the clamp is the whole cost. Timers still
|
||||
* run between consecutive turns (the loop re-enters the event loop each
|
||||
* iteration, passing through the timers phase), so chains parked on the
|
||||
* safety-net dispenser's `setTimeout` cadence are not starved.
|
||||
*/
|
||||
function quiescenceTurn(delayMs: number): Promise<void> {
|
||||
if (delayMs === 0 && typeof setImmediate === 'function') {
|
||||
return new Promise<void>((resolve) => setImmediate(resolve));
|
||||
}
|
||||
return new Promise<void>((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits until the workflow can make no further progress without another
|
||||
* delivery: repeated (promise-queue drain + event-loop turn)s until a full
|
||||
* turn passes with no new ULID draws and no hydration in flight.
|
||||
*
|
||||
* Termination: each extra iteration requires a new ULID draw or a hydration
|
||||
* started in the previous turn. `mintCount` counts EVERY draw from the run's
|
||||
* sequence — correlation ids and the serialization-driven draws (stream ids
|
||||
* minted through the `STABLE_ULID` global while dehydrating) — which is
|
||||
* conservative in the safe direction: serialization draws only extend the
|
||||
* wait, and both body progress and the serialization work one cascade can
|
||||
* schedule are finite between deliveries, so the fixpoint is reached. The
|
||||
* loop holds this delivery's own barrier registered (its `markDelivered` has
|
||||
* not run), so `isDeliveryIdle` stays false and no suspension can preempt the
|
||||
* cascade being waited out.
|
||||
*
|
||||
* A rejected `promiseQueue` settles immediately and forever, so looping at
|
||||
* the normal cadence on a failed run would degenerate into a busy loop (the
|
||||
* same hazard {@link ensureBarrierSafetyNet} documents). Iterations that
|
||||
* observe the queue rejected back off to a 50ms tick instead.
|
||||
*/
|
||||
async function quiesceEarlierCascades(
|
||||
ctx: WorkflowOrchestratorContext,
|
||||
eventIndex: number
|
||||
): Promise<void> {
|
||||
for (;;) {
|
||||
const mintsBefore = ctx.mintCount ?? 0;
|
||||
const pendingBefore = ctx.pendingDeliveries;
|
||||
// Settled or rejected, the queue snapshot only orders us behind work
|
||||
// already chained; a rejection is the run failing elsewhere.
|
||||
const queueRejected = await ctx.promiseQueue.then(
|
||||
() => false,
|
||||
() => true
|
||||
);
|
||||
await quiescenceTurn(queueRejected ? 50 : 0);
|
||||
if (
|
||||
(ctx.mintCount ?? 0) !== mintsBefore ||
|
||||
pendingBefore !== 0 ||
|
||||
ctx.pendingDeliveries !== 0
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
// Quiet is not enough on its own: several delivery chains can be sitting
|
||||
// in this loop at once, and letting the first quiet observation resolve
|
||||
// would break the tie by timer arrival — the arrival-order dependence this
|
||||
// mode removes. A lower-index ARMED barrier is a delivery committed to
|
||||
// happening that has not happened yet, so this one keeps waiting. Unarmed
|
||||
// entries (buffered payloads nobody has claimed) do not block, exactly as
|
||||
// in `gatesOn`: their handover is claim-driven, which is body-position
|
||||
// determined and therefore already a function of the prefix.
|
||||
//
|
||||
// This is deliberately a WIDER waits-for relation than `gatesOn` (which,
|
||||
// e.g., excludes wait→wait): under log-order draws EVERY branch-deciding
|
||||
// delivery must resolve in log order, kinds included. The width is safe
|
||||
// against the dispenser deadlock that `resolvesOnItsOwn` guards, because
|
||||
// the one edge the wider relation adds — waiting on an armed entry that
|
||||
// gatesOn does not model — always bottoms out at the same unarmed payload:
|
||||
// a lower armed WAIT is non-self-resolving only when it is (transitively)
|
||||
// parked behind an unclaimed buffered payload, and a wait gates on every
|
||||
// lower hook and step directly, so the spinner here also gates on that
|
||||
// payload through `gatesOn` and is itself reported non-self-resolving.
|
||||
// The dispenser therefore stays unblocked and retires the chain head; see
|
||||
// the parked-chain test in delivery-barrier-coverage.test.ts.
|
||||
let lowerArmed = false;
|
||||
for (const [index, entry] of ctx.pendingDeliveryBarriers ?? []) {
|
||||
if (index < eventIndex && entry.armed) {
|
||||
lowerArmed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!lowerArmed) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function awaitEarlierDeliveries(
|
||||
ctx: WorkflowOrchestratorContext,
|
||||
eventIndex: number | undefined,
|
||||
kind: DeliveryKind
|
||||
): Promise<void> {
|
||||
// Defensive: tolerate contexts that predate this field (test harnesses).
|
||||
if (
|
||||
eventIndex === undefined ||
|
||||
!ctx.pendingDeliveryBarriers ||
|
||||
ctx.pendingDeliveryBarriers.size === 0
|
||||
) {
|
||||
if (eventIndex === undefined || !ctx.pendingDeliveryBarriers) {
|
||||
return;
|
||||
}
|
||||
const barriers = ctx.pendingDeliveryBarriers;
|
||||
@@ -454,6 +577,19 @@ export async function awaitEarlierDeliveries(
|
||||
}
|
||||
if (earlier.length > 0) {
|
||||
await Promise.all(earlier);
|
||||
}
|
||||
if (isLogOrderDrawsEnabled()) {
|
||||
// Unconditional, not just when a barrier was still registered: an earlier
|
||||
// delivery's barrier deregisters when its resolve() runs, but the branch
|
||||
// it woke may still be hops away from its next draw. A later delivery
|
||||
// consumed after that deregistration sees an empty gate set, and without
|
||||
// this it would resolve mid-cascade and overtake the draw — the exact
|
||||
// arrival-order dependence this mode exists to remove. Costs one quiet
|
||||
// macrotask turn when nothing is in flight.
|
||||
await quiesceEarlierCascades(ctx, eventIndex);
|
||||
return;
|
||||
}
|
||||
if (earlier.length > 0) {
|
||||
// An earlier delivery being "delivered" only means its `resolve()` ran.
|
||||
// The branch it woke may need an arbitrary number of further microtask
|
||||
// hops before it reaches its next `useStep` call and draws a ULID — a
|
||||
|
||||
@@ -60,6 +60,10 @@ function setupWorkflowContext(
|
||||
const context = createContext({ seed: SEED, fixedTimestamp: FIXED_TS });
|
||||
const ulid = monotonicFactory(() => context.globalThis.Math.random());
|
||||
const workflowStartedAt = context.globalThis.Date.now();
|
||||
// Real-session parity: the log-order-draws quiescence fixpoint keys its
|
||||
// progress metric on `mintCount`; without it the loop degrades to a single
|
||||
// turn and this suite would only exercise a degraded variant.
|
||||
let mintCount = 0;
|
||||
const promiseQueueHolder = { current: Promise.resolve() };
|
||||
const ctxRef: { current?: WorkflowOrchestratorContext } = {};
|
||||
const ctx: WorkflowOrchestratorContext = {
|
||||
@@ -81,7 +85,13 @@ function setupWorkflowContext(
|
||||
getPromiseQueue: () => promiseQueueHolder.current,
|
||||
}),
|
||||
invocationsQueue: new Map(),
|
||||
generateUlid: () => ulid(workflowStartedAt),
|
||||
generateUlid: () => {
|
||||
mintCount += 1;
|
||||
return ulid(workflowStartedAt);
|
||||
},
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * context.globalThis.Math.random())
|
||||
),
|
||||
|
||||
@@ -36,6 +36,10 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
const context = createContext({ seed: SEED, fixedTimestamp: FIXED_TS });
|
||||
const ulid = monotonicFactory(() => context.globalThis.Math.random());
|
||||
const workflowStartedAt = context.globalThis.Date.now();
|
||||
// Real-session parity: the log-order-draws quiescence fixpoint keys its
|
||||
// progress metric on `mintCount`; without it the loop degrades to a single
|
||||
// turn and this suite would only exercise a degraded variant.
|
||||
let mintCount = 0;
|
||||
const promiseQueueHolder = { current: Promise.resolve() };
|
||||
const ctxRef: { current?: WorkflowOrchestratorContext } = {};
|
||||
const ctx: WorkflowOrchestratorContext = {
|
||||
@@ -57,7 +61,13 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext {
|
||||
getPromiseQueue: () => promiseQueueHolder.current,
|
||||
}),
|
||||
invocationsQueue: new Map(),
|
||||
generateUlid: () => ulid(workflowStartedAt),
|
||||
generateUlid: () => {
|
||||
mintCount += 1;
|
||||
return ulid(workflowStartedAt);
|
||||
},
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * context.globalThis.Math.random())
|
||||
),
|
||||
|
||||
@@ -23,6 +23,10 @@ export function setupWorkflowContext(
|
||||
fixedTimestamp: 1753481739458,
|
||||
});
|
||||
const ulid = monotonicFactory(() => context.globalThis.Math.random());
|
||||
// Real-session parity: the log-order-draws quiescence fixpoint keys its
|
||||
// progress metric on `mintCount`. Without it the loop degrades to a single
|
||||
// turn and these suites would only exercise a degraded variant.
|
||||
let mintCount = 0;
|
||||
const workflowStartedAt = context.globalThis.Date.now();
|
||||
const promiseQueueHolder = { current: Promise.resolve() };
|
||||
// Forward onUnconsumedEvent through ctx.onWorkflowError so tests that wire
|
||||
@@ -50,7 +54,13 @@ export function setupWorkflowContext(
|
||||
getPromiseQueue: () => promiseQueueHolder.current,
|
||||
}),
|
||||
invocationsQueue: new Map(),
|
||||
generateUlid: () => ulid(workflowStartedAt),
|
||||
generateUlid: () => {
|
||||
mintCount += 1;
|
||||
return ulid(workflowStartedAt);
|
||||
},
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * context.globalThis.Math.random())
|
||||
),
|
||||
|
||||
@@ -391,7 +391,18 @@ async function createWorkflowSession({
|
||||
const ulid = monotonicFactory(() => vmGlobalThis.Math.random());
|
||||
// Correlation IDs must be replay-stable. `startedAt` differs between a turbo
|
||||
// delivery and a later server-backed replay, so use fixedTimestamp.
|
||||
const generateUlid = () => ulid(fixedTimestamp);
|
||||
// The draw counter is the progress metric for `quiesceEarlierCascades`
|
||||
// (WORKFLOW_LOG_ORDER_DRAWS): a quiet turn is one that drew nothing. It
|
||||
// counts EVERY draw from this sequence — this same function is installed as
|
||||
// the `STABLE_ULID` global below, which serialization draws stream ids
|
||||
// from during dehydration — deliberately: quiescence must also wait out
|
||||
// serialization-driven draws, and counting extra draws only extends the
|
||||
// wait (see the termination note on `quiesceEarlierCascades`).
|
||||
let mintCount = 0;
|
||||
const generateUlid = () => {
|
||||
mintCount += 1;
|
||||
return ulid(fixedTimestamp);
|
||||
};
|
||||
const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) =>
|
||||
new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random())
|
||||
);
|
||||
@@ -480,6 +491,9 @@ async function createWorkflowSession({
|
||||
eventsConsumer,
|
||||
generateUlid,
|
||||
generateNanoid,
|
||||
get mintCount() {
|
||||
return mintCount;
|
||||
},
|
||||
invocationsQueue: new Map(),
|
||||
// Use getter/setter so the EventsConsumer's getPromiseQueue() always
|
||||
// sees the latest queue state as it's mutated by step/hook/sleep callbacks.
|
||||
|
||||
Reference in New Issue
Block a user