vercel__eve/scripts/workflow-stress-report.mjs

import { readdir, readFile, mkdir, writeFile } from "node:fs/promises";
import { dirname, relative, resolve, sep } from "node:path";
import { pathToFileURL } from "node:url";

const PERFORMANCE_LOG_PREFIX = "EVE_WORKFLOW_STRESS_METRIC=";
const REQUIRED_SCENARIOS = ["concurrent", "sequential"];

export async function collectWorkflowStressMetrics(artifactsRoot) {
  const artifactPaths = (await findJsonFiles(artifactsRoot)).sort();
  const metricsByRun = new Map();

  for (const artifactPath of artifactPaths) {
    const artifact = JSON.parse(await readFile(artifactPath, "utf8"));
    const logs = artifact?.result?.logs;

    if (!Array.isArray(logs)) {
      continue;
    }

    for (const log of logs) {
      if (typeof log !== "string") {
        continue;
      }

      const markerIndex = log.indexOf(PERFORMANCE_LOG_PREFIX);

      if (markerIndex === -1) {
        continue;
      }

      const metric = JSON.parse(log.slice(markerIndex + PERFORMANCE_LOG_PREFIX.length));
      validateMetric(metric, artifactPath);
      const runDirectory = findEvalRunDirectory(artifactsRoot, artifactPath);
      const metricsByScenario = metricsByRun.get(runDirectory) ?? new Map();
      metricsByScenario.set(metric.scenario, { artifactPath, metric });
      metricsByRun.set(runDirectory, metricsByScenario);
    }
  }

  const latestRun = [...metricsByRun.entries()]
    .sort(([left], [right]) => left.localeCompare(right, "en"))
    .at(-1);

  if (latestRun === undefined) {
    throw new Error(`Missing Workflow stress metrics under ${artifactsRoot}`);
  }

  const [runDirectory, metricsByScenario] = latestRun;

  for (const scenario of REQUIRED_SCENARIOS) {
    if (!metricsByScenario.has(scenario)) {
      throw new Error(
        `Latest Workflow stress metric run ${runDirectory} is missing the ${scenario} scenario`,
      );
    }
  }

  return {
    runDirectory,
    ...Object.fromEntries(
      [...metricsByScenario.entries()].map(([scenario, entry]) => [scenario, entry]),
    ),
  };
}

export function createWorkflowStressReport(metrics, metadata = {}) {
  const sequentialSamples = metrics.sequential.metric.samples;
  const firstConcurrentBatch = metrics.concurrent.metric.batches.find(
    (batch) => batch.turnNumber === 1,
  );
  const secondConcurrentBatch = metrics.concurrent.metric.batches.find(
    (batch) => batch.turnNumber === 2,
  );

  if (firstConcurrentBatch === undefined || secondConcurrentBatch === undefined) {
    throw new Error("Concurrent Workflow stress metric must include turn 1 and turn 2 batches");
  }

  return {
    generatedAt: new Date().toISOString(),
    metadata,
    schemaVersion: 1,
    scenarios: {
      concurrent: {
        firstTurns: summarizeSamples(firstConcurrentBatch.samples),
        firstTurnsBatchDurationMs: firstConcurrentBatch.batchDurationMs,
        secondTurns: summarizeSamples(secondConcurrentBatch.samples),
        secondTurnsBatchDurationMs: secondConcurrentBatch.batchDurationMs,
      },
      sequential: {
        allTurns: summarizeSamples(sequentialSamples),
        coldTurn: summarizeSamples(sequentialSamples.slice(0, 1)),
        firstTenWarmTurns: summarizeSamples(sequentialSamples.slice(1, 11)),
        sequentialTurnOrderSlopeMsPerTurn: calculateLinearSlope(
          sequentialSamples.slice(1).map((sample) => [sample.turnNumber, sample.durationMs]),
        ),
        lastTenTurns: summarizeSamples(sequentialSamples.slice(-10)),
        warmTurns: summarizeSamples(sequentialSamples.slice(1)),
      },
    },
    sources: {
      concurrent: metrics.concurrent.artifactPath,
      runDirectory: metrics.runDirectory,
      sequential: metrics.sequential.artifactPath,
    },
  };
}

export function renderWorkflowStressMarkdown(report) {
  const sequential = report.scenarios.sequential;
  const concurrent = report.scenarios.concurrent;
  const rows = [
    ["Sequential, all turns", sequential.allTurns],
    ["Sequential, first/cold turn", sequential.coldTurn],
    ["Sequential, warm turns 2–100", sequential.warmTurns],
    ["Sequential, first 10 warm turns (2–11)", sequential.firstTenWarmTurns],
    ["Sequential, turns 91–100", sequential.lastTenTurns],
    ["Concurrent, first turn", concurrent.firstTurns],
    ["Concurrent, second turn", concurrent.secondTurns],
  ];
  const lines = [
    "## Workflow stress performance",
    "",
    "> Informational hosted measurement. Compare paired base/head runs before attributing a change; this report is not a performance gate.",
    "",
    "| Scenario | Samples | Mean | p50 | p90 | p95 | Min | Max |",
    "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
    ...rows.map(([label, summary]) =>
      [
        `| ${label}`,
        summary.count,
        formatDuration(summary.meanMs),
        formatDuration(summary.p50Ms),
        formatDuration(summary.p90Ms),
        formatDuration(summary.p95Ms),
        formatDuration(summary.minMs),
        `${formatDuration(summary.maxMs)} |`,
      ].join(" | "),
    ),
    "",
    `Sequential warm turn-order slope: **${sequential.sequentialTurnOrderSlopeMsPerTurn.toFixed(2)} ms/turn** (history-correlated, but also time-order confounded).`,
    `Concurrent batch wall time: **${formatDuration(concurrent.firstTurnsBatchDurationMs)}** first turns, **${formatDuration(concurrent.secondTurnsBatchDurationMs)}** second turns.`,
  ];

  const metadata = Object.entries(report.metadata).filter(([, value]) => value !== undefined);

  if (metadata.length > 0) {
    lines.push(
      "",
      `Run: ${metadata.map(([key, value]) => `${key}=\`${String(value)}\``).join(", ")}`,
    );
  }

  return `${lines.join("\n")}\n`;
}

function summarizeSamples(samples) {
  const values = samples.map((sample) => sample.durationMs).sort((left, right) => left - right);

  if (values.length === 0) {
    throw new Error("Cannot summarize an empty Workflow stress sample set");
  }

  return {
    count: values.length,
    maxMs: values.at(-1),
    meanMs: values.reduce((total, value) => total + value, 0) / values.length,
    minMs: values[0],
    p50Ms: percentile(values, 0.5),
    p90Ms: percentile(values, 0.9),
    p95Ms: percentile(values, 0.95),
  };
}

function percentile(sortedValues, probability) {
  const position = (sortedValues.length - 1) * probability;
  const lowerIndex = Math.floor(position);
  const upperIndex = Math.ceil(position);
  const lowerValue = sortedValues[lowerIndex];
  const upperValue = sortedValues[upperIndex];

  return lowerValue + (upperValue - lowerValue) * (position - lowerIndex);
}

function calculateLinearSlope(points) {
  const xMean = points.reduce((total, [x]) => total + x, 0) / points.length;
  const yMean = points.reduce((total, [, y]) => total + y, 0) / points.length;
  let numerator = 0;
  let denominator = 0;

  for (const [x, y] of points) {
    numerator += (x - xMean) * (y - yMean);
    denominator += (x - xMean) ** 2;
  }

  return denominator === 0 ? 0 : numerator / denominator;
}

function formatDuration(milliseconds) {
  return `${(milliseconds / 1_000).toFixed(3)}s`;
}

function validateMetric(metric, artifactPath) {
  if (
    metric?.schemaVersion !== 1 ||
    metric.fixture !== "agent-workflow-stress" ||
    !REQUIRED_SCENARIOS.includes(metric.scenario) ||
    metric.unit !== "milliseconds"
  ) {
    throw new Error(`Invalid Workflow stress metric in ${artifactPath}`);
  }

  if (metric.scenario === "sequential") {
    validateSamples(metric.samples, artifactPath, "turnNumber");
    return;
  }

  if (!Array.isArray(metric.batches) || metric.batches.length === 0) {
    throw new Error(`Invalid concurrent Workflow stress batches in ${artifactPath}`);
  }

  for (const batch of metric.batches) {
    if (!Number.isInteger(batch?.turnNumber) || !isFiniteNonnegative(batch?.batchDurationMs)) {
      throw new Error(`Invalid concurrent Workflow stress batch in ${artifactPath}`);
    }

    validateSamples(batch.samples, artifactPath, "sessionNumber");
  }
}

function validateSamples(samples, artifactPath, ordinalKey) {
  if (
    !Array.isArray(samples) ||
    samples.length === 0 ||
    samples.some(
      (sample) =>
        !isFiniteNonnegative(sample?.durationMs) || !Number.isInteger(sample?.[ordinalKey]),
    )
  ) {
    throw new Error(`Invalid Workflow stress samples in ${artifactPath}`);
  }
}

function isFiniteNonnegative(value) {
  return typeof value === "number" && Number.isFinite(value) && value >= 0;
}

async function findJsonFiles(root) {
  const entries = await readdir(root, { withFileTypes: true });
  const paths = await Promise.all(
    entries.map(async (entry) => {
      const path = resolve(root, entry.name);

      if (entry.isDirectory()) {
        return findJsonFiles(path);
      }

      return entry.isFile() && entry.name.endsWith(".json") ? [path] : [];
    }),
  );

  return paths.flat();
}

function findEvalRunDirectory(artifactsRoot, artifactPath) {
  const root = resolve(artifactsRoot);
  const segments = relative(root, artifactPath).split(sep);
  const evalsIndex = segments.indexOf("evals");

  if (evalsIndex === -1) {
    throw new Error(`Workflow stress metric is outside an eval run directory: ${artifactPath}`);
  }

  return resolve(root, ...segments.slice(0, evalsIndex));
}

function parseArguments(argv) {
  const options = {};

  for (let index = 0; index < argv.length; index += 1) {
    const name = argv[index];
    const value = argv[index + 1];

    if (!["--artifacts", "--json", "--markdown"].includes(name) || value === undefined) {
      throw new Error(
        "Usage: workflow-stress-report.mjs --artifacts <dir> [--json <path>] [--markdown <path>]",
      );
    }

    options[name.slice(2)] = value;
    index += 1;
  }

  if (options.artifacts === undefined) {
    throw new Error("--artifacts is required");
  }

  return options;
}

async function writeOutput(path, value) {
  await mkdir(dirname(path), { recursive: true });
  await writeFile(path, value);
}

async function main() {
  const options = parseArguments(process.argv.slice(2));
  const metrics = await collectWorkflowStressMetrics(options.artifacts);
  const report = createWorkflowStressReport(metrics, {
    attempt: process.env.GITHUB_RUN_ATTEMPT,
    model: process.env.EVE_E2E_MODEL,
    runId: process.env.GITHUB_RUN_ID,
    sha: process.env.GITHUB_SHA,
  });
  const markdown = renderWorkflowStressMarkdown(report);

  if (options.json !== undefined) {
    await writeOutput(options.json, `${JSON.stringify(report, null, 2)}\n`);
  }

  if (options.markdown !== undefined) {
    await writeOutput(options.markdown, markdown);
  }

  process.stdout.write(markdown);
}

if (
  process.argv[1] !== undefined &&
  import.meta.url === pathToFileURL(resolve(process.argv[1])).href
) {
  await main();
}