#!/usr/bin/env node // @ts-check // // resolve-cpuprofile.mjs — de-minify V8 .cpuprofile stacks using server source maps. // // WHY: prod pods emit V8 `.cpuprofile`s to find what blocks the event loop, but the // standalone build's server chunks are minified, so frames read like // `p @ chunks/_0eaaij7._.js:4398:12` — unnameable. This tool loads the matching // `.js.map` files and rewrites each frame's `(url, lineNumber, columnNumber)` back to // its original `source.ts:line:col` + original function name via `source-map`. // // Server source maps are NOT shipped in the runtime image (they added ~761 MB per // prod pod). They are published per build as a separate, fetched-on-demand artifact // image `ghcr.io/civitai/civitai-web-maps:` keyed by the SAME tag as the runtime // image. This tool can fetch that artifact for a given tag (`--image`) and resolve // against it, OR resolve against a local maps dir (`--maps`). // // USAGE: // # Fetch the maps for the build that produced the profile, then resolve: // node scripts/resolve-cpuprofile.mjs --image [options] // # Or resolve against a local directory of .js.map files: // node scripts/resolve-cpuprofile.mjs --maps [options] // // OPTIONS: // --image Fetch this build's server maps from the maps artifact image, // then resolve. may be a bare tag (e.g. v5.0.1806-datapacket // or 20260610123456-abc1234) — expanded against --maps-repo // (default ghcr.io/civitai/civitai-web-maps) — or a full // repo:tag reference. Requires `crane` (or `oras`) on PATH and // registry read auth (e.g. `crane auth login ghcr.io`, or a // docker config already logged in). Mutually exclusive with --maps. // --image-sha Like --image, but you pass only the commit sha. The runtime // image tag (-) and the maps tag (-) share // the sha but not the build timestamp, so a runtime tag would // 404 as a maps tag. This lists --maps-repo via `crane ls` and // picks the newest tag ending in `-`. If both --image and // --image-sha are given, --image-sha wins. Mutually exclusive // with --maps. // --maps-repo Override the maps artifact repo used to expand a bare --image // tag (default ghcr.io/civitai/civitai-web-maps). // --maps Directory to search recursively for `.js.map` files // (offline/local mode; e.g. an already-extracted maps tree). // --top Show the N hottest self-time functions (default 25). // --block Run the "longest synchronous block" analysis (see below) and // print its stack, de-minified. // --json Emit the full resolved profile as JSON instead of a report. // --no-resolve Skip map resolution (raw frames) — useful to diff before/after. // // Exactly one of --image / --image-sha or --maps is required (unless --no-resolve). // // "Longest synchronous block": V8 sampling profiles record per-sample deltas // (`timeDeltas`). A run of consecutive samples that stay inside the same top-of-stack // leaf (no return to the scheduler) approximates an uninterrupted synchronous block. // We find the single longest such run and print its (de-minified) leaf stack — the // likely event-loop culprit. // // V8 coordinate convention: callFrame.lineNumber / columnNumber are 0-based. // `source-map` originalPositionFor wants { line: 1-based, column: 0-based }. import { readFile, readdir, stat, mkdtemp, rm } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { spawnSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; import process from 'node:process'; const DEFAULT_MAPS_REPO = 'ghcr.io/civitai/civitai-web-maps'; // `source-map` is a direct dependency (package.json). Imported lazily so --no-resolve // and --help work even if it's somehow absent. async function loadSourceMapModule() { try { return await import('source-map'); } catch { console.error( "error: the 'source-map' package is required for resolution.\n" + ' run `pnpm add -D source-map` (it is already a dependency of this repo),\n' + ' or pass --no-resolve to print raw frames.' ); process.exit(1); } } function parseArgs(argv) { const args = { profile: /** @type {string | null} */ (null), mapsDir: /** @type {string | null} */ (null), image: /** @type {string | null} */ (null), imageSha: /** @type {string | null} */ (null), mapsRepo: DEFAULT_MAPS_REPO, top: 25, block: false, json: false, resolve: true, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (a === '--maps') args.mapsDir = argv[++i]; else if (a === '--image') args.image = argv[++i]; else if (a === '--image-sha') args.imageSha = argv[++i]; else if (a === '--maps-repo') args.mapsRepo = argv[++i]; else if (a === '--top') args.top = Number(argv[++i]); else if (a === '--block') args.block = true; else if (a === '--json') args.json = true; else if (a === '--no-resolve') args.resolve = false; else if (a === '-h' || a === '--help') { printHelp(); process.exit(0); } else if (!a.startsWith('-') && !args.profile) args.profile = a; else { console.error(`error: unexpected argument: ${a}`); process.exit(1); } } return args; } function printHelp() { console.log( 'Usage: node scripts/resolve-cpuprofile.mjs (--image | --image-sha | --maps )\n' + ' [--maps-repo ] [--top N] [--block] [--json] [--no-resolve]' ); } // Resolve a bare tag to a full maps-image reference. A value that already contains // a '/' (a repo path) or starts with a registry host is treated as a full ref; a // bare tag is expanded against mapsRepo. function resolveImageRef(image, mapsRepo) { if (image.includes('/')) return image; // already repo[:tag] or registry/repo:tag // bare tag (possibly with a leading ':' someone added) -> repo:tag const tag = image.startsWith(':') ? image.slice(1) : image; return `${mapsRepo}:${tag}`; } // Resolve a bare commit sha to a full maps-image reference by listing the maps repo // and picking the newest tag ending in `-`. The runtime image tag and the maps // tag share the sha but carry different build timestamps (`-`), so a caller // who only knows the runtime sha can't name the maps tag directly. Requires `crane` // on PATH and registry read auth. // // Tag selection: build tags are `-`, but a floating `sha-` alias // may also match `-`. Prefer the newest *timestamped* tag (largest numeric // prefix = newest build); fall back to the lexically-largest match only when no // timestamped tag is present. Pure/side-effect-free so it can be unit-tested. function pickTagBySha(tags, sha) { const matches = tags.map((t) => t.trim()).filter((t) => t && t.endsWith(`-${sha}`)); if (!matches.length) return null; const timestamped = matches .filter((t) => /^\d+-/.test(t)) .sort((a, b) => Number(a.split('-')[0]) - Number(b.split('-')[0])); if (timestamped.length) return timestamped[timestamped.length - 1]; return matches.sort()[matches.length - 1]; } function resolveTagBySha(sha, mapsRepo) { const r = spawnSync('crane', ['ls', mapsRepo], { encoding: 'utf8' }); if (r.status !== 0) { throw new Error( `crane ls ${mapsRepo} failed (exit ${r.status}): ${(r.stderr || '').trim()}. ` + 'Ensure `crane` is on PATH and registry read auth is configured.' ); } const tag = pickTagBySha(r.stdout.split('\n'), sha); if (!tag) throw new Error(`no maps tag ending in -${sha} found in ${mapsRepo}`); return `${mapsRepo}:${tag}`; } // Fetch the server maps for a build into a fresh temp dir and return that dir. // Prefers `crane export` (single static binary, already cached in the Tekton build // node); falls back to `oras pull`. The maps artifact stores files under // /server-maps mirroring .next/server, so we return /server-maps if present // (else , so a differently-rooted artifact still resolves). // // Auth: relies on ambient registry auth — a docker config already logged in to the // registry (DOCKER_CONFIG / ~/.docker/config.json) or `crane auth login ghcr.io`. // No credentials are embedded here. async function fetchMapsImage(ref) { const tmp = await mkdtemp(path.join(os.tmpdir(), 'cpuprofile-maps-')); const haveCrane = spawnSync('crane', ['version'], { stdio: 'ignore' }).status === 0; const haveOras = !haveCrane && spawnSync('oras', ['version'], { stdio: 'ignore' }).status === 0; if (!haveCrane && !haveOras) { await rm(tmp, { recursive: true, force: true }); throw new Error( "neither 'crane' nor 'oras' found on PATH. Install one " + '(e.g. `go install github.com/google/go-containerregistry/cmd/crane@latest`) ' + 'or download a release binary, then ensure registry read auth is configured.' ); } console.error(`info: fetching maps artifact ${ref} via ${haveCrane ? 'crane' : 'oras'} ...`); let res; if (haveCrane) { // `crane export` streams the image filesystem as a tar; pipe it into tar -x. // Using a shell pipeline keeps memory flat for large map sets. res = spawnSync('sh', ['-c', `crane export "${ref}" - | tar -x -C "${tmp}"`], { stdio: ['ignore', 'inherit', 'inherit'], }); } else { res = spawnSync('oras', ['pull', ref, '-o', tmp], { stdio: ['ignore', 'inherit', 'inherit'] }); } if (res.status !== 0) { await rm(tmp, { recursive: true, force: true }); throw new Error( `failed to fetch maps artifact ${ref} (exit ${res.status}). ` + 'Check the tag exists in the maps repo and that registry read auth is configured.' ); } const nested = path.join(tmp, 'server-maps'); const dir = existsSync(nested) ? nested : tmp; return { dir, cleanup: () => rm(tmp, { recursive: true, force: true }) }; } // Recursively index every *.js.map under dir, keyed by the basename of the // chunk it maps (i.e. `.js`). Turbopack/webpack name maps `.js.map`. async function indexMaps(dir) { /** @type {Map} */ const byChunkBasename = new Map(); /** @type {Map} */ const byMapBasename = new Map(); /** @param {string} d */ async function walk(d) { let entries; try { entries = await readdir(d, { withFileTypes: true }); } catch { return; } for (const e of entries) { const full = path.join(d, e.name); if (e.isDirectory()) { await walk(full); } else if (e.isFile() && e.name.endsWith('.js.map')) { const mapBase = e.name; // e.g. _0eaaij7._.js.map const chunkBase = e.name.slice(0, -'.map'.length); // e.g. _0eaaij7._.js // First write wins; collisions across dirs are rare for hashed chunk names. if (!byChunkBasename.has(chunkBase)) byChunkBasename.set(chunkBase, full); if (!byMapBasename.has(mapBase)) byMapBasename.set(mapBase, full); } } } await walk(dir); return { byChunkBasename, byMapBasename }; } // Extract the chunk basename from a frame url. Frame urls in the wild look like: // /app/.next/server/chunks/_0eaaij7._.js // file:///app/.next/server/chunks/27592.js // chunks/src_17njnbr._.js // 27592.js // We reduce to the trailing `.js` basename and look that up in the map index. function chunkBasenameFromUrl(url) { if (!url) return null; // strip query/hash and protocol let u = url.replace(/[?#].*$/, ''); if (u.startsWith('file://')) u = u.slice('file://'.length); const base = u.split('/').pop() || ''; if (!base.endsWith('.js')) return null; // ignore node internals / non-chunk urls if (u.startsWith('node:') || base === 'server.js') return null; return base; } /** * Build per-map SourceMapConsumer lazily and cache it. Returns a resolver fn that, * given (chunkBasename, line0, col0), returns the original frame or null. */ function makeResolver(SourceMapConsumer, mapIndex) { /** @type {Map>} */ const consumerCache = new Map(); /** @param {string} chunkBase */ function getConsumer(chunkBase) { if (consumerCache.has(chunkBase)) return consumerCache.get(chunkBase); const p = (async () => { const mapPath = mapIndex.byChunkBasename.get(chunkBase); if (!mapPath) return null; try { const raw = await readFile(mapPath, 'utf8'); const json = JSON.parse(raw); return await new SourceMapConsumer(json); } catch (err) { console.error(`warn: failed to load map for ${chunkBase}: ${err.message}`); return null; } })(); consumerCache.set(chunkBase, p); return p; } /** * @param {string} chunkBase * @param {number} line0 0-based line (V8) * @param {number} col0 0-based column (V8) */ async function resolve(chunkBase, line0, col0) { const consumer = await getConsumer(chunkBase); if (!consumer) return null; const pos = consumer.originalPositionFor({ line: (line0 ?? 0) + 1, // source-map is 1-based for lines column: col0 ?? 0, // 0-based for columns (matches V8) bias: SourceMapConsumer.LEAST_UPPER_BOUND, }); if (pos.source == null && pos.line == null) return null; return { source: pos.source, line: pos.line, column: pos.column, name: pos.name }; } function destroy() { for (const p of consumerCache.values()) { Promise.resolve(p).then((c) => c && c.destroy && c.destroy()); } } return { resolve, destroy }; } function formatFrame(original, callFrame) { const rawFn = callFrame.functionName || '(anonymous)'; if (original) { const fn = original.name || rawFn; const loc = original.source ? `${original.source}:${original.line ?? '?'}:${original.column ?? '?'}` : '(unknown source)'; return `${fn} @ ${loc}`; } const url = callFrame.url || ''; const base = url ? url.split('/').pop() : '(native)'; return `${rawFn} @ ${base}:${callFrame.lineNumber ?? '?'}:${callFrame.columnNumber ?? '?'} [unresolved]`; } async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.profile) { printHelp(); process.exit(1); } if ((args.image || args.imageSha) && args.mapsDir) { console.error('error: pass only one of --image / --image-sha or --maps, not both.'); process.exit(1); } if (args.resolve && !args.mapsDir && !args.image && !args.imageSha) { console.error( 'error: --image , --image-sha , or --maps is required (or pass --no-resolve).' ); process.exit(1); } if (args.mapsDir && !existsSync(args.mapsDir)) { console.error(`error: maps dir not found: ${args.mapsDir}`); process.exit(1); } const profileRaw = await readFile(args.profile, 'utf8'); /** @type {{ nodes: any[], samples: number[], timeDeltas: number[] }} */ const profile = JSON.parse(profileRaw); if (!Array.isArray(profile.nodes)) { console.error('error: not a V8 .cpuprofile (missing `nodes` array).'); process.exit(1); } // If --image was given, fetch that build's maps artifact into a temp dir and // resolve against it (cleaned up before we return). let mapsDir = args.mapsDir; let fetchCleanup = null; if (args.resolve && (args.image || args.imageSha)) { // --image-sha wins if both are passed: it auto-resolves the maps tag from the // commit sha, whereas --image expects an exact tag/ref. const ref = args.imageSha ? resolveTagBySha(args.imageSha, args.mapsRepo) : resolveImageRef(args.image, args.mapsRepo); if (args.imageSha) console.error(`info: resolved --image-sha ${args.imageSha} -> ${ref}`); const fetched = await fetchMapsImage(ref); mapsDir = fetched.dir; fetchCleanup = fetched.cleanup; } let resolver = null; // try/finally so a fetched maps temp dir (can be ~hundreds of MB) is always // cleaned up, even if resolution throws partway through. try { if (args.resolve) { const sm = await loadSourceMapModule(); const mapIndex = await indexMaps(mapsDir); if (mapIndex.byChunkBasename.size === 0) { console.error(`warn: no .js.map files found under ${mapsDir} — frames will be unresolved.`); } else { console.error(`info: indexed ${mapIndex.byChunkBasename.size} source maps under ${mapsDir}`); } resolver = makeResolver(sm.SourceMapConsumer, mapIndex); } // Resolve every node's callFrame once, cache by node id. /** @type {Map} */ const nodeInfo = new Map(); const byId = new Map(profile.nodes.map((n) => [n.id, n])); for (const node of profile.nodes) { const cf = node.callFrame || {}; let original = null; if (resolver) { const chunkBase = chunkBasenameFromUrl(cf.url); if (chunkBase) { original = await resolver.resolve(chunkBase, cf.lineNumber, cf.columnNumber); } } nodeInfo.set(node.id, { callFrame: cf, original, label: formatFrame(original, cf) }); } // --- self-time accounting (hottest leaves) --- // Each sample attributes its timeDelta to the sampled (leaf) node's self time. /** @type {Map} */ const selfTime = new Map(); const samples = profile.samples || []; const deltas = profile.timeDeltas || []; for (let i = 0; i < samples.length; i++) { const id = samples[i]; const dt = deltas[i] ?? 0; selfTime.set(id, (selfTime.get(id) || 0) + dt); } if (args.json) { const out = profile.nodes.map((n) => { const info = nodeInfo.get(n.id); const o = info.original; return { id: n.id, // Resolved location. `original` is { source, line, column, name } with no // `url` key, so add a `url` alias (== source) for consumers that key on // `.url`, plus a ready-made "fn @ source:line:col" label. original: o ? { ...o, url: o.source ?? null } : null, label: info.label, callFrame: n.callFrame, selfTimeMicros: selfTime.get(n.id) || 0, }; }); process.stdout.write(JSON.stringify({ nodes: out }, null, 2) + '\n'); if (resolver) resolver.destroy(); if (fetchCleanup) await fetchCleanup(); return; } // ancestry map (child id -> parent id) for printing stacks /** @type {Map} */ const parentOf = new Map(); for (const node of profile.nodes) { for (const child of node.children || []) parentOf.set(child, node.id); } /** @param {number} id */ function stackOf(id) { const out = []; let cur = id; const seen = new Set(); while (cur != null && !seen.has(cur)) { seen.add(cur); const info = nodeInfo.get(cur); if (info) out.push(info.label); cur = parentOf.get(cur); } return out; } const totalMicros = deltas.reduce((a, b) => a + (b || 0), 0); console.log(`\n=== CPU profile: ${path.basename(args.profile)} ===`); console.log(`samples=${samples.length} duration=${(totalMicros / 1000).toFixed(1)}ms\n`); console.log(`--- Top ${args.top} self-time functions ---`); const ranked = [...selfTime.entries()].sort((a, b) => b[1] - a[1]).slice(0, args.top); for (const [id, micros] of ranked) { const info = nodeInfo.get(id); const pct = totalMicros ? ((micros / totalMicros) * 100).toFixed(1) : '?'; console.log(` ${(micros / 1000).toFixed(1).padStart(8)}ms ${pct.padStart(5)}% ${info ? info.label : id}`); } if (args.block) { // Longest run of consecutive samples sharing the same leaf node id. let bestLeaf = null; let bestMicros = 0; let curLeaf = null; let curMicros = 0; for (let i = 0; i < samples.length; i++) { const id = samples[i]; const dt = deltas[i] ?? 0; if (id === curLeaf) { curMicros += dt; } else { curLeaf = id; curMicros = dt; } if (curMicros > bestMicros) { bestMicros = curMicros; bestLeaf = curLeaf; } } console.log(`\n--- Longest synchronous block ---`); if (bestLeaf != null) { console.log(` duration=${(bestMicros / 1000).toFixed(1)}ms leaf stack (innermost first):`); for (const frame of stackOf(bestLeaf)) console.log(` at ${frame}`); } else { console.log(' (no samples)'); } } } finally { if (resolver) resolver.destroy(); if (fetchCleanup) await fetchCleanup(); } } main().catch((err) => { console.error(err); process.exit(1); });