mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-14 18:01:20 +08:00
fix(cli): stop chrome-headless-shell download hangs from stalling forever
@puppeteer/browsers' download path has no AbortController/timeout anywhere in its chain, and its proxy support is inert (proxy-agent is only an optional peer dependency hyperframes doesn't install) — so on a network that requires a proxy, the download attempts a direct connection and hangs at "Downloading Chrome... 0%" forever instead of failing. Add a stall watchdog (resets on every progress tick, not a total-duration cap, so a slow-but-live download is unaffected) around the install() call, and name HTTP(S)_PROXY as the likely cause in the existing HYPERFRAMES_BROWSER_PATH remediation hint when one is configured. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -135,7 +135,9 @@ function installPuppeteerBrowsersMock(
|
|||||||
browserPlatform?: string;
|
browserPlatform?: string;
|
||||||
installedInHfCacheError?: Error;
|
installedInHfCacheError?: Error;
|
||||||
installResult?: { executablePath: string };
|
installResult?: { executablePath: string };
|
||||||
installImpl?: () => Promise<{ executablePath: string }>;
|
installImpl?: (installArgs: {
|
||||||
|
downloadProgressCallback?: (downloaded: number, total: number) => void;
|
||||||
|
}) => Promise<{ executablePath: string }>;
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
vi.doMock("@puppeteer/browsers", () => ({
|
vi.doMock("@puppeteer/browsers", () => ({
|
||||||
@@ -909,6 +911,121 @@ describe("installWithCorruptArchiveRecovery", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// PRINFRA-682: a chrome-headless-shell download that never gets a response
|
||||||
|
// (e.g. a network that only egresses through a proxy `@puppeteer/browsers`
|
||||||
|
// silently fails to use — see `proxyDownloadStallHint`) hangs at "Downloading
|
||||||
|
// Chrome... 0%" forever, with no timeout anywhere in the install() call
|
||||||
|
// chain. This guards the watchdog that makes that fail loud instead.
|
||||||
|
describe("withDownloadStallGuard", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves normally when the run completes before any stall", async () => {
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
const result = await withDownloadStallGuard(
|
||||||
|
async (onProgress) => {
|
||||||
|
onProgress(50, 100);
|
||||||
|
onProgress(100, 100);
|
||||||
|
return "done";
|
||||||
|
},
|
||||||
|
{ timeoutMs: 1_000 },
|
||||||
|
);
|
||||||
|
expect(result).toBe("done");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects if no progress arrives before the timeout (connection never responds)", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
|
||||||
|
// A run that never resolves and never reports progress — the "0% forever"
|
||||||
|
// symptom: no response ever comes back at all.
|
||||||
|
const guarded = withDownloadStallGuard(() => new Promise<void>(() => {}), {
|
||||||
|
timeoutMs: 1_000,
|
||||||
|
});
|
||||||
|
const outcome = guarded.then(
|
||||||
|
() => "resolved",
|
||||||
|
(err: unknown) => (err instanceof Error ? err.message : String(err)),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
|
||||||
|
await expect(outcome).resolves.toContain("stalled");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resets the watchdog on every progress tick, so a slow-but-live download never trips it", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
|
||||||
|
const result = withDownloadStallGuard(
|
||||||
|
async (onProgress) => {
|
||||||
|
for (let i = 1; i <= 5; i += 1) {
|
||||||
|
// Each tick arrives just under the timeout — only safe if the
|
||||||
|
// watchdog resets rather than measuring total elapsed duration.
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 900));
|
||||||
|
onProgress(i * 20, 100);
|
||||||
|
}
|
||||||
|
return "done";
|
||||||
|
},
|
||||||
|
{ timeoutMs: 1_000 },
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(900 * 5 + 10);
|
||||||
|
|
||||||
|
await expect(result).resolves.toBe("done");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still trips if progress stops arriving partway through (mid-download stall)", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
|
||||||
|
const guarded = withDownloadStallGuard(
|
||||||
|
(onProgress) =>
|
||||||
|
new Promise<void>((resolve) => {
|
||||||
|
onProgress(10, 100);
|
||||||
|
// ...then nothing further ever arrives — resolve() is never called.
|
||||||
|
void resolve;
|
||||||
|
}),
|
||||||
|
{ timeoutMs: 1_000 },
|
||||||
|
);
|
||||||
|
const outcome = guarded.then(
|
||||||
|
() => "resolved",
|
||||||
|
(err: unknown) => (err instanceof Error ? err.message : String(err)),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
|
||||||
|
await expect(outcome).resolves.toContain("stalled");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates a genuine rejection unchanged, not a stall error", async () => {
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
await expect(
|
||||||
|
withDownloadStallGuard(
|
||||||
|
async () => {
|
||||||
|
throw new Error("ENOTFOUND");
|
||||||
|
},
|
||||||
|
{ timeoutMs: 1_000 },
|
||||||
|
),
|
||||||
|
).rejects.toThrow("ENOTFOUND");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forwards progress ticks to the caller's onProgress callback", async () => {
|
||||||
|
const { withDownloadStallGuard } = await import("./manager.js");
|
||||||
|
const onProgress = vi.fn();
|
||||||
|
await withDownloadStallGuard(
|
||||||
|
async (tick) => {
|
||||||
|
tick(10, 100);
|
||||||
|
tick(100, 100);
|
||||||
|
return "done";
|
||||||
|
},
|
||||||
|
{ timeoutMs: 1_000, onProgress },
|
||||||
|
);
|
||||||
|
expect(onProgress).toHaveBeenNthCalledWith(1, 10, 100);
|
||||||
|
expect(onProgress).toHaveBeenNthCalledWith(2, 100, 100);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// Sibling failure mode to #2078 (SIGTRAP at launch): the field feedback in
|
// Sibling failure mode to #2078 (SIGTRAP at launch): the field feedback in
|
||||||
// #hyperframes-cli-feedback ts 1784055194.202169 (darwin/arm64, HF CLI 0.7.57)
|
// #hyperframes-cli-feedback ts 1784055194.202169 (darwin/arm64, HF CLI 0.7.57)
|
||||||
// hit `All providers failed for chrome-headless-shell 152.0.7928.2` at download
|
// hit `All providers failed for chrome-headless-shell 152.0.7928.2` at download
|
||||||
@@ -1007,6 +1124,67 @@ describe("downloadBrowser — install failure surfaces HYPERFRAMES_BROWSER_PATH
|
|||||||
expect((caught as Error).cause).toBe(originalError);
|
expect((caught as Error).cause).toBe(originalError);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
it("appends a proxy hint when HTTP(S)_PROXY is set", async () => {
|
||||||
|
process.env["HTTPS_PROXY"] = "http://127.0.0.1:18080";
|
||||||
|
installFsMocks({ existing: new Set([CACHE_ROOT]) });
|
||||||
|
installPuppeteerBrowsersMock({
|
||||||
|
installedInHfCache: [],
|
||||||
|
installImpl: async () => {
|
||||||
|
throw new Error("connect ETIMEDOUT");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const { ensureBrowser } = await import("./manager.js");
|
||||||
|
await expect(ensureBrowser()).rejects.toThrow(/HTTPS_PROXY.*is set/s);
|
||||||
|
|
||||||
|
delete process.env["HTTPS_PROXY"];
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits the proxy hint when no proxy env var is set", async () => {
|
||||||
|
installFsMocks({ existing: new Set([CACHE_ROOT]) });
|
||||||
|
installPuppeteerBrowsersMock({
|
||||||
|
installedInHfCache: [],
|
||||||
|
installImpl: async () => {
|
||||||
|
throw new Error("connect ETIMEDOUT");
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const { ensureBrowser } = await import("./manager.js");
|
||||||
|
let caught: unknown;
|
||||||
|
try {
|
||||||
|
await ensureBrowser();
|
||||||
|
} catch (err) {
|
||||||
|
caught = err;
|
||||||
|
}
|
||||||
|
expect((caught as Error).message).not.toContain("HTTPS_PROXY");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("PRINFRA-682: a download that never responds surfaces a stall error with the HYPERFRAMES_BROWSER_PATH hint instead of hanging forever", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
installFsMocks({ existing: new Set([CACHE_ROOT]) });
|
||||||
|
installPuppeteerBrowsersMock({
|
||||||
|
installedInHfCache: [],
|
||||||
|
// Mirrors the real bug: install() never resolves, rejects, or reports
|
||||||
|
// progress — exactly what a proxy-only network's silently-bypassed
|
||||||
|
// direct connection attempt looks like from the caller's side.
|
||||||
|
installImpl: () => new Promise(() => {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const { ensureBrowser } = await import("./manager.js");
|
||||||
|
const attempt = ensureBrowser();
|
||||||
|
const outcome = attempt.then(
|
||||||
|
() => "resolved",
|
||||||
|
(err: unknown) => (err instanceof Error ? err.message : String(err)),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(90_000);
|
||||||
|
|
||||||
|
const message = await outcome;
|
||||||
|
expect(message).toContain("Download stalled");
|
||||||
|
expect(message).toContain("HYPERFRAMES_BROWSER_PATH");
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// Regression guard for HF#2103: `hyperframes render` hung forever on macOS
|
// Regression guard for HF#2103: `hyperframes render` hung forever on macOS
|
||||||
|
|||||||
@@ -44,6 +44,21 @@ const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "chrome");
|
|||||||
// too or it silently picks system Chrome over a perfectly good headless-shell.
|
// too or it silently picks system Chrome over a perfectly good headless-shell.
|
||||||
const PUPPETEER_CACHE_DIR = join(homedir(), ".cache", "puppeteer", "chrome-headless-shell");
|
const PUPPETEER_CACHE_DIR = join(homedir(), ".cache", "puppeteer", "chrome-headless-shell");
|
||||||
|
|
||||||
|
// `@puppeteer/browsers`' downloadFile() (lib/httpUtil.js) opens a bare
|
||||||
|
// node:http(s) request with no AbortController/timeout anywhere in the chain
|
||||||
|
// (confirmed by source read of the resolved 3.x package) — a connection that
|
||||||
|
// never gets a response (e.g. a network that only egresses through an
|
||||||
|
// HTTP(S)_PROXY, which downloadFile's own proxy-agent support silently no-ops
|
||||||
|
// on: `proxy-agent` is only an *optional peer* dependency, so the package's
|
||||||
|
// dynamic import of it throws, is caught, and the request falls back to a
|
||||||
|
// bare agent that never reads proxy env vars) just hangs at "Downloading
|
||||||
|
// Chrome... 0%" forever. See PRINFRA-682. This is a stall watchdog, not a
|
||||||
|
// total-download-duration cap: it resets on every progress tick, so a
|
||||||
|
// slow-but-actually-progressing download (large binary over a slow line)
|
||||||
|
// never trips it — only a connection that stops producing any bytes for this
|
||||||
|
// long does.
|
||||||
|
const DOWNLOAD_STALL_TIMEOUT_MS = 45_000;
|
||||||
|
|
||||||
// `@puppeteer/browsers`' install() has no concurrency guard of its own — two
|
// `@puppeteer/browsers`' install() has no concurrency guard of its own — two
|
||||||
// CLI invocations that both miss the cache at the same time both extract into
|
// CLI invocations that both miss the cache at the same time both extract into
|
||||||
// the same target directory simultaneously. A killed/interrupted extraction
|
// the same target directory simultaneously. A killed/interrupted extraction
|
||||||
@@ -707,6 +722,62 @@ export async function installWithCorruptArchiveRecovery<T>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Race an install-style call against a "no progress" watchdog instead of a
|
||||||
|
* total-duration cap. The timer (re)arms on every progress tick — and once
|
||||||
|
* before the first one, since a connection that never gets a response never
|
||||||
|
* ticks at all — so a download that is merely slow keeps resetting it
|
||||||
|
* indefinitely, while one that has truly stalled (no bytes, ever, or bytes
|
||||||
|
* that stop arriving mid-stream) trips it after `timeoutMs` of silence.
|
||||||
|
*
|
||||||
|
* This cannot abort the underlying socket (`@puppeteer/browsers` exposes no
|
||||||
|
* `AbortSignal`), so a stalled connection lingers until the OS reclaims it —
|
||||||
|
* but the CLI's root exit handler calls `process.exit()` unconditionally once
|
||||||
|
* a command settles (see `cli.ts`'s `registerRootExitRequester`), so
|
||||||
|
* rejecting here still unblocks the process instead of leaving it hung.
|
||||||
|
*/
|
||||||
|
export function withDownloadStallGuard<T>(
|
||||||
|
run: (onProgress: (downloadedBytes: number, totalBytes: number) => void) => Promise<T>,
|
||||||
|
options: {
|
||||||
|
timeoutMs: number;
|
||||||
|
onProgress?: (downloadedBytes: number, totalBytes: number) => void;
|
||||||
|
},
|
||||||
|
): Promise<T> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
let timer: ReturnType<typeof setTimeout>;
|
||||||
|
const armTimer = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
timer = setTimeout(() => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
reject(
|
||||||
|
new Error(`Download stalled: no progress for ${Math.round(options.timeoutMs / 1000)}s.`),
|
||||||
|
);
|
||||||
|
}, options.timeoutMs);
|
||||||
|
};
|
||||||
|
armTimer();
|
||||||
|
run((downloadedBytes, totalBytes) => {
|
||||||
|
if (settled) return;
|
||||||
|
armTimer();
|
||||||
|
options.onProgress?.(downloadedBytes, totalBytes);
|
||||||
|
}).then(
|
||||||
|
(result) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(result);
|
||||||
|
},
|
||||||
|
(err: unknown) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(err);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* When `@puppeteer/browsers`' install() rejects for any reason the corrupt-
|
* When `@puppeteer/browsers`' install() rejects for any reason the corrupt-
|
||||||
* archive recovery path can't handle (all CDN providers rejected — the
|
* archive recovery path can't handle (all CDN providers rejected — the
|
||||||
@@ -731,6 +802,33 @@ function browserPathHintForPlatform(): string {
|
|||||||
return "/usr/bin/google-chrome";
|
return "/usr/bin/google-chrome";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `@puppeteer/browsers`' download request does attempt real proxy support
|
||||||
|
* (a dynamic import of the `proxy-agent` package), but that package is only
|
||||||
|
* an *optional peer* dependency that a normal hyperframes install never
|
||||||
|
* pulls in — so the import throws, is caught silently, and the request falls back
|
||||||
|
* to a bare node:http(s) agent that never reads HTTP_PROXY/HTTPS_PROXY on
|
||||||
|
* its own. On a network that only egresses through that proxy, the direct
|
||||||
|
* connection attempt doesn't fail fast; it stalls, which is what
|
||||||
|
* `withDownloadStallGuard` above catches. Naming the proxy as the likely
|
||||||
|
* cause (when one is actually configured) turns "download stalled, no idea
|
||||||
|
* why" into an actionable diagnosis instead of just repeating the same
|
||||||
|
* escape hatch with no context.
|
||||||
|
*/
|
||||||
|
function proxyDownloadStallHint(): string {
|
||||||
|
const proxyConfigured = ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"].some((key) =>
|
||||||
|
Boolean(process.env[key]?.trim()),
|
||||||
|
);
|
||||||
|
if (!proxyConfigured) return "";
|
||||||
|
return (
|
||||||
|
`\n\nHTTP_PROXY/HTTPS_PROXY is set in this environment, but chrome-headless-shell's ` +
|
||||||
|
`download does not honor it (the proxy support it depends on is an optional ` +
|
||||||
|
`dependency hyperframes does not install) — on a network that requires the proxy for ` +
|
||||||
|
`internet access, the download attempts a direct connection and stalls instead of ` +
|
||||||
|
`failing fast. Use HYPERFRAMES_BROWSER_PATH above to skip the download entirely.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function wrapDownloadFailureWithBrowserPathHint(cause: unknown): Error {
|
function wrapDownloadFailureWithBrowserPathHint(cause: unknown): Error {
|
||||||
const original = normalizeErrorMessage(cause);
|
const original = normalizeErrorMessage(cause);
|
||||||
const example = browserPathHintForPlatform();
|
const example = browserPathHintForPlatform();
|
||||||
@@ -741,7 +839,8 @@ function wrapDownloadFailureWithBrowserPathHint(cause: unknown): Error {
|
|||||||
`Then re-run your command. Any Chrome build works for the screenshot ` +
|
`Then re-run your command. Any Chrome build works for the screenshot ` +
|
||||||
`capture path; install a real chrome-headless-shell later if you need the ` +
|
`capture path; install a real chrome-headless-shell later if you need the ` +
|
||||||
`perf-optimized BeginFrame path. Alternatively, run inside the hyperframes ` +
|
`perf-optimized BeginFrame path. Alternatively, run inside the hyperframes ` +
|
||||||
`Docker image which ships a compatible headless-shell.`;
|
`Docker image which ships a compatible headless-shell.` +
|
||||||
|
proxyDownloadStallHint();
|
||||||
return new Error(message, { cause: cause instanceof Error ? cause : undefined });
|
return new Error(message, { cause: cause instanceof Error ? cause : undefined });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -758,13 +857,17 @@ async function downloadBrowser(options?: EnsureBrowserOptions): Promise<BrowserR
|
|||||||
}
|
}
|
||||||
|
|
||||||
const runInstall = () =>
|
const runInstall = () =>
|
||||||
install({
|
withDownloadStallGuard(
|
||||||
cacheDir: CACHE_DIR,
|
(onProgress) =>
|
||||||
browser: Browser.CHROMEHEADLESSSHELL,
|
install({
|
||||||
buildId: CHROME_VERSION,
|
cacheDir: CACHE_DIR,
|
||||||
platform,
|
browser: Browser.CHROMEHEADLESSSHELL,
|
||||||
downloadProgressCallback: options?.onProgress,
|
buildId: CHROME_VERSION,
|
||||||
});
|
platform,
|
||||||
|
downloadProgressCallback: onProgress,
|
||||||
|
}),
|
||||||
|
{ timeoutMs: DOWNLOAD_STALL_TIMEOUT_MS, onProgress: options?.onProgress },
|
||||||
|
);
|
||||||
|
|
||||||
let installed;
|
let installed;
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user