fix(gemini): fail typed on image failures and expand ~ in the output path (#2246)

* fix(gemini): fail typed on image failures and expand ~ in the output path

* fix(gemini): unwrap image bridge envelopes

* fix(gemini): clear transient image candidates

* fix(gemini): fail closed on malformed image probes

---------

Co-authored-by: jackwener <jakevingoo@gmail.com>
This commit is contained in:
Bo Liu
2026-08-08 18:36:49 +09:00
committed by GitHub
parent e169cc19b3
commit ce5f3762a1
4 changed files with 388 additions and 19 deletions
+213
View File
@@ -0,0 +1,213 @@
import { describe, expect, it, vi } from 'vitest';
import { JSDOM } from 'jsdom';
import { exportGeminiImages, getGeminiVisibleImageUrls, waitForGeminiImages } from './utils.js';
const CONVERSATION_URL = 'https://gemini.google.com/app/abc123';
const PNG_DATA_URL = 'data:image/png;base64,iVBORw0KGgo=';
/**
* Build a page whose evaluate runs the generated scripts against a JSDOM
* conversation, with the generation state supplied per poll.
*/
function createPageMock({ generating = [false] } = {}) {
const dom = new JSDOM('<main></main>', { url: CONVERSATION_URL, runScripts: 'outside-only' });
let index = 0;
return {
dom,
goto: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn((script) => {
if (script === 'window.location.href') return Promise.resolve(CONVERSATION_URL);
if (script.includes('stop response')) {
const value = generating[Math.min(index, generating.length - 1)] ?? false;
index += 1;
return typeof value === 'boolean' ? Promise.resolve(value) : Promise.reject(new Error('probe failed'));
}
return Promise.resolve(dom.window.eval(script));
}),
};
}
function wrapEvaluate(page) {
const original = page.evaluate;
page.evaluate = vi.fn(async (script) => ({ session: 'site:gemini', data: await original(script) }));
return page;
}
/** Mount an image whose decode state and layout box are controlled per case. */
function mountImage(dom, { src, naturalWidth, complete, box = 708 }) {
dom.window.document.querySelector('main').innerHTML = `<img src="${src}" alt="AI generated">`;
const img = dom.window.document.querySelector('main img');
Object.defineProperty(img, 'complete', { value: complete });
Object.defineProperty(img, 'naturalWidth', { value: naturalWidth });
Object.defineProperty(img, 'naturalHeight', { value: naturalWidth });
Object.defineProperty(img, 'width', { value: box });
Object.defineProperty(img, 'height', { value: box });
img.getBoundingClientRect = () => ({ width: box, height: box });
return img;
}
/** Answer the in-page fetch with a blob of the given type. */
function stubFetch(dom, { type }) {
dom.window.fetch = () => Promise.resolve({
ok: true,
blob: () => Promise.resolve({ type }),
});
dom.window.FileReader = class {
readAsDataURL(blob) {
this.result = blob.type === 'image/png' ? PNG_DATA_URL : 'data:text/html;base64,PGh0bWw+';
this.onloadend?.();
}
};
}
describe('gemini image detection', () => {
it('ignores an image that is still streaming its bytes', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/pending', naturalWidth: 0, complete: false });
await expect(getGeminiVisibleImageUrls(page)).resolves.toEqual([]);
});
it('ignores an image that finished loading without decodable bytes', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/broken', naturalWidth: 0, complete: true });
await expect(getGeminiVisibleImageUrls(page)).resolves.toEqual([]);
});
it('ignores an image that reports dimensions before it finished loading', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/partial', naturalWidth: 1024, complete: false });
await expect(getGeminiVisibleImageUrls(page)).resolves.toEqual([]);
});
it('accepts a decoded generated image', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/generated.png', naturalWidth: 1024, complete: true });
await expect(getGeminiVisibleImageUrls(page)).resolves.toEqual([
'https://lh3.googleusercontent.com/generated.png',
]);
});
it('unwraps Browser Bridge envelopes before returning image URLs', async () => {
const page = wrapEvaluate(createPageMock());
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/generated.png', naturalWidth: 1024, complete: true });
await expect(getGeminiVisibleImageUrls(page)).resolves.toEqual([
'https://lh3.googleusercontent.com/generated.png',
]);
});
});
describe('gemini image wait contract', () => {
it('does not settle on the page while Gemini is still generating', async () => {
const page = createPageMock({ generating: [true] });
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/mid.png', naturalWidth: 1024, complete: true });
await expect(waitForGeminiImages(page, [], 9)).rejects.toThrow(/gemini image timed out/);
});
it('fails closed when the generation probe stops answering', async () => {
const page = createPageMock({ generating: [true, 'unreadable'] });
await expect(waitForGeminiImages(page, [], 9)).rejects.toMatchObject({ code: 'COMMAND_EXEC' });
});
it('does not accept a visible image when the first generation probe fails', async () => {
const page = createPageMock({ generating: ['unreadable'] });
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/unknown-state.png', naturalWidth: 1024, complete: true });
await expect(waitForGeminiImages(page, [], 9)).rejects.toMatchObject({ code: 'COMMAND_EXEC' });
});
it('drops candidate images if the generation probe resumes as still generating', async () => {
const page = createPageMock({ generating: [false, true, true] });
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/transient.png', naturalWidth: 1024, complete: true });
await expect(waitForGeminiImages(page, [], 9)).rejects.toThrow(/gemini image timed out/);
});
it('returns the image once generation has stopped', async () => {
const page = createPageMock({ generating: [false] });
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/final.png', naturalWidth: 1024, complete: true });
await expect(waitForGeminiImages(page, [], 9)).resolves.toEqual([
'https://lh3.googleusercontent.com/final.png',
]);
});
it('unwraps Browser Bridge envelopes before reading the generation probe', async () => {
const page = wrapEvaluate(createPageMock({ generating: [true] }));
mountImage(page.dom, { src: 'https://lh3.googleusercontent.com/mid.png', naturalWidth: 1024, complete: true });
await expect(waitForGeminiImages(page, [], 9)).rejects.toThrow(/gemini image timed out/);
});
it('resolves empty when generation finished without producing an image', async () => {
const page = createPageMock({ generating: [false] });
await expect(waitForGeminiImages(page, [], 9)).resolves.toEqual([]);
});
});
describe('gemini image export', () => {
it('exports an image whose blob carries an image type', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/ok', naturalWidth: 1024, complete: true });
stubFetch(page.dom, { type: 'image/png' });
await expect(exportGeminiImages(page, ['blob:https://gemini.google.com/ok'])).resolves.toEqual([
expect.objectContaining({ dataUrl: PNG_DATA_URL, mimeType: 'image/png' }),
]);
});
it('unwraps Browser Bridge envelopes before returning exported assets', async () => {
const page = wrapEvaluate(createPageMock());
mountImage(page.dom, { src: 'blob:https://gemini.google.com/ok', naturalWidth: 1024, complete: true });
stubFetch(page.dom, { type: 'image/png' });
await expect(exportGeminiImages(page, ['blob:https://gemini.google.com/ok'])).resolves.toEqual([
expect.objectContaining({ dataUrl: PNG_DATA_URL, mimeType: 'image/png' }),
]);
});
it('fails closed when Browser Bridge returns malformed exported assets', async () => {
const page = {
goto: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn((script) => {
if (script === 'window.location.href')
return Promise.resolve(CONVERSATION_URL);
return Promise.resolve({
session: 'site:gemini',
data: [{ dataUrl: PNG_DATA_URL, mimeType: 'image/png' }],
});
}),
};
await expect(exportGeminiImages(page, ['blob:https://gemini.google.com/bad'])).rejects.toMatchObject({ code: 'COMMAND_EXEC' });
});
it('drops a response that came back as a page instead of an image', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/error', naturalWidth: 1024, complete: true });
stubFetch(page.dom, { type: 'text/html' });
await expect(exportGeminiImages(page, ['blob:https://gemini.google.com/error'])).resolves.toEqual([]);
});
it('does not redraw an image whose bytes never decoded', async () => {
const page = createPageMock();
mountImage(page.dom, { src: 'blob:https://gemini.google.com/blank', naturalWidth: 0, complete: false });
page.dom.window.fetch = () => Promise.reject(new Error('CORS'));
const drawImage = vi.fn();
page.dom.window.HTMLCanvasElement.prototype.getContext = () => ({ drawImage });
page.dom.window.HTMLCanvasElement.prototype.toDataURL = () => PNG_DATA_URL;
await expect(exportGeminiImages(page, ['blob:https://gemini.google.com/blank'])).resolves.toEqual([]);
expect(drawImage).not.toHaveBeenCalled();
});
});
+14 -4
View File
@@ -2,7 +2,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { cli, Strategy } from '@jackwener/opencli/registry';
import { saveBase64ToFile } from '@jackwener/opencli/utils';
import { ArgumentError } from '@jackwener/opencli/errors';
import { ArgumentError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
import { GEMINI_DOMAIN, exportGeminiImages, getGeminiVisibleImageUrls, sendGeminiMessage, startNewGeminiChat, waitForGeminiImages } from './utils.js';
function extFromMime(mime) {
if (mime.includes('png'))
@@ -23,6 +23,16 @@ function displayPath(filePath) {
const home = os.homedir();
return filePath.startsWith(home) ? `~${filePath.slice(home.length)}` : filePath;
}
export function resolveOutputDir(value) {
const raw = String(value || '').trim();
if (!raw)
return path.join(os.homedir(), 'tmp', 'gemini-images');
if (raw === '~')
return os.homedir();
if (raw.startsWith('~/'))
return path.join(os.homedir(), raw.slice(2));
return path.resolve(raw);
}
function buildImagePrompt(prompt, options) {
const extras = [];
if (options.ratio)
@@ -68,7 +78,7 @@ export const imageCommand = cli({
const prompt = kwargs.prompt;
const ratio = normalizeRatio(String(kwargs.rt ?? '1:1'));
const style = String(kwargs.st ?? '').trim();
const outputDir = kwargs.op || path.join(os.homedir(), 'tmp', 'gemini-images');
const outputDir = resolveOutputDir(kwargs.op);
const timeout = kwargs.timeout;
if (!Number.isInteger(timeout) || timeout < 1) {
throw new ArgumentError('--timeout must be a positive integer (seconds)');
@@ -87,14 +97,14 @@ export const imageCommand = cli({
const urls = await waitForGeminiImages(page, beforeUrls, timeout);
const link = await currentGeminiLink(page);
if (!urls.length) {
return [{ status: '⚠️ no-images', file: '📁 -', link: `🔗 ${link}` }];
throw new EmptyResultError('gemini image', `No generated image was detected. Open ${link} and check whether Gemini produced one.`);
}
if (skipDownload) {
return [{ status: '🎨 generated', file: '📁 -', link: `🔗 ${link}` }];
}
const assets = await exportGeminiImages(page, urls);
if (!assets.length) {
return [{ status: '⚠️ export-failed', file: '📁 -', link: `🔗 ${link}` }];
throw new CommandExecutionError('Failed to export the generated Gemini image', `Open ${link} and verify the image is visible, then retry.`);
}
const stamp = Date.now();
const results = [];
+82
View File
@@ -0,0 +1,82 @@
import * as os from 'node:os';
import * as path from 'node:path';
import { beforeEach, describe, expect, it, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
getGeminiVisibleImageUrls: vi.fn(),
sendGeminiMessage: vi.fn(),
startNewGeminiChat: vi.fn(),
waitForGeminiImages: vi.fn(),
exportGeminiImages: vi.fn(),
saveBase64ToFile: vi.fn(),
}));
vi.mock('./utils.js', () => ({
GEMINI_DOMAIN: 'gemini.google.com',
exportGeminiImages: mocks.exportGeminiImages,
getGeminiVisibleImageUrls: mocks.getGeminiVisibleImageUrls,
sendGeminiMessage: mocks.sendGeminiMessage,
startNewGeminiChat: mocks.startNewGeminiChat,
waitForGeminiImages: mocks.waitForGeminiImages,
}));
vi.mock('@jackwener/opencli/utils', () => ({
saveBase64ToFile: mocks.saveBase64ToFile,
}));
const { imageCommand, resolveOutputDir } = await import('./image.js');
const page = { evaluate: vi.fn().mockResolvedValue('https://gemini.google.com/app/abc123') };
const kwargs = { prompt: 'a red maple leaf', rt: '1:1', st: '', op: '/tmp/gemini-test', sd: false, timeout: 60 };
beforeEach(() => {
vi.clearAllMocks();
mocks.getGeminiVisibleImageUrls.mockResolvedValue([]);
mocks.startNewGeminiChat.mockResolvedValue(undefined);
mocks.sendGeminiMessage.mockResolvedValue(undefined);
});
describe('gemini image output directory', () => {
it('expands the home shorthand the default output path uses', () => {
expect(resolveOutputDir('~/tmp/gemini-images')).toBe(path.join(os.homedir(), 'tmp', 'gemini-images'));
expect(resolveOutputDir('~')).toBe(os.homedir());
});
it('resolves a relative path against the working directory', () => {
expect(resolveOutputDir('out/images')).toBe(path.resolve('out/images'));
expect(resolveOutputDir('')).toBe(path.join(os.homedir(), 'tmp', 'gemini-images'));
});
});
describe('gemini image command', () => {
it('typed-fails instead of returning a row when no image was produced', async () => {
mocks.waitForGeminiImages.mockResolvedValue([]);
await expect(imageCommand.func(page, kwargs)).rejects.toMatchObject({
code: 'EMPTY_RESULT',
exitCode: 66,
});
expect(mocks.saveBase64ToFile).not.toHaveBeenCalled();
});
it('typed-fails instead of returning a row when the export produced nothing', async () => {
mocks.waitForGeminiImages.mockResolvedValue(['https://lh3.googleusercontent.com/final.png']);
mocks.exportGeminiImages.mockResolvedValue([]);
await expect(imageCommand.func(page, kwargs)).rejects.toMatchObject({ code: 'COMMAND_EXEC' });
expect(mocks.saveBase64ToFile).not.toHaveBeenCalled();
});
it('saves the exported image and reports the file', async () => {
mocks.waitForGeminiImages.mockResolvedValue(['https://lh3.googleusercontent.com/final.png']);
mocks.exportGeminiImages.mockResolvedValue([
{ url: 'https://lh3.googleusercontent.com/final.png', dataUrl: 'data:image/png;base64,AAAA', mimeType: 'image/png', width: 1024, height: 1024 },
]);
const rows = await imageCommand.func(page, kwargs);
expect(rows).toHaveLength(1);
expect(rows[0].status).toBe('✅ saved');
expect(mocks.saveBase64ToFile).toHaveBeenCalledWith('AAAA', expect.stringContaining(path.join(path.resolve(kwargs.op), 'gemini_')));
});
});
+79 -15
View File
@@ -1,4 +1,4 @@
import { CommandExecutionError } from '@jackwener/opencli/errors';
import { CommandExecutionError, TimeoutError } from '@jackwener/opencli/errors';
export const GEMINI_DOMAIN = 'gemini.google.com';
export const GEMINI_APP_URL = 'https://gemini.google.com/app';
export const GEMINI_DEEP_RESEARCH_DEFAULT_TOOL_LABELS = ['Deep Research', 'Deep research', '\u6df1\u5ea6\u7814\u7a76'];
@@ -298,20 +298,24 @@ function getStateScript() {
})()
`;
}
function readGeminiSnapshotScript() {
return `
(() => {
${buildGeminiComposerLocatorScript()}
const composer = findComposer();
const composerText = composer?.textContent?.replace(/\\u00a0/g, ' ').trim() || '';
const isGenerating = !!Array.from(document.querySelectorAll('button, [role="button"]')).find((node) => {
/** Expression yielding whether the stop-response control is mounted. */
function isGeneratingExpression() {
return `!!Array.from(document.querySelectorAll('button, [role="button"]')).find((node) => {
const text = (node.textContent || '').trim().toLowerCase();
const aria = (node.getAttribute('aria-label') || '').trim().toLowerCase();
return text === 'stop response'
|| aria === 'stop response'
|| text === '停止回答'
|| aria === '停止回答';
});
})`;
}
function readGeminiSnapshotScript() {
return `
(() => {
${buildGeminiComposerLocatorScript()}
const composer = findComposer();
const composerText = composer?.textContent?.replace(/\\u00a0/g, ' ').trim() || '';
const isGenerating = ${isGeneratingExpression()};
const turns = ${getTurnsScript().trim()};
const transcriptLines = ${getTranscriptLinesScript().trim()};
@@ -2325,7 +2329,7 @@ export const __test__ = {
};
export async function getGeminiVisibleImageUrls(page) {
await ensureGeminiPage(page);
return await page.evaluate(`
const result = await page.evaluate(`
(() => {
const isVisible = (el) => {
if (!(el instanceof HTMLElement)) return false;
@@ -2346,6 +2350,10 @@ export async function getGeminiVisibleImageUrls(page) {
const height = img.naturalHeight || img.height || 0;
if (!src) continue;
if (alt.includes('avatar') || alt.includes('logo') || alt.includes('icon')) continue;
// A placeholder still streaming its bytes reports complete false and
// naturalWidth 0 while laying out at full size, so the size gate below
// would accept it and the export step would save a blank frame (#2245).
if (!img.complete || !img.naturalWidth) continue;
if (width < 128 && height < 128) continue;
if (seen.has(src)) continue;
seen.add(src);
@@ -2354,6 +2362,27 @@ export async function getGeminiVisibleImageUrls(page) {
return urls;
})()
`);
const urls = requireGeminiArrayResult(result, 'Gemini image detection');
if (!urls.every((url) => typeof url === 'string')) {
throw new CommandExecutionError('Gemini image detection returned a malformed result');
}
return urls;
}
/** Cheap generation probe: the snapshot read walks the whole transcript. */
export async function isGeminiGenerating(page) {
let result;
try {
result = await page.evaluate(`(() => ${isGeneratingExpression()})()`);
}
catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new CommandExecutionError('Gemini generation probe failed', message);
}
const value = unwrapGeminiEvaluateResult(result, 'Gemini generation probe');
if (typeof value !== 'boolean') {
throw new CommandExecutionError('Gemini generation probe returned a malformed result');
}
return value;
}
export async function waitForGeminiImages(page, beforeUrls, timeoutSeconds) {
const beforeSet = new Set(beforeUrls);
@@ -2361,8 +2390,23 @@ export async function waitForGeminiImages(page, beforeUrls, timeoutSeconds) {
const maxPolls = Math.max(1, Math.ceil(timeoutSeconds / pollIntervalSeconds));
let lastUrls = [];
let stableCount = 0;
let stillGenerating = false;
for (let index = 0; index < maxPolls; index += 1) {
await page.wait(index === 0 ? 2 : pollIntervalSeconds);
// The text waits already gate on this signal; without it the image wait
// can settle on whatever is on screen mid-generation (#2245). An
// unreadable probe keeps the last known state so the deadline still
// reports the right typed error.
const generating = await isGeminiGenerating(page);
if (generating !== null)
stillGenerating = generating;
if (generating === true) {
lastUrls = [];
stableCount = 0;
continue;
}
if (stillGenerating)
continue;
const urls = (await getGeminiVisibleImageUrls(page)).filter((url) => !beforeSet.has(url));
if (urls.length === 0)
continue;
@@ -2377,12 +2421,15 @@ export async function waitForGeminiImages(page, beforeUrls, timeoutSeconds) {
if (stableCount >= 2 || index === maxPolls - 1)
return lastUrls;
}
if (stillGenerating) {
throw new TimeoutError('gemini image', timeoutSeconds, 'Gemini was still generating at the deadline. Re-run with a higher --timeout.');
}
return lastUrls;
}
export async function exportGeminiImages(page, urls) {
await ensureGeminiPage(page);
const urlsJson = JSON.stringify(urls);
return await page.evaluate(`
const result = await page.evaluate(`
(async (targetUrls) => {
const blobToDataUrl = (blob) => new Promise((resolve, reject) => {
const reader = new FileReader();
@@ -2424,11 +2471,13 @@ export async function exportGeminiImages(page, urls) {
}
} catch {}
if (!dataUrl && img instanceof HTMLImageElement) {
// drawImage silently draws nothing for an image whose bytes have not
// decoded, so redrawing one would export a blank PNG (#2245).
if (!dataUrl && img instanceof HTMLImageElement && img.complete && img.naturalWidth) {
try {
const canvas = document.createElement('canvas');
canvas.width = img.naturalWidth || img.width;
canvas.height = img.naturalHeight || img.height;
canvas.width = img.naturalWidth;
canvas.height = img.naturalHeight;
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(img, 0, 0);
@@ -2438,7 +2487,10 @@ export async function exportGeminiImages(page, urls) {
} catch {}
}
if (dataUrl) {
// A failed fetch can still resolve, so require image bytes rather than
// any truthy string: an HTML error body reaches here as text/html, and
// an empty canvas export as a payload-less data URL (#2245).
if (/^data:[^;,]+;base64,[A-Za-z0-9+/]+=*$/.test(dataUrl) && String(mimeType).startsWith('image/')) {
results.push({ url: String(targetUrl), dataUrl, mimeType, width, height });
}
}
@@ -2446,6 +2498,18 @@ export async function exportGeminiImages(page, urls) {
return results;
})(${urlsJson})
`);
const assets = requireGeminiArrayResult(result, 'Gemini image export');
for (const asset of assets) {
if (!isObjectRecord(asset)
|| typeof asset.url !== 'string'
|| typeof asset.dataUrl !== 'string'
|| !/^data:[^;,]+;base64,[A-Za-z0-9+/]+=*$/.test(asset.dataUrl)
|| typeof asset.mimeType !== 'string'
|| !asset.mimeType.startsWith('image/')) {
throw new CommandExecutionError('Gemini image export returned a malformed result');
}
}
return assets;
}
export async function waitForGeminiResponse(page, baseline, promptText, timeoutSeconds) {
if (timeoutSeconds <= 0)