diff --git a/cli-manifest.json b/cli-manifest.json index 41781753..dc173906 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -15126,6 +15126,596 @@ "modulePath": "github-trending/repos.js", "sourceFile": "github-trending/repos.js" }, + { + "site": "gmail", + "name": "attachments", + "description": "List attachment metadata for a Gmail thread", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "thread", + "type": "string", + "required": true, + "positional": true, + "help": "Thread id from gmail search, legacy id, or Gmail thread URL" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "messageId", + "attachmentId", + "name", + "mimeType", + "size" + ], + "type": "js", + "modulePath": "gmail/attachments.js", + "sourceFile": "gmail/attachments.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "drafts", + "description": "List Gmail draft threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "important", + "description": "List important Gmail threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "inbox", + "description": "List Gmail inbox threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "labels", + "description": "List Gmail system and user labels with counts", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "id", + "name", + "type", + "unreadCount", + "totalCount" + ], + "type": "js", + "modulePath": "gmail/labels.js", + "sourceFile": "gmail/labels.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "login", + "description": "Open gmail login and wait until the browser session is authenticated", + "access": "write", + "domain": "mail.google.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "timeout", + "type": "int", + "default": 300, + "required": false, + "help": "Maximum seconds to wait for the user to finish login" + } + ], + "columns": [ + "status", + "logged_in", + "site", + "email", + "name" + ], + "type": "js", + "modulePath": "gmail/auth.js", + "sourceFile": "gmail/auth.js", + "navigateBefore": false, + "siteSession": "persistent", + "defaultWindowMode": "foreground" + }, + { + "site": "gmail", + "name": "search", + "description": "Search Gmail using Gmail search syntax", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "query", + "type": "string", + "required": true, + "positional": true, + "help": "Gmail search query" + }, + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "sent", + "description": "List sent Gmail threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "snoozed", + "description": "List snoozed Gmail threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "spam", + "description": "List Gmail spam threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "starred", + "description": "List starred Gmail threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "thread", + "description": "Read every message in a Gmail thread", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "thread", + "type": "string", + "required": true, + "positional": true, + "help": "Thread id from gmail search, legacy id, or Gmail thread URL" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "messageId", + "legacyMessageId", + "threadId", + "subject", + "from", + "fromName", + "to", + "cc", + "date", + "snippet", + "body", + "attachments" + ], + "type": "js", + "modulePath": "gmail/thread.js", + "sourceFile": "gmail/thread.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "trash", + "description": "List Gmail trash threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "unread", + "description": "List unread Gmail threads", + "access": "read", + "domain": "mail.google.com", + "strategy": "intercept", + "browser": true, + "args": [ + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Maximum threads to return (1-200)" + }, + { + "name": "account", + "type": "int", + "default": 0, + "required": false, + "help": "Gmail account index from the /mail/u// URL" + } + ], + "columns": [ + "threadId", + "subject", + "from", + "fromName", + "snippet", + "messageCount", + "unread", + "starred", + "date", + "labels" + ], + "type": "js", + "modulePath": "gmail/search.js", + "sourceFile": "gmail/search.js", + "navigateBefore": false, + "siteSession": "persistent" + }, + { + "site": "gmail", + "name": "whoami", + "description": "Show the current logged-in gmail account", + "access": "read", + "domain": "mail.google.com", + "strategy": "cookie", + "browser": true, + "args": [], + "columns": [ + "logged_in", + "site", + "email", + "name" + ], + "type": "js", + "modulePath": "gmail/auth.js", + "sourceFile": "gmail/auth.js", + "navigateBefore": false, + "siteSession": "persistent" + }, { "site": "google", "name": "images", diff --git a/clis/gmail/attachments.js b/clis/gmail/attachments.js new file mode 100644 index 00000000..ddb298bb --- /dev/null +++ b/clis/gmail/attachments.js @@ -0,0 +1,32 @@ +import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { fetchThread, parseAccount } from './utils.js'; + +cli({ + site: 'gmail', + name: 'attachments', + access: 'read', + description: 'List attachment metadata for a Gmail thread', + domain: 'mail.google.com', + strategy: Strategy.INTERCEPT, + browser: true, + navigateBefore: false, + siteSession: 'persistent', + args: [ + { name: 'thread', type: 'string', positional: true, required: true, help: 'Thread id from gmail search, legacy id, or Gmail thread URL' }, + { name: 'account', type: 'int', default: 0, help: 'Gmail account index from the /mail/u// URL' }, + ], + columns: ['messageId', 'attachmentId', 'name', 'mimeType', 'size'], + func: async (page, kwargs) => { + if (!page) throw new CommandExecutionError('Browser session required for gmail attachments'); + const messages = await fetchThread(page, kwargs.thread, parseAccount(kwargs.account)); + const rows = messages.flatMap((message) => message.attachments.map((attachment) => ({ + messageId: message.messageId, + ...attachment, + }))); + if (rows.length === 0) { + throw new EmptyResultError('gmail attachments', 'The Gmail thread has no attachments'); + } + return rows; + }, +}); diff --git a/clis/gmail/auth.js b/clis/gmail/auth.js new file mode 100644 index 00000000..7e181f67 --- /dev/null +++ b/clis/gmail/auth.js @@ -0,0 +1,48 @@ +import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { registerSiteAuthCommands } from '../_shared/site-auth.js'; +import { GMAIL_HOST, GMAIL_ORIGIN, unwrapBrowserResult } from './utils.js'; + +async function hasGoogleSession(page) { + const cookies = await page.getCookies({ url: GMAIL_ORIGIN }); + const names = new Set(cookies.map((cookie) => cookie.name)); + return names.has('SID') || names.has('__Secure-1PSID') || names.has('SAPISID'); +} + +async function verifyGmailIdentity(page) { + if (!await hasGoogleSession(page)) { + throw new AuthRequiredError(GMAIL_HOST, 'Google session cookies are missing'); + } + await page.goto(`${GMAIL_ORIGIN}/mail/u/0/#inbox`); + await page.sleep(2); + const result = unwrapBrowserResult(await page.evaluate(`(() => { + const account = Array.from(document.querySelectorAll('a[aria-label], button[aria-label]')) + .map((node) => String(node.getAttribute('aria-label') || '').trim()) + .find((label) => /@/.test(label) && /(google account|google 帐号|google 账号)/i.test(label)); + if (!account) { + const login = document.querySelector('a[href*="accounts.google.com/ServiceLogin"], input[type="email"]'); + return login + ? { kind: 'auth', detail: 'Gmail shows a Google sign-in surface' } + : { kind: 'shape', detail: 'Gmail account control was not found' }; + } + const email = account.match(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/i)?.[0] || ''; + const beforeEmail = email ? account.slice(0, account.indexOf(email)) : account; + const name = beforeEmail + .replace(/^.*?(?:google account|google 帐号|google 账号)\s*[::]?\s*/i, '') + .replace(/[,((]\s*$/, '') + .trim(); + return email ? { ok: true, email, name } : { kind: 'shape', detail: 'Gmail account control did not expose an email address' }; + })()`), 'identity probe'); + if (result?.kind === 'auth') throw new AuthRequiredError(GMAIL_HOST, result.detail); + if (!result?.ok) throw new CommandExecutionError(result?.detail || 'Gmail identity probe returned an unexpected result'); + return { email: result.email, name: result.name || null }; +} + +registerSiteAuthCommands({ + site: 'gmail', + domain: GMAIL_HOST, + loginUrl: 'https://accounts.google.com/ServiceLogin?service=mail&continue=https%3A%2F%2Fmail.google.com%2Fmail%2Fu%2F0%2F%23inbox', + columns: ['email', 'name'], + quickCheck: hasGoogleSession, + verify: verifyGmailIdentity, + poll: verifyGmailIdentity, +}); diff --git a/clis/gmail/commands.test.js b/clis/gmail/commands.test.js new file mode 100644 index 00000000..cb2ab6bf --- /dev/null +++ b/clis/gmail/commands.test.js @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import { getRegistry, Strategy } from '@jackwener/opencli/registry'; +import './auth.js'; +import './search.js'; +import './labels.js'; +import './thread.js'; +import './attachments.js'; + +describe('gmail command registry', () => { + it('registers the rich read surface with explicit strategies', () => { + for (const name of [ + 'search', 'inbox', 'unread', 'starred', 'sent', 'drafts', 'trash', 'spam', + 'snoozed', 'important', 'labels', 'thread', 'attachments', + ]) { + expect(getRegistry().get(`gmail/${name}`)).toMatchObject({ + access: 'read', + strategy: Strategy.INTERCEPT, + domain: 'mail.google.com', + }); + } + expect(getRegistry().get('gmail/whoami')).toMatchObject({ access: 'read', strategy: Strategy.COOKIE }); + expect(getRegistry().get('gmail/login')).toMatchObject({ access: 'write', strategy: Strategy.COOKIE }); + expect(getRegistry().has('gmail/draft')).toBe(false); + expect(getRegistry().has('gmail/send')).toBe(false); + expect(getRegistry().has('gmail/delete')).toBe(false); + }); +}); diff --git a/clis/gmail/labels.js b/clis/gmail/labels.js new file mode 100644 index 00000000..931c1663 --- /dev/null +++ b/clis/gmail/labels.js @@ -0,0 +1,23 @@ +import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { listLabels, parseAccount } from './utils.js'; + +cli({ + site: 'gmail', + name: 'labels', + access: 'read', + description: 'List Gmail system and user labels with counts', + domain: 'mail.google.com', + strategy: Strategy.INTERCEPT, + browser: true, + navigateBefore: false, + siteSession: 'persistent', + args: [ + { name: 'account', type: 'int', default: 0, help: 'Gmail account index from the /mail/u// URL' }, + ], + columns: ['id', 'name', 'type', 'unreadCount', 'totalCount'], + func: async (page, kwargs) => { + if (!page) throw new CommandExecutionError('Browser session required for gmail labels'); + return listLabels(page, parseAccount(kwargs.account)); + }, +}); diff --git a/clis/gmail/search.js b/clis/gmail/search.js new file mode 100644 index 00000000..74db05ac --- /dev/null +++ b/clis/gmail/search.js @@ -0,0 +1,69 @@ +import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { parseAccount, parseLimit, queryThreads } from './utils.js'; + +const THREAD_COLUMNS = [ + 'threadId', 'subject', 'from', 'fromName', 'snippet', 'messageCount', + 'unread', 'starred', 'date', 'labels', +]; + +function requirePage(page, command) { + if (!page) throw new CommandExecutionError(`Browser session required for gmail ${command}`); + return page; +} + +const commonArgs = [ + { name: 'limit', type: 'int', default: 20, help: 'Maximum threads to return (1-200)' }, + { name: 'account', type: 'int', default: 0, help: 'Gmail account index from the /mail/u// URL' }, +]; + +function registerThreadQuery(name, description, query) { + cli({ + site: 'gmail', + name, + access: 'read', + description, + domain: 'mail.google.com', + strategy: Strategy.INTERCEPT, + browser: true, + navigateBefore: false, + siteSession: 'persistent', + args: commonArgs, + columns: THREAD_COLUMNS, + func: async (page, kwargs) => queryThreads(requirePage(page, name), query, { + account: parseAccount(kwargs.account), + limit: parseLimit(kwargs.limit), + }), + }); +} + +cli({ + site: 'gmail', + name: 'search', + access: 'read', + description: 'Search Gmail using Gmail search syntax', + domain: 'mail.google.com', + strategy: Strategy.INTERCEPT, + browser: true, + navigateBefore: false, + siteSession: 'persistent', + args: [ + { name: 'query', type: 'string', positional: true, required: true, help: 'Gmail search query' }, + ...commonArgs, + ], + columns: THREAD_COLUMNS, + func: async (page, kwargs) => queryThreads(requirePage(page, 'search'), kwargs.query, { + account: parseAccount(kwargs.account), + limit: parseLimit(kwargs.limit), + }), +}); + +registerThreadQuery('inbox', 'List Gmail inbox threads', 'in:inbox'); +registerThreadQuery('unread', 'List unread Gmail threads', 'is:unread'); +registerThreadQuery('starred', 'List starred Gmail threads', 'is:starred'); +registerThreadQuery('sent', 'List sent Gmail threads', 'in:sent'); +registerThreadQuery('drafts', 'List Gmail draft threads', 'in:drafts'); +registerThreadQuery('trash', 'List Gmail trash threads', 'in:trash'); +registerThreadQuery('spam', 'List Gmail spam threads', 'in:spam'); +registerThreadQuery('snoozed', 'List snoozed Gmail threads', 'is:snoozed'); +registerThreadQuery('important', 'List important Gmail threads', 'is:important'); diff --git a/clis/gmail/thread.js b/clis/gmail/thread.js new file mode 100644 index 00000000..b2ea3dcb --- /dev/null +++ b/clis/gmail/thread.js @@ -0,0 +1,27 @@ +import { CommandExecutionError } from '@jackwener/opencli/errors'; +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { fetchThread, parseAccount } from './utils.js'; + +cli({ + site: 'gmail', + name: 'thread', + access: 'read', + description: 'Read every message in a Gmail thread', + domain: 'mail.google.com', + strategy: Strategy.INTERCEPT, + browser: true, + navigateBefore: false, + siteSession: 'persistent', + args: [ + { name: 'thread', type: 'string', positional: true, required: true, help: 'Thread id from gmail search, legacy id, or Gmail thread URL' }, + { name: 'account', type: 'int', default: 0, help: 'Gmail account index from the /mail/u// URL' }, + ], + columns: [ + 'messageId', 'legacyMessageId', 'threadId', 'subject', 'from', 'fromName', + 'to', 'cc', 'date', 'snippet', 'body', 'attachments', + ], + func: async (page, kwargs) => { + if (!page) throw new CommandExecutionError('Browser session required for gmail thread'); + return fetchThread(page, kwargs.thread, parseAccount(kwargs.account)); + }, +}); diff --git a/clis/gmail/utils.js b/clis/gmail/utils.js new file mode 100644 index 00000000..b27c6ea8 --- /dev/null +++ b/clis/gmail/utils.js @@ -0,0 +1,609 @@ +import { + ArgumentError, + AuthRequiredError, + CommandExecutionError, + EmptyResultError, + TimeoutError, +} from '@jackwener/opencli/errors'; + +export const GMAIL_ORIGIN = 'https://mail.google.com'; +export const GMAIL_HOST = 'mail.google.com'; +export const DEFAULT_LIMIT = 20; +export const MAX_LIMIT = 200; +const PAGE_SIZE = 50; +const CAPTURE_WAIT_SECONDS = 10; +const MAX_BODY_CHARS = 20_000; + +export function unwrapBrowserResult(value, label = 'browser probe') { + if (value && typeof value === 'object' && !Array.isArray(value) && 'session' in value) { + if (typeof value.session === 'string' && Object.prototype.hasOwnProperty.call(value, 'data')) { + return value.data; + } + throw new CommandExecutionError(`Gmail ${label} returned a malformed Browser Bridge envelope`); + } + return value; +} + +export function parseLimit(raw, fallback = DEFAULT_LIMIT, max = MAX_LIMIT) { + const value = raw ?? fallback; + const limit = Number(value); + if (!Number.isInteger(limit) || limit <= 0) { + throw new ArgumentError('limit must be a positive integer'); + } + if (limit > max) { + throw new ArgumentError(`limit must be <= ${max}`); + } + return limit; +} + +export function parseAccount(raw) { + const value = raw ?? 0; + const account = Number(value); + if (!Number.isInteger(account) || account < 0 || account > 20) { + throw new ArgumentError('account must be an integer between 0 and 20'); + } + return account; +} + +function cleanString(value) { + return typeof value === 'string' ? value.trim() : ''; +} + +function gmailDate(value, label) { + let timestamp = Number(value); + if (!Number.isFinite(timestamp) || timestamp <= 0) { + throw new CommandExecutionError(`Gmail ${label} had an invalid timestamp`); + } + if (timestamp < 10_000_000_000) timestamp *= 1000; + if (timestamp > 10_000_000_000_000) timestamp /= 1000; + const date = new Date(timestamp); + if (Number.isNaN(date.getTime())) { + throw new CommandExecutionError(`Gmail ${label} had an invalid timestamp`); + } + return date.toISOString(); +} + +function decodeEntities(value) { + return String(value || '') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'|'/gi, "'") + .replace(/&#(\d+);/g, (_match, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([\da-f]+);/gi, (_match, code) => String.fromCodePoint(Number.parseInt(code, 16))); +} + +export function htmlToText(value) { + return decodeEntities(String(value || '') + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(//gi, '\n') + .replace(/<\/(p|div|li|tr|h[1-6])>/gi, '\n') + .replace(/<[^>]+>/g, ' ')) + .replace(/[ \t]+/g, ' ') + .replace(/ *\n */g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +function parseJsonCapture(entry, operation) { + const status = Number(entry?.responseStatus || 0); + if (status === 401 || status === 403) { + throw new AuthRequiredError(GMAIL_HOST, `Gmail ${operation} returned HTTP ${status}`); + } + if (status !== 200) { + throw new CommandExecutionError(`Gmail ${operation} returned HTTP ${status || 'unknown'}`); + } + if (entry?.responseBodyTruncated === true) { + throw new CommandExecutionError(`Gmail ${operation} response exceeded the browser capture limit`); + } + const body = entry?.responsePreview; + if (Array.isArray(body)) return body; + if (typeof body !== 'string') { + throw new CommandExecutionError(`Gmail ${operation} response body was unavailable`); + } + try { + const parsed = JSON.parse(body.replace(/^\)\]\}'\s*/, '')); + if (!Array.isArray(parsed)) throw new Error('not an array'); + return parsed; + } catch { + throw new CommandExecutionError(`Gmail ${operation} returned malformed JSON`); + } +} + +function addressRef(value) { + if (!Array.isArray(value)) return null; + const address = cleanString(value[1]); + if (!address.includes('@')) return null; + return { address, name: cleanString(value[2]) || null }; +} + +function senderFromSummary(message) { + return addressRef(Array.isArray(message) ? message[1] : null); +} + +function labelIdsFromMessages(messages) { + return [...new Set((Array.isArray(messages) ? messages : []) + .flatMap((message) => Array.isArray(message?.[10]) ? message[10] : []) + .filter((label) => typeof label === 'string' && label.startsWith('^')))]; +} + +export function parseBatchView(body) { + if (!Array.isArray(body) || body.length !== 19) { + throw new CommandExecutionError('Gmail batch-view response had an unexpected shape'); + } + const rows = Array.isArray(body[2]) ? body[2] : []; + return rows.map((wrapper, index) => { + const record = Array.isArray(wrapper?.[0]) ? wrapper[0] : null; + if (!record || record.length < 5) { + throw new CommandExecutionError(`Gmail batch-view returned a malformed thread at index ${index}`); + } + const threadId = cleanString(record[3]).replace(/^#/, ''); + const messages = Array.isArray(record[4]) ? record[4] : []; + const latest = messages.at(-1); + const sender = senderFromSummary(latest); + if (!threadId) throw new CommandExecutionError(`Gmail batch-view returned a thread without an id at index ${index}`); + const labels = labelIdsFromMessages(messages); + return { + threadId, + subject: cleanString(record[0]) || '(no subject)', + from: sender?.address || null, + fromName: sender?.name || null, + snippet: cleanString(record[1]) || null, + messageCount: messages.length, + unread: labels.includes('^u'), + starred: labels.includes('^t'), + date: gmailDate(record[2], `thread ${threadId}`), + labels, + }; + }); +} + +function labelCounts(value) { + const result = new Map(); + const rows = Array.isArray(value?.[0]) ? value[0] : []; + for (const row of rows) { + const id = cleanString(row?.[0]); + if (!id) continue; + result.set(id, { + unread: Number.isFinite(Number(row?.[1])) ? Number(row[1]) : null, + total: Number.isFinite(Number(row?.[2])) ? Number(row[2]) : null, + }); + } + return result; +} + +export function parseLabels(body) { + if (!Array.isArray(body) || body.length !== 19) { + throw new CommandExecutionError('Gmail batch-view response had an unexpected shape'); + } + const counts = labelCounts(body[6]); + const rows = Array.isArray(body[1]) ? body[1] : []; + return rows.map((wrapper, index) => { + const record = Array.isArray(wrapper?.[0]) ? wrapper[0] : null; + const id = cleanString(record?.[0]); + if (!id) throw new CommandExecutionError(`Gmail returned a malformed label at index ${index}`); + const count = counts.get(id) || {}; + return { + id, + name: cleanString(record?.[1]) || id, + type: id.startsWith('^x_') ? 'user' : 'system', + unreadCount: count.unread ?? null, + totalCount: count.total ?? null, + }; + }); +} + +function senderFromRecord(record) { + const card = Array.isArray(record?.[10]) ? record[10] : null; + const address = cleanString(card?.[16]); + if (!address.includes('@')) return null; + return { address, name: cleanString(card?.[14]) || null }; +} + +function addressList(value) { + return (Array.isArray(value) ? value : []).map(addressRef).filter(Boolean); +} + +function messageBody(record) { + const block = Array.isArray(record?.[5]) ? record[5] : []; + const plain = cleanString(Array.isArray(block[4]) ? block[4][6] : ''); + let html = ''; + for (const part of Array.isArray(block[1]) ? block[1] : []) { + html += cleanString(Array.isArray(part?.[2]) ? part[2][1] : ''); + } + const text = plain && !/[.#@][\w-]+\s*\{[^}]+\}/.test(plain.slice(0, 500)) + ? plain + : (htmlToText(html) || plain || cleanString(record?.[6])); + const clipped = Number(block[2]) === 1 ? '\n\n[message clipped by Gmail]' : ''; + return `${text.slice(0, MAX_BODY_CHARS)}${clipped}`.trim(); +} + +function parseAttachments(record) { + const out = []; + for (const wrapper of Array.isArray(record?.[13]) ? record[13] : []) { + const node = Array.isArray(wrapper?.[0]) ? wrapper[0] : null; + const data = Array.isArray(node?.[3]) ? node[3] : null; + const attachmentId = cleanString(node?.[1]); + if (!data || !attachmentId) continue; + out.push({ + attachmentId, + name: cleanString(data[2]) || null, + mimeType: cleanString(data[3]) || null, + size: Number.isFinite(Number(data[4])) ? Number(data[4]) : null, + }); + } + return out; +} + +export function parseFetchData(body) { + if (!Array.isArray(body) || !Array.isArray(body[1])) { + throw new CommandExecutionError('Gmail fetch-data response had an unexpected shape'); + } + const messages = []; + for (const threadWrapper of body[1]) { + const threadId = cleanString(threadWrapper?.[0]).replace(/^#/, ''); + const rows = Array.isArray(threadWrapper?.[2]) ? threadWrapper[2] : []; + for (const wrapper of rows) { + const messageId = cleanString(wrapper?.[0]).replace(/^#/, ''); + const record = Array.isArray(wrapper?.[1]) ? wrapper[1] : null; + if (!threadId || !messageId || !record) { + throw new CommandExecutionError('Gmail fetch-data returned a malformed message'); + } + const from = senderFromRecord(record); + const attachments = parseAttachments(record); + messages.push({ + messageId, + legacyMessageId: cleanString(record[34]) || null, + threadId, + subject: cleanString(record[4]) || '(no subject)', + from: from?.address || null, + fromName: from?.name || null, + to: addressList(record[0]).map((item) => item.address).join(', ') || null, + cc: addressList(record[1]).map((item) => item.address).join(', ') || null, + date: gmailDate(record[16], `message ${messageId}`), + snippet: cleanString(record[6]) || null, + body: messageBody(record) || null, + attachments, + }); + } + } + return messages; +} + +async function ensureGmailReady(page, account, operation) { + const currentUrl = typeof page.getCurrentUrl === 'function' ? await page.getCurrentUrl() : null; + if (!currentUrl?.startsWith(`${GMAIL_ORIGIN}/mail/u/${account}/`)) { + await page.goto(`${GMAIL_ORIGIN}/mail/u/${account}/#inbox`); + } + for (let attempt = 0; attempt < 60; attempt += 1) { + const ready = unwrapBrowserResult(await page.evaluate(`() => !!document.querySelector('input[name="q"]')`), `${operation} readiness probe`); + if (ready === true) return; + await page.sleep(0.5); + } + throw new TimeoutError(`Gmail ${operation} page`, 30, 'The Gmail search surface did not become ready. Reload Gmail in the browser and retry.'); +} + +async function installGmailCapture(page, account, endpoint, operation) { + if ( + typeof page?.startNetworkCapture !== 'function' + || typeof page?.readNetworkCapture !== 'function' + ) { + throw new CommandExecutionError(`Gmail ${operation} requires browser response interception`); + } + if (!await page.startNetworkCapture(`/sync/u/${account}/i/${endpoint}`)) { + throw new CommandExecutionError(`Gmail ${operation} could not start browser response interception`); + } + await page.readNetworkCapture(); +} + +async function waitGmailCaptures(page, endpoint, operation, timeoutSeconds = CAPTURE_WAIT_SECONDS) { + const deadline = Date.now() + timeoutSeconds * 1000; + let bodylessCaptureObserved = false; + // The bridge capture queue is request-oriented: reading it while a response + // is still in flight drains that request before its status/body arrive. + // Give Gmail's own action time to settle before the first read. + await page.sleep(endpoint === 'fd' ? 1 : 3); + while (Date.now() < deadline) { + const entries = await page.readNetworkCapture(); + const endpointEntries = (Array.isArray(entries) ? entries : []) + .filter((entry) => String(entry?.url || '').includes(`/i/${endpoint}`)); + if (endpointEntries.some((entry) => ( + typeof entry?.responsePreview !== 'string' + && entry?.responseBodyTruncated !== true + ))) { + bodylessCaptureObserved = true; + } + const matches = endpointEntries.filter((entry) => ( + typeof entry?.responsePreview === 'string' || entry?.responseBodyTruncated === true + )); + if (matches.length > 0) { + await page.sleep(1); + const settled = await page.readNetworkCapture(); + const settledEndpointEntries = (Array.isArray(settled) ? settled : []) + .filter((entry) => String(entry?.url || '').includes(`/i/${endpoint}`)); + if (settledEndpointEntries.some((entry) => ( + typeof entry?.responsePreview !== 'string' + && entry?.responseBodyTruncated !== true + ))) { + bodylessCaptureObserved = true; + } + matches.push(...settledEndpointEntries.filter((entry) => ( + typeof entry?.responsePreview === 'string' || entry?.responseBodyTruncated === true + ))); + if (bodylessCaptureObserved) { + throw new CommandExecutionError( + `Gmail ${operation} capture included a response without a body; refusing possibly partial results`, + ); + } + return matches.map((entry) => parseJsonCapture(entry, operation)); + } + await page.sleep(0.25); + } + if (bodylessCaptureObserved) { + throw new CommandExecutionError( + `Gmail ${operation} capture lost a response body; refusing possibly partial results`, + ); + } + throw new TimeoutError(`Gmail ${operation} capture`, timeoutSeconds, `No /${endpoint} response was observed after the Gmail action.`); +} + +async function renderedLabels(page, account) { + await page.goto(`${GMAIL_ORIGIN}/mail/u/${account}/#settings/labels`); + await page.sleep(2); + const rows = unwrapBrowserResult(await page.evaluate(`() => { + const routes = new RegExp('^(?:#(?:inbox|starred|snoozed|sent|drafts|important|spam|trash)|#label/)'); + const systemNames = { + inbox: 'Inbox', starred: 'Starred', snoozed: 'Snoozed', sent: 'Sent', + drafts: 'Drafts', important: 'Important', spam: 'Spam', trash: 'Trash', + scheduled: 'Scheduled', all: 'All Mail', + 'category/purchases': 'Purchases', 'category/social': 'Social', + 'category/updates': 'Updates', 'category/forums': 'Forums', + 'category/promotions': 'Promotions', + }; + const seen = new Set(); + const result = []; + const add = (id, name, type, unreadCount = null) => { + const key = String(id || '').toLowerCase(); + if (!key || !name || seen.has(key)) return; + seen.add(key); + result.push({ id, name, type, unreadCount, totalCount: null }); + }; + for (const link of document.querySelectorAll('a[href]')) { + const href = String(link.getAttribute('href') || ''); + if (!routes.test(href)) continue; + const raw = String(link.getAttribute('aria-label') || link.getAttribute('title') || link.textContent || '').trim(); + const route = decodeURIComponent(href.slice(1)); + const systemName = systemNames[route]; + const name = systemName || (route.startsWith('label/') ? route.slice(6) : raw.replace(/\\s+\\d[\\d,]*\\s*$/, '').trim()); + if (!name) continue; + const count = raw.match(/(?:^|\\s)(\\d[\\d,]*)\\s*$/)?.[1]; + add(route, name, systemName ? 'system' : 'user', count ? Number(count.replace(/,/g, '')) : null); + } + const headings = new Set([ + 'labels', 'label', 'system labels', 'categories', 'create new label', + 'show in label list', 'show in message list', 'actions', + ]); + for (const row of document.querySelectorAll('[role="main"] tr, main tr')) { + const cell = row.querySelector('td'); + const name = String(cell?.innerText || cell?.textContent || '').split('\\n').map((text) => text.trim()).find(Boolean) || ''; + if (!name || headings.has(name.toLowerCase()) || /^note:/i.test(name) || name.length > 120) continue; + const systemKey = Object.keys(systemNames).find((key) => systemNames[key].toLowerCase() === name.toLowerCase()); + add(systemKey || 'label/' + name, name, systemKey ? 'system' : 'user'); + } + return result; + }`), 'rendered labels'); + return Array.isArray(rows) ? rows : []; +} + +async function renderedThread(page, target) { + const rows = unwrapBrowserResult(await page.evaluate(`() => { + const subject = String(document.querySelector('h2[data-thread-perm-id], h2.hP')?.textContent || '').trim() || '(no subject)'; + const seen = new Set(); + const result = []; + for (const node of document.querySelectorAll('[data-message-id]')) { + const messageId = String(node.getAttribute('data-message-id') || '').trim(); + if (!messageId || seen.has(messageId)) continue; + const bodyNode = node.querySelector('.a3s.aiL, .a3s'); + if (!bodyNode) continue; + seen.add(messageId); + const sender = node.querySelector('.gD[email], span[email]'); + const dateNode = node.querySelector('.g3[title], [data-tooltip*="202"], [title]'); + const attachments = []; + for (const attachment of node.querySelectorAll('.aQH, [download_url]')) { + const name = String(attachment.querySelector('.aV3')?.textContent || attachment.getAttribute('aria-label') || '').trim(); + if (!name || attachments.some((item) => item.name === name)) continue; + attachments.push({ + attachmentId: String(attachment.getAttribute('data-attachment-id') || '') || null, + name, + mimeType: null, + size: null, + }); + } + result.push({ + messageId, + legacyMessageId: String(node.getAttribute('data-legacy-message-id') || '') || null, + subject, + from: String(sender?.getAttribute('email') || '').trim() || null, + fromName: String(sender?.getAttribute('name') || sender?.textContent || '').trim() || null, + to: null, + cc: null, + dateText: String(dateNode?.getAttribute('title') || dateNode?.textContent || '').trim(), + snippet: null, + body: String(bodyNode.innerText || bodyNode.textContent || '').trim(), + attachments, + }); + } + return result; + }`), 'rendered thread'); + if (!Array.isArray(rows)) return []; + return rows.map((row) => { + const parsedDate = row.dateText ? new Date(row.dateText) : null; + return { + ...row, + threadId: cleanString(target).replace(/^#/, ''), + date: parsedDate && !Number.isNaN(parsedDate.getTime()) ? parsedDate.toISOString() : null, + body: cleanString(row.body).slice(0, MAX_BODY_CHARS) || null, + }; + }).map(({ dateText: _dateText, ...row }) => row); +} + +async function submitSearch(page, query, operation) { + const prepared = unwrapBrowserResult(await page.evaluate(`() => { + const field = document.querySelector('input[name="q"]'); + if (!field) return false; + field.focus(); + field.select(); + return true; + }`), `${operation} search preparation`); + if (prepared !== true) throw new CommandExecutionError(`Gmail ${operation} could not find the search input`); + if (typeof page.nativeType !== 'function') { + throw new CommandExecutionError(`Gmail ${operation} requires native browser input`); + } + await page.nativeType(query); + const actual = unwrapBrowserResult(await page.evaluate(`() => document.querySelector('input[name="q"]')?.value || ''`), `${operation} search input verification`); + if (actual !== query) throw new CommandExecutionError(`Gmail ${operation} could not set the search query exactly`); + if (typeof page.cdp === 'function') { + const key = { key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 }; + await page.cdp('Input.dispatchKeyEvent', { type: 'rawKeyDown', ...key }); + await page.cdp('Input.dispatchKeyEvent', { type: 'keyUp', ...key }); + } else if (typeof page.nativeKeyPress === 'function') { + await page.nativeKeyPress('Enter'); + } else { + throw new CommandExecutionError(`Gmail ${operation} requires native browser keyboard input`); + } +} + +export async function queryThreads(page, query, { account = 0, limit = DEFAULT_LIMIT } = {}) { + const normalizedQuery = cleanString(query); + if (!normalizedQuery) throw new ArgumentError('Gmail search query cannot be empty'); + await ensureGmailReady(page, account, 'thread list'); + await installGmailCapture(page, account, 'bv', 'thread list'); + + const rows = []; + const seen = new Set(); + const pages = Math.ceil(limit / PAGE_SIZE); + for (let pageNumber = 1; pageNumber <= pages; pageNumber += 1) { + if (pageNumber === 1) { + await submitSearch(page, normalizedQuery, 'thread list'); + } else { + const target = unwrapBrowserResult(await page.evaluate(`() => { + const labels = /(older|较旧|較舊|下一页|下一頁)/i; + const button = Array.from(document.querySelectorAll('button, [role="button"]')).find((node) => { + const value = [node.getAttribute('aria-label'), node.getAttribute('data-tooltip'), node.getAttribute('title')] + .filter(Boolean).join(' '); + const rect = node.getBoundingClientRect(); + return labels.test(value) && node.getAttribute('aria-disabled') !== 'true' && rect.width > 0 && rect.height > 0; + }); + if (!button) return { found: false }; + const rect = button.getBoundingClientRect(); + return { found: true, x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + }`), 'thread pagination'); + if (!target?.found) break; + if (typeof page.nativeClick === 'function') await page.nativeClick(target.x, target.y); + else { + const clicked = unwrapBrowserResult(await page.evaluate(`() => { + const labels = /(older|较旧|較舊|下一页|下一頁)/i; + const button = Array.from(document.querySelectorAll('button, [role="button"]')).find((node) => { + const value = [node.getAttribute('aria-label'), node.getAttribute('data-tooltip'), node.getAttribute('title')] + .filter(Boolean).join(' '); + const rect = node.getBoundingClientRect(); + return labels.test(value) && node.getAttribute('aria-disabled') !== 'true' && rect.width > 0 && rect.height > 0; + }); + if (!button) return false; + button.click(); + return true; + }`), 'thread pagination click'); + if (clicked !== true) break; + } + } + const bodies = await waitGmailCaptures( + page, + 'bv', + pageNumber === 1 ? 'thread list' : `thread pagination page ${pageNumber}`, + ); + const pageRows = bodies.flatMap(parseBatchView); + let added = 0; + for (const row of pageRows) { + if (seen.has(row.threadId)) continue; + seen.add(row.threadId); + rows.push(row); + added += 1; + if (rows.length >= limit) return rows; + } + if (pageNumber > 1 && added === 0) { + throw new CommandExecutionError(`Gmail thread pagination page ${pageNumber} repeated an earlier response; refusing partial results`); + } + if (pageRows.length < PAGE_SIZE) break; + } + if (rows.length === 0) { + throw new EmptyResultError('gmail search', `No threads matched "${normalizedQuery}"`); + } + return rows.slice(0, limit); +} + +export async function listLabels(page, account = 0) { + await ensureGmailReady(page, account, 'labels'); + await installGmailCapture(page, account, 'bv', 'labels'); + await submitSearch(page, 'in:anywhere', 'labels'); + const bodies = await waitGmailCaptures(page, 'bv', 'labels'); + const labels = bodies.flatMap(parseLabels); + const fallback = labels.length === 0 ? await renderedLabels(page, account) : []; + const unique = [...new Map([...labels, ...fallback].map((row) => [row.id, row])).values()]; + if (unique.length === 0) throw new EmptyResultError('gmail labels', 'Gmail returned no labels'); + return unique; +} + +export function legacyThreadId(value) { + const raw = cleanString(value); + const fromUrl = raw.match(/\/(?:[a-f\d]{10,})$/i)?.[0]?.slice(1); + if (fromUrl) return fromUrl.toLowerCase(); + if (/^[a-f\d]{10,}$/i.test(raw)) return raw.toLowerCase(); + const sync = raw.replace(/^#/, '').match(/^thread-f:(\d+)$/); + if (sync) return BigInt(sync[1]).toString(16); + throw new ArgumentError('thread must be a Gmail thread id from `gmail search` or a Gmail thread URL'); +} + +export async function fetchThread(page, target, account = 0) { + const legacyId = legacyThreadId(target); + await ensureGmailReady(page, account, 'thread'); + await installGmailCapture(page, account, 'fd', 'thread'); + const targetState = unwrapBrowserResult(await page.evaluate(`async () => { + const row = document.querySelector('[data-legacy-thread-id="${legacyId}"]'); + if (!row) return { found: false }; + const clickable = row.querySelector('.y6, .bog, td:nth-child(5)') || row; + const before = location.href; + clickable.click(); + await new Promise((resolve) => setTimeout(resolve, 150)); + const rect = clickable.getBoundingClientRect(); + return { found: true, changed: location.href !== before, x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }; + }`), 'thread row lookup'); + if (targetState?.found && !targetState.changed && typeof page.nativeClick === 'function') { + await page.nativeClick(targetState.x, targetState.y); + } else if (!targetState?.found) { + const navigated = unwrapBrowserResult(await page.evaluate(`() => { + const target = ${JSON.stringify(`#all/${legacyId}`)}; + if (window.location.hash === target) window.location.hash = '#inbox'; + setTimeout(() => { window.location.hash = target; }, 50); + return true; + }`), 'thread navigation'); + if (navigated !== true) throw new CommandExecutionError('Gmail thread navigation failed'); + } + let messages = []; + try { + const bodies = await waitGmailCaptures(page, 'fd', 'thread', 3); + messages = bodies.flatMap(parseFetchData); + } catch (error) { + if (!(error instanceof TimeoutError)) throw error; + await page.sleep(0.5); + messages = await renderedThread(page, target); + if (messages.length === 0) throw error; + } + const unique = [...new Map(messages.map((message) => [message.messageId, message])).values()]; + if (unique.length === 0) { + throw new EmptyResultError('gmail thread', `No messages found for thread ${target}`); + } + return unique; +} diff --git a/clis/gmail/utils.test.js b/clis/gmail/utils.test.js new file mode 100644 index 00000000..b2cd2458 --- /dev/null +++ b/clis/gmail/utils.test.js @@ -0,0 +1,291 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + ArgumentError, + AuthRequiredError, + CommandExecutionError, + EmptyResultError, + TimeoutError, +} from '@jackwener/opencli/errors'; +import { + fetchThread, + htmlToText, + legacyThreadId, + listLabels, + parseAccount, + parseBatchView, + parseFetchData, + parseLabels, + parseLimit, + queryThreads, +} from './utils.js'; + +function batchViewFixture({ threads = true, labels = true } = {}) { + const body = Array(19).fill(null); + if (threads) { + const summary = []; + summary[1] = [null, 'sender@example.com', 'Sender Name']; + summary[10] = ['^u', '^t']; + body[2] = [[[ + 'Fixture subject', + 'Fixture snippet', + 1_700_000_000_000, + 'thread-f:1234567890123456789', + [summary], + ], null]]; + } else { + body[2] = []; + } + if (labels) { + body[1] = [ + [[ '^i', 'Inbox' ], null], + [[ '^x_project', 'Project' ], null], + ]; + body[6] = [[['^i', 2, 10], ['^x_project', 1, 3]]]; + } else { + body[1] = []; + body[6] = [[]]; + } + return body; +} + +function fetchDataFixture() { + const body = []; + const record = Array(35).fill(null); + record[0] = [[null, 'to@example.com', 'To']]; + record[1] = [[null, 'cc@example.com', 'Cc']]; + record[4] = 'Fixture subject'; + record[5] = [null, [[null, null, [null, '

Hello world

']]], 0, null, [null, null, null, null, null, null, 'Hello world']]; + record[6] = 'Fixture snippet'; + const sender = []; + sender[14] = 'Sender Name'; + sender[16] = 'sender@example.com'; + record[10] = sender; + record[13] = [[[null, 'attachment-1', null, [null, null, 'file.pdf', 'application/pdf', 1234]]]]; + record[16] = 1_700_000_001_000; + record[34] = 'legacy-message-id'; + body[1] = [['thread-f:1234567890123456789', null, [['msg-f:1', record]]]]; + return body; +} + +function batchPage(start, count) { + const body = batchViewFixture({ labels: false }); + body[2] = Array.from({ length: count }, (_unused, offset) => { + const record = structuredClone(body[2]?.[0]?.[0]); + record[0] = `Fixture subject ${start + offset}`; + record[2] = 1_700_000_000_000 + start + offset; + record[3] = `thread-f:${1234567890123456789n + BigInt(start + offset)}`; + return [[...record], null]; + }); + return body; +} + +function captureEntry(path, body, overrides = {}) { + return { + url: `https://mail.google.com/sync/u/0/i/${path}`, + responseStatus: 200, + responsePreview: JSON.stringify(body), + ...overrides, + }; +} + +function capturePage(entry, query = 'is:unread') { + let typed = ''; + const page = { + getCurrentUrl: vi.fn().mockResolvedValue('https://mail.google.com/mail/u/0/#inbox'), + goto: vi.fn().mockResolvedValue(undefined), + sleep: vi.fn().mockResolvedValue(undefined), + startNetworkCapture: vi.fn().mockResolvedValue(true), + readNetworkCapture: vi.fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(entry ? [entry] : []) + .mockResolvedValue([]), + nativeType: vi.fn().mockImplementation(async (value) => { typed = value; }), + nativeKeyPress: vi.fn().mockResolvedValue(undefined), + nativeClick: vi.fn().mockResolvedValue(undefined), + cdp: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn().mockImplementation(async (source) => { + const code = String(source); + if (code.includes('!!document.querySelector')) return true; + if (code.includes('field.select()')) return true; + if (code.includes("document.querySelector('input[name=\"q\"]')?.value")) return typed || query; + if (code.includes('data-legacy-thread-id')) return { found: false }; + if (code.includes("const target = '#all/")) return true; + return true; + }), + }; + return page; +} + +describe('gmail positional response parsers', () => { + it('parses an arity-19 batch-view thread and labels', () => { + const body = batchViewFixture(); + expect(parseBatchView(body)).toEqual([expect.objectContaining({ + threadId: 'thread-f:1234567890123456789', + subject: 'Fixture subject', + from: 'sender@example.com', + unread: true, + starred: true, + date: '2023-11-14T22:13:20.000Z', + })]); + expect(parseLabels(body)).toEqual([ + { id: '^i', name: 'Inbox', type: 'system', unreadCount: 2, totalCount: 10 }, + { id: '^x_project', name: 'Project', type: 'user', unreadCount: 1, totalCount: 3 }, + ]); + }); + + it('parses fetch-data messages, bodies, recipients, and attachments', () => { + expect(parseFetchData(fetchDataFixture())).toEqual([expect.objectContaining({ + messageId: 'msg-f:1', + legacyMessageId: 'legacy-message-id', + from: 'sender@example.com', + to: 'to@example.com', + cc: 'cc@example.com', + body: 'Hello world', + attachments: [{ + attachmentId: 'attachment-1', + name: 'file.pdf', + mimeType: 'application/pdf', + size: 1234, + }], + })]); + }); + + it('rejects malformed positional arrays instead of returning partial data', () => { + expect(() => parseBatchView([])).toThrow(CommandExecutionError); + expect(() => parseLabels(Array(18).fill(null))).toThrow(CommandExecutionError); + expect(() => parseFetchData([null, 'wrong'])).toThrow(CommandExecutionError); + }); + + it('converts HTML to readable text without scripts or styles', () => { + expect(htmlToText('

Hello world

')) + .toBe('Hello world'); + }); +}); + +describe('gmail input validation', () => { + it('strictly validates limit and account values', () => { + expect(parseLimit(50)).toBe(50); + expect(parseAccount(2)).toBe(2); + for (const value of [0, -1, 1.5, 'x', 201]) expect(() => parseLimit(value)).toThrow(ArgumentError); + for (const value of [-1, 1.5, 'x', 21]) expect(() => parseAccount(value)).toThrow(ArgumentError); + }); + + it('accepts sync ids, legacy hex ids, and Gmail thread URLs', () => { + expect(legacyThreadId('thread-f:1234567890123456789')).toBe(BigInt('1234567890123456789').toString(16)); + expect(legacyThreadId('18ABCDEF123')).toBe('18abcdef123'); + expect(legacyThreadId('https://mail.google.com/mail/u/0/#all/18ABCDEF123')).toBe('18abcdef123'); + expect(() => legacyThreadId('not-a-thread')).toThrow(ArgumentError); + }); +}); + +describe('gmail browser capture path', () => { + it('queries threads through Gmail natural navigation and captured bv data', async () => { + const page = capturePage(captureEntry('bv', batchViewFixture({ labels: false })), 'from:sender@example.com'); + const rows = await queryThreads(page, 'from:sender@example.com', { limit: 1, account: 0 }); + expect(page.nativeType).toHaveBeenCalledWith('from:sender@example.com'); + expect(page.cdp).toHaveBeenCalledWith('Input.dispatchKeyEvent', expect.objectContaining({ key: 'Enter' })); + expect(rows).toHaveLength(1); + }); + + it('merges all completed captures so pagination cannot select a stale response', async () => { + const first = captureEntry('bv', batchPage(0, 50)); + const second = captureEntry('bv', batchPage(50, 5)); + const page = capturePage(null, 'in:anywhere'); + page.readNetworkCapture + .mockReset() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([first]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([second, first]) + .mockResolvedValueOnce([]); + page.evaluate.mockImplementation(async (source) => { + const code = String(source); + if (code.includes('!!document.querySelector')) return true; + if (code.includes('field.select()')) return true; + if (code.includes("document.querySelector('input[name=\"q\"]')?.value")) return 'in:anywhere'; + if (code.includes('getBoundingClientRect')) return { found: true, x: 1, y: 1 }; + return true; + }); + + const rows = await queryThreads(page, 'in:anywhere', { limit: 55, account: 0 }); + expect(rows).toHaveLength(55); + expect(new Set(rows.map((row) => row.threadId)).size).toBe(55); + expect(page.nativeClick).toHaveBeenCalledOnce(); + }); + + it('rejects a repeated pagination response instead of returning partial rows', async () => { + const first = captureEntry('bv', batchPage(0, 50)); + const page = capturePage(null, 'in:anywhere'); + page.readNetworkCapture + .mockReset() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([first]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([first]) + .mockResolvedValueOnce([]); + page.evaluate.mockImplementation(async (source) => { + const code = String(source); + if (code.includes('!!document.querySelector')) return true; + if (code.includes('field.select()')) return true; + if (code.includes("document.querySelector('input[name=\"q\"]')?.value")) return 'in:anywhere'; + if (code.includes('getBoundingClientRect')) return { found: true, x: 1, y: 1 }; + return true; + }); + + await expect(queryThreads(page, 'in:anywhere', { limit: 55, account: 0 })) + .rejects.toThrow(/refusing partial results/); + }); + + it('rejects a completed short page when another capture was drained without a body', async () => { + const completed = captureEntry('bv', batchPage(0, 5)); + const bodyless = { + url: 'https://mail.google.com/sync/u/0/i/bv', + responseStatus: 200, + }; + const page = capturePage(null, 'in:anywhere'); + page.readNetworkCapture + .mockReset() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([bodyless, completed]) + .mockResolvedValueOnce([]); + + await expect(queryThreads(page, 'in:anywhere', { limit: 10, account: 0 })) + .rejects.toThrow(/refusing possibly partial results/); + }); + + it('lists labels through a fresh captured bv response', async () => { + const page = capturePage(captureEntry('bv', batchViewFixture({ threads: false }))); + await expect(listLabels(page, 0)).resolves.toHaveLength(2); + expect(page.nativeType).toHaveBeenCalledWith('in:anywhere'); + }); + + it('loads a thread through a fresh captured fd response', async () => { + const page = capturePage(captureEntry('fd', fetchDataFixture())); + const messages = await fetchThread(page, 'thread-f:1234567890123456789', 0); + expect(page.evaluate).toHaveBeenCalledWith(expect.stringContaining('#all/112210f47de98115')); + expect(messages).toHaveLength(1); + }); + + it('preserves typed auth, truncation, malformed, empty, and timeout failures', async () => { + const authPage = capturePage(captureEntry('bv', [], { responseStatus: 401 })); + await expect(queryThreads(authPage, 'is:unread', { limit: 1 })).rejects.toThrow(AuthRequiredError); + + const truncatedPage = capturePage(captureEntry('bv', [], { responseBodyTruncated: true })); + await expect(queryThreads(truncatedPage, 'is:unread', { limit: 1 })).rejects.toThrow(/capture limit/); + + const malformedPage = capturePage(captureEntry('bv', [], { responsePreview: '{' })); + await expect(queryThreads(malformedPage, 'is:unread', { limit: 1 })).rejects.toThrow(/malformed JSON/); + + const emptyPage = capturePage(captureEntry('bv', batchViewFixture({ threads: false, labels: false }))); + await expect(queryThreads(emptyPage, 'is:unread', { limit: 1 })).rejects.toThrow(EmptyResultError); + + vi.useFakeTimers(); + try { + const timeoutPage = capturePage(null); + timeoutPage.sleep.mockImplementation(async (seconds) => { vi.advanceTimersByTime(seconds * 1000); }); + await expect(queryThreads(timeoutPage, 'is:unread', { limit: 1 })).rejects.toThrow(TimeoutError); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/docs/adapters/browser/gmail.md b/docs/adapters/browser/gmail.md new file mode 100644 index 00000000..d3620bba --- /dev/null +++ b/docs/adapters/browser/gmail.md @@ -0,0 +1,53 @@ +# Gmail + +**Mode**: 🔐 Browser · **Domain**: `mail.google.com` + +## Commands + +| Command | Description | +|---------|-------------| +| `opencli gmail whoami` | Show the signed-in Gmail identity | +| `opencli gmail login` | Open Gmail's Google sign-in flow | +| `opencli gmail search ` | Search threads with Gmail search syntax | +| `opencli gmail inbox` | List inbox threads | +| `opencli gmail unread` | List unread threads | +| `opencli gmail starred` | List starred threads | +| `opencli gmail sent` | List sent threads | +| `opencli gmail drafts` | List draft threads | +| `opencli gmail trash` | List trashed threads | +| `opencli gmail spam` | List spam threads | +| `opencli gmail snoozed` | List snoozed threads | +| `opencli gmail important` | List important threads | +| `opencli gmail labels` | List system and user labels | +| `opencli gmail thread ` | Read all messages in a thread | +| `opencli gmail attachments ` | List attachment metadata for a thread | + +## Usage Examples + +```bash +# Search and use the returned threadId to read a conversation +opencli gmail search 'from:alerts@example.com newer_than:30d' --limit 20 -f json +opencli gmail thread 'thread-f:1234567890123456789' -f json + +# Common mailbox views and labels +opencli gmail inbox --limit 50 +opencli gmail unread --limit 50 +opencli gmail labels -f json + +# List attachment metadata without downloading files +opencli gmail attachments 'thread-f:1234567890123456789' -f json +``` + +All list commands accept `--account ` for Gmail URLs under `/mail/u//`. Thread-list commands also accept `--limit` from 1 to 200. + +## Output + +Search and mailbox views return thread identity, subject, sender, snippet, message count, unread/starred flags, timestamp, and Gmail label ids. `thread` returns one row per message, including recipients, body text, and an attachment array. `attachments` flattens that array to one row per file. + +Reads let Gmail perform its normal search/navigation and parse the resulting browser response. Recently cached threads may use the already rendered message containers instead. The adapter does not reconstruct Gmail's private authentication or write requests. + +## Prerequisites + +- Chrome running and **logged into** Gmail +- [Browser Bridge extension](/guide/browser-bridge) installed +- Use `opencli gmail whoami` to verify the active account before reading mail diff --git a/sitemaps/gmail/SITE.md b/sitemaps/gmail/SITE.md new file mode 100644 index 00000000..7c6a58b9 --- /dev/null +++ b/sitemaps/gmail/SITE.md @@ -0,0 +1,30 @@ +--- +schema_version: 1.1 +site: mail.google.com +last_verified: 2026-08-25 +source: global +login_required: true +auth_strategy: COOKIE +--- + +## Overview + +Gmail desktop mail UI. Prefer `opencli gmail ...`: reads use Gmail's own search/navigation and intercept natural responses. This sitemap does not define write actions because no supported browser-session API contract is currently available. + +## Top-level routes + +- `/mail/u//#inbox` → pages/search.md +- `/mail/u//#search/` → pages/search.md +- `/mail/u//#all/` → pages/thread.md +- `/mail/u//#settings/labels` → settings labels, read by `opencli gmail labels` +- other settings routes → outside this sitemap; inspect current browser state + +## Common goals + +- search/list mail → workflows/search.md +- read a complete thread or attachment metadata → workflows/read-thread.md +- inspect identity → `opencli gmail whoami` + +## Site-wide pitfalls + +See pitfalls.md. Gmail uses private positional arrays, may serve cached thread content without a new request, and its search box requires native input events. diff --git a/sitemaps/gmail/pages/search.md b/sitemaps/gmail/pages/search.md new file mode 100644 index 00000000..3a308f03 --- /dev/null +++ b/sitemaps/gmail/pages/search.md @@ -0,0 +1,42 @@ +--- +schema_version: 1.1 +page_id: search +url_patterns: + - https://mail.google.com/mail/u//#inbox + - https://mail.google.com/mail/u//#search/ +purpose: inbox, Gmail search results, and common mailbox views +last_verified: 2026-08-25 +source: global +--- + +## Visual anchors + +- selector_pattern: `input[name="q"]` +- selector_pattern: `[data-legacy-thread-id]` +- pattern: result rows expose subject, sender, snippet, labels, and time + +## Actions + +```yaml +### action:search_threads +pre: signed in; Gmail search input mounted +do: opencli gmail search "" --limit +post: rows contain threadId, subject, sender, flags, date; exact limit reached or upstream exhausted +fail: AuthRequired | capture Timeout | positional-shape CommandExecutionError +recover: adapter_health_update: opencli gmail search -> suspect; confirm input[name="q"] and run browser network while submitting the query +evidence: live exact-sender query + 55-row pagination on 2026-08-25 +``` + +```yaml +### action:open_thread +pre: target result row known by threadId +do: opencli gmail thread +post: one row per rendered/structured message with body +fail: target row absent and direct route fails | no fd response and no rendered message +recover: rerun gmail search for the target, then retry thread; adapter_health_update: opencli gmail thread -> suspect +evidence: live result row opened by data-legacy-thread-id +``` + +## Linked APIs + +API details are in `skills/opencli-adapter-author/references/site-memory/gmail.md`; no endpoint ids are duplicated here. diff --git a/sitemaps/gmail/pages/thread.md b/sitemaps/gmail/pages/thread.md new file mode 100644 index 00000000..e4710d09 --- /dev/null +++ b/sitemaps/gmail/pages/thread.md @@ -0,0 +1,43 @@ +--- +schema_version: 1.1 +page_id: thread +url_patterns: + - https://mail.google.com/mail/u//#all/ +purpose: expanded Gmail conversation with messages and attachments +last_verified: 2026-08-25 +source: global +--- + +## Visual anchors + +- selector_pattern: `[data-message-id][data-legacy-message-id]` +- selector_pattern: `.a3s` inside the matching message container +- selector_pattern: `.aQH` or `[download_url]` inside the message container +- pattern: subject heading `h2[data-thread-perm-id]` or `h2.hP` + +## Actions + +```yaml +### action:read_messages +pre: signed in; target thread id known +do: opencli gmail thread +post: every expanded message returns id, sender, subject, body, and attachment array +fail: fd shape changed | rendered fallback finds no data-message-id/body pair +recover: adapter_health_update: opencli gmail thread -> suspect; expand collapsed messages, then inspect within each message container +evidence: live cached thread exercised rendered fallback on 2026-08-25 +``` + +```yaml +### action:list_attachments +pre: target thread id known +do: opencli gmail attachments +post: one row per attachment; EmptyResult when none +fail: attachment cards visible but no rows | wrong message scope +recover: adapter_health_update: opencli gmail attachments -> suspect; inspect .aQH within each data-message-id container +evidence: live PDF attachment metadata on 2026-08-25 +``` + +## Page-specific pitfalls + +- Opening a recently cached thread may produce no fresh `/fd`; rendered message containers are the intended fallback. +- Scope body and attachment selectors to each message container. diff --git a/sitemaps/gmail/pitfalls.md b/sitemaps/gmail/pitfalls.md new file mode 100644 index 00000000..e6c9f403 --- /dev/null +++ b/sitemaps/gmail/pitfalls.md @@ -0,0 +1,31 @@ +--- +schema_version: 1.1 +last_verified: 2026-08-25 +source: global +--- + +## Site-specific pitfalls + +### pitfall:search_does_not_submit_with_synthetic_events +trigger: an agent fills Gmail search by assigning `.value` and dispatching synthetic Enter +symptom: input text changes but URL/results stay on the previous query +workaround: use `opencli gmail search`; browser fallback must use native text insertion plus raw native Enter +verified_at: 2026-08-25 + +### pitfall:cached_thread_has_no_fresh_response +trigger: an agent opens a recently loaded thread expecting a new network response +symptom: no `/fd` capture although the thread body is visibly rendered +workaround: use `opencli gmail thread`; fallback scopes `[data-message-id]` and `.a3s` to the rendered thread +verified_at: 2026-08-25 + +### pitfall:private_post_is_not_a_write_signal +trigger: an agent classifies Gmail requests only by HTTP method +symptom: read-only `/bv` and `/fd` POSTs are mistaken for writes, or private write POSTs are replayed +workaround: classify by user-visible action and observed effect; never replay private Gmail writes +verified_at: 2026-08-25 + +### pitfall:private_sync_writes_are_not_replayable_contracts +trigger: an agent tries to turn one captured `/sync` send or delete request into a production command +symptom: replay depends on opaque positional state or a page-generated anti-abuse token and may fail, duplicate, or mutate the wrong message +workaround: do not fabricate or replay Gmail private writes; use Google OAuth with Gmail scopes for supported API-backed write commands +verified_at: 2026-08-25 diff --git a/sitemaps/gmail/workflows/read-thread.md b/sitemaps/gmail/workflows/read-thread.md new file mode 100644 index 00000000..11e1f658 --- /dev/null +++ b/sitemaps/gmail/workflows/read-thread.md @@ -0,0 +1,37 @@ +--- +schema_version: 1.1 +workflow_id: read-thread +intent: read a Gmail conversation and its attachment metadata +last_verified: 2026-08-25 +source: global +--- + +## Goal + +Resolve a search thread id into complete message bodies and optional attachment metadata. + +## State signature + +- entry: thread id from Gmail search, legacy hex id, or Gmail thread URL +- success: thread page has scoped message containers and command returns message rows + +## Best path + +adapter: `opencli gmail thread `; use `opencli gmail attachments ` for attachment rows +adapter_health: healthy +preconditions: signed in; target id valid +estimated_turns: 1 + +## Fallback path + +on_adapter_fail: +1. `adapter_health_update: opencli gmail thread -> suspect` +2. Search again so the target `[data-legacy-thread-id]` row is mounted. +3. Open that row; expand collapsed messages. +4. Extract `.a3s` and `.aQH` only inside each `[data-message-id]` container. + +## Avoid + +- Page-level first-match body/attachment selectors. +- Retrying an `fd` capture indefinitely when Gmail served cached rendered content. +- Logging or saving real message bodies as fixtures. diff --git a/sitemaps/gmail/workflows/search.md b/sitemaps/gmail/workflows/search.md new file mode 100644 index 00000000..5611d1d6 --- /dev/null +++ b/sitemaps/gmail/workflows/search.md @@ -0,0 +1,37 @@ +--- +schema_version: 1.1 +workflow_id: search +intent: find or list Gmail threads with complete pagination +last_verified: 2026-08-25 +source: global +--- + +## Goal + +Return structured Gmail threads using Gmail search syntax or a common mailbox view. + +## State signature + +- entry: signed-in Gmail page with `input[name="q"]` +- success: URL `#search/` and structured rows matching the requested query + +## Best path + +adapter: `opencli gmail search `; use inbox/unread/starred/sent/drafts/trash/spam/snoozed/important shortcuts when applicable +adapter_health: healthy +preconditions: signed in; limit 1-200 +estimated_turns: 1 + +## Fallback path + +on_adapter_fail: +1. `adapter_health_update: opencli gmail search -> suspect` +2. Verify `input[name="q"]` in current browser state. +3. Fill the exact Gmail query with native input and submit with a native Enter event. +4. Read visible `[data-legacy-thread-id]` rows; do not claim pagination completeness. + +## Avoid + +- Replaying private `/sync` POST bodies or copying XSRF/BTAI values. +- Treating GET as safe or POST as a write; Gmail read queries use POST. +- Returning partial rows after a continuation capture fails.