diff --git a/AGENTS.md b/AGENTS.md index eda0b19e..8e9f37bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,8 @@ claude --plugin-dir ./apps/hook | `PLANNOTATOR_SHARE` | Set to `disabled` to turn off URL sharing entirely. Default: enabled. | | `PLANNOTATOR_SHARE_URL` | Custom base URL for share links (self-hosted portal). Default: `https://share.plannotator.ai`. | | `PLANNOTATOR_PASTE_URL` | Base URL of the paste service API for short URL sharing. Default: `https://plannotator-paste.plannotator.workers.dev`. | +| `PLANNOTATOR_JINA` | Set to `0` / `false` to disable Jina Reader for URL annotation, or `1` / `true` to enable. Default: enabled. Can also be set via `~/.plannotator/config.json` (`{ "jina": false }`) or per-invocation via `--no-jina`. | +| `JINA_API_KEY` | Optional Jina Reader API key for higher rate limits (500 RPM vs 20 RPM unauthenticated). Free keys include 10M tokens. | | `PLANNOTATOR_VERIFY_ATTESTATION` | **Read by the install scripts only**, not by the runtime binary. Set to `1` / `true` to have `scripts/install.sh` / `install.ps1` / `install.cmd` run `gh attestation verify` on every install. Off by default. Can also be set persistently via `~/.plannotator/config.json` (`{ "verifyAttestation": true }`) or per-invocation via `--verify-attestation`. Requires `gh` installed and authenticated. | **Legacy:** `SSH_TTY` and `SSH_CONNECTION` are still detected when `PLANNOTATOR_REMOTE` is unset. Set `PLANNOTATOR_REMOTE=1` / `true` to force remote mode or `0` / `false` to force local mode. @@ -152,16 +154,20 @@ Approve → "LGTM" sent to agent session ## Annotate Flow ``` -User runs /plannotator-annotate command +User runs /plannotator-annotate ↓ Claude Code: plannotator annotate subcommand runs -OpenCode: event handler intercepts command +OpenCode/Pi: event handler intercepts command ↓ -Markdown file read from disk +Input type detected: + .md/.mdx → file read from disk + .html/.htm → file read, converted to markdown via Turndown + https:// → fetched via Jina Reader (default) or fetch+Turndown (--no-jina) + folder/ → file browser opened, files converted on demand ↓ Annotate server starts (reuses plan editor HTML with mode:"annotate") ↓ -User annotates markdown, provides feedback +User annotates content, provides feedback ↓ Send Annotations → feedback sent to agent session ``` @@ -247,7 +253,7 @@ During normal plan review, an Archive sidebar tab provides the same browsing via | Endpoint | Method | Purpose | | --------------------- | ------ | ------------------------------------------ | -| `/api/plan` | GET | Returns `{ plan, origin, mode: "annotate", filePath }` | +| `/api/plan` | GET | Returns `{ plan, origin, mode: "annotate", filePath, sourceInfo? }` | | `/api/feedback` | POST | Submit annotations (body: feedback, annotations) | | `/api/image` | GET | Serve image by path query param | | `/api/upload` | POST | Upload image, returns `{ path, originalName }` | diff --git a/apps/hook/server/cli.test.ts b/apps/hook/server/cli.test.ts index c1dc60bf..3a5ad440 100644 --- a/apps/hook/server/cli.test.ts +++ b/apps/hook/server/cli.test.ts @@ -19,7 +19,7 @@ describe("CLI top-level help", () => { expect(output).toContain("plannotator --help"); expect(output).toContain("plannotator [--browser ]"); expect(output).toContain("plannotator review [PR_URL]"); - expect(output).toContain("plannotator annotate "); + expect(output).toContain("plannotator annotate "); expect(output).toContain("running 'plannotator' without arguments is for hook integration"); }); }); diff --git a/apps/hook/server/cli.ts b/apps/hook/server/cli.ts index c1d98b25..b39ef544 100644 --- a/apps/hook/server/cli.ts +++ b/apps/hook/server/cli.ts @@ -15,7 +15,7 @@ export function formatTopLevelHelp(): string { " plannotator --help", " plannotator [--browser ]", " plannotator review [PR_URL]", - " plannotator annotate ", + " plannotator annotate [--no-jina]", " plannotator last", " plannotator archive", " plannotator sessions", @@ -33,7 +33,7 @@ export function formatInteractiveNoArgClarification(): string { "", "For interactive use, try:", " plannotator review", - " plannotator annotate ", + " plannotator annotate ", " plannotator last", " plannotator archive", " plannotator sessions", diff --git a/apps/hook/server/index.ts b/apps/hook/server/index.ts index 7572d43c..89284c5e 100644 --- a/apps/hook/server/index.ts +++ b/apps/hook/server/index.ts @@ -64,17 +64,20 @@ import { handleAnnotateServerReady, } from "@plannotator/server/annotate"; import { type DiffType, getVcsContext, runVcsDiff, gitRuntime } from "@plannotator/server/vcs"; -import { loadConfig, resolveDefaultDiffType } from "@plannotator/shared/config"; +import { loadConfig, resolveDefaultDiffType, resolveUseJina } from "@plannotator/shared/config"; +import { htmlToMarkdown } from "@plannotator/shared/html-to-markdown"; +import { urlToMarkdown } from "@plannotator/shared/url-to-markdown"; import { fetchRef, createWorktree, removeWorktree, ensureObjectAvailable } from "@plannotator/shared/worktree"; import { parsePRUrl, checkPRAuth, fetchPR, getCliName, getCliInstallUrl, getMRLabel, getMRNumberLabel, getDisplayRepo } from "@plannotator/server/pr"; import { writeRemoteShareLink } from "@plannotator/server/share-url"; import { resolveMarkdownFile, hasMarkdownFiles } from "@plannotator/shared/resolve-file"; import { FILE_BROWSER_EXCLUDED } from "@plannotator/shared/reference-common"; -import { statSync, rmSync, realpathSync } from "fs"; +import { statSync, rmSync, realpathSync, existsSync } from "fs"; import { parseRemoteUrl } from "@plannotator/shared/repo"; import { registerSession, unregisterSession, listSessions } from "@plannotator/server/sessions"; import { openBrowser } from "@plannotator/server/browser"; import { detectProjectName } from "@plannotator/server/project"; +import { hostnameOrFallback } from "@plannotator/shared/project"; import { planDenyFeedback } from "@plannotator/shared/feedback-templates"; import { readImprovementHook } from "@plannotator/shared/improvement-hooks"; import type { Origin } from "@plannotator/shared/agents"; @@ -109,6 +112,11 @@ if (browserIdx !== -1 && args[browserIdx + 1]) { args.splice(browserIdx, 2); } +// Global flag: --no-jina (disables Jina Reader for URL annotation) +const noJinaIdx = args.indexOf("--no-jina"); +const cliNoJina = noJinaIdx !== -1; +if (cliNoJina) args.splice(noJinaIdx, 1); + if (isTopLevelHelpInvocation(args)) { console.log(formatTopLevelHelp()); process.exit(0); @@ -451,7 +459,7 @@ if (args[0] === "sessions") { let filePath = args[1]; if (!filePath) { - console.error("Usage: plannotator annotate "); + console.error("Usage: plannotator annotate [--no-jina]"); process.exit(1); } @@ -468,50 +476,87 @@ if (args[0] === "sessions") { console.error(`[DEBUG] File path arg: ${filePath}`); } - // Check if the argument is a directory (folder annotation mode) - const resolvedArg = path.resolve(projectRoot, filePath); - let isFolder = false; - try { - isFolder = statSync(resolvedArg).isDirectory(); - } catch { - // Not a directory, fall through to file resolution - } - let markdown: string; let absolutePath: string; let folderPath: string | undefined; let annotateMode: "annotate" | "annotate-folder" = "annotate"; + let sourceInfo: string | undefined; - if (isFolder) { - // Folder annotation mode - if (!hasMarkdownFiles(resolvedArg, FILE_BROWSER_EXCLUDED)) { - console.error(`No markdown files found in ${resolvedArg}`); - process.exit(1); - } - folderPath = resolvedArg; - absolutePath = resolvedArg; - markdown = ""; - annotateMode = "annotate-folder"; - console.error(`Folder: ${resolvedArg}`); - } else { - // Single file annotation mode - const resolved = resolveMarkdownFile(filePath, projectRoot); + // --- URL annotation --- + const isUrl = /^https?:\/\//i.test(filePath); - if (resolved.kind === "ambiguous") { - console.error(`Ambiguous filename "${resolved.input}" — found ${resolved.matches.length} matches:`); - for (const match of resolved.matches) { - console.error(` ${match}`); + if (isUrl) { + const useJina = resolveUseJina(cliNoJina, loadConfig()); + console.error(`Fetching: ${filePath}${useJina ? " (via Jina Reader)" : " (via fetch+Turndown)"}`); + try { + const result = await urlToMarkdown(filePath, { useJina }); + markdown = result.markdown; + if (process.env.PLANNOTATOR_DEBUG) { + console.error(`[DEBUG] Fetched via ${result.source} (${markdown.length} chars)`); } + } catch (err) { + console.error(`Failed to fetch URL: ${err instanceof Error ? err.message : String(err)}`); process.exit(1); } - if (resolved.kind === "not_found") { - console.error(`File not found: ${resolved.input}`); - process.exit(1); + absolutePath = filePath; // Use URL as the "path" for display + sourceInfo = filePath; // Full URL for source attribution + } else { + // Check if the argument is a directory (folder annotation mode) + const resolvedArg = path.resolve(projectRoot, filePath); + let isFolder = false; + try { + isFolder = statSync(resolvedArg).isDirectory(); + } catch { + // Not a directory, fall through to file resolution } - absolutePath = resolved.path; - markdown = await Bun.file(absolutePath).text(); - console.error(`Resolved: ${absolutePath}`); + if (isFolder) { + // Folder annotation mode (markdown + HTML files) + if (!hasMarkdownFiles(resolvedArg, FILE_BROWSER_EXCLUDED, /\.(mdx?|html?)$/i)) { + console.error(`No markdown or HTML files found in ${resolvedArg}`); + process.exit(1); + } + folderPath = resolvedArg; + absolutePath = resolvedArg; + markdown = ""; + annotateMode = "annotate-folder"; + console.error(`Folder: ${resolvedArg}`); + } else if (/\.html?$/i.test(resolvedArg)) { + // HTML file annotation mode — convert to markdown via Turndown + if (!existsSync(resolvedArg)) { + console.error(`File not found: ${filePath}`); + process.exit(1); + } + const htmlFile = Bun.file(resolvedArg); + if (htmlFile.size > 10 * 1024 * 1024) { + console.error(`File too large (${Math.round(htmlFile.size / 1024 / 1024)}MB, max 10MB): ${resolvedArg}`); + process.exit(1); + } + const html = await htmlFile.text(); + markdown = htmlToMarkdown(html); + absolutePath = resolvedArg; + sourceInfo = path.basename(resolvedArg); + console.error(`Converted: ${absolutePath}`); + } else { + // Single markdown file annotation mode + const resolved = resolveMarkdownFile(filePath, projectRoot); + + if (resolved.kind === "ambiguous") { + console.error(`Ambiguous filename "${resolved.input}" — found ${resolved.matches.length} matches:`); + for (const match of resolved.matches) { + console.error(` ${match}`); + } + process.exit(1); + } + if (resolved.kind === "not_found") { + console.error(`File not found: ${resolved.input}`); + process.exit(1); + } + + absolutePath = resolved.path; + markdown = await Bun.file(absolutePath).text(); + console.error(`Resolved: ${absolutePath}`); + } } const annotateProject = (await detectProjectName()) ?? "_unknown"; @@ -523,6 +568,7 @@ if (args[0] === "sessions") { origin: detectedOrigin, mode: annotateMode, folderPath, + sourceInfo, sharingEnabled, shareBaseUrl, pasteApiUrl, @@ -543,7 +589,9 @@ if (args[0] === "sessions") { mode: "annotate", project: annotateProject, startedAt: new Date().toISOString(), - label: folderPath ? `annotate-${path.basename(folderPath)}` : `annotate-${path.basename(absolutePath)}`, + label: folderPath + ? `annotate-${path.basename(folderPath)}` + : `annotate-${isUrl ? hostnameOrFallback(absolutePath) : path.basename(absolutePath)}`, }); // Wait for user feedback diff --git a/apps/opencode-plugin/commands.ts b/apps/opencode-plugin/commands.ts index 523f4723..4f1350c2 100644 --- a/apps/opencode-plugin/commands.ts +++ b/apps/opencode-plugin/commands.ts @@ -20,8 +20,12 @@ import { } from "@plannotator/server/annotate"; import { getGitContext, runGitDiffWithContext } from "@plannotator/server/git"; import { parsePRUrl, checkPRAuth, fetchPR, getCliName, getMRLabel, getMRNumberLabel, getDisplayRepo } from "@plannotator/server/pr"; -import { loadConfig, resolveDefaultDiffType } from "@plannotator/shared/config"; +import { loadConfig, resolveDefaultDiffType, resolveUseJina } from "@plannotator/shared/config"; import { resolveMarkdownFile } from "@plannotator/shared/resolve-file"; +import { htmlToMarkdown } from "@plannotator/shared/html-to-markdown"; +import { urlToMarkdown } from "@plannotator/shared/url-to-markdown"; +import { statSync } from "fs"; +import path from "path"; /** Shared dependencies injected by the plugin */ export interface CommandDeps { @@ -149,35 +153,79 @@ export async function handleAnnotateCommand( const filePath = event.properties?.arguments || event.arguments || ""; if (!filePath) { - client.app.log({ level: "error", message: "Usage: /plannotator-annotate " }); + client.app.log({ level: "error", message: "Usage: /plannotator-annotate " }); return; } - client.app.log({ level: "info", message: `Opening annotation UI for ${filePath}...` }); + let markdown: string; + let absolutePath: string; + let sourceInfo: string | undefined; - const projectRoot = process.cwd(); - const resolved = await resolveMarkdownFile(filePath, projectRoot); + // --- URL annotation --- + const isUrl = /^https?:\/\//i.test(filePath); - if (resolved.kind === "ambiguous") { - client.app.log({ - level: "error", - message: `Ambiguous filename "${resolved.input}" — found ${resolved.matches.length} matches:\n${resolved.matches.map((m) => ` ${m}`).join("\n")}`, - }); - return; + if (isUrl) { + const useJina = resolveUseJina(false, loadConfig()); + client.app.log({ level: "info", message: `Fetching: ${filePath}${useJina ? " (via Jina Reader)" : " (via fetch+Turndown)"}...` }); + try { + const result = await urlToMarkdown(filePath, { useJina }); + markdown = result.markdown; + } catch (err) { + client.app.log({ level: "error", message: `Failed to fetch URL: ${err instanceof Error ? err.message : String(err)}` }); + return; + } + absolutePath = filePath; + sourceInfo = filePath; + } else { + const projectRoot = process.cwd(); + const resolvedArg = path.resolve(projectRoot, filePath); + + if (/\.html?$/i.test(resolvedArg)) { + // HTML file annotation — convert to markdown via Turndown + let fileSize: number; + try { + fileSize = statSync(resolvedArg).size; + } catch { + client.app.log({ level: "error", message: `File not found: ${filePath}` }); + return; + } + if (fileSize > 10 * 1024 * 1024) { + client.app.log({ level: "error", message: `File too large (${Math.round(fileSize / 1024 / 1024)}MB, max 10MB)` }); + return; + } + const html = await Bun.file(resolvedArg).text(); + markdown = htmlToMarkdown(html); + absolutePath = resolvedArg; + sourceInfo = path.basename(resolvedArg); + client.app.log({ level: "info", message: `Converted: ${absolutePath}` }); + } else { + // Markdown file annotation + client.app.log({ level: "info", message: `Opening annotation UI for ${filePath}...` }); + const resolved = await resolveMarkdownFile(filePath, projectRoot); + + if (resolved.kind === "ambiguous") { + client.app.log({ + level: "error", + message: `Ambiguous filename "${resolved.input}" — found ${resolved.matches.length} matches:\n${resolved.matches.map((m) => ` ${m}`).join("\n")}`, + }); + return; + } + if (resolved.kind === "not_found") { + client.app.log({ level: "error", message: `File not found: ${resolved.input}` }); + return; + } + + absolutePath = resolved.path; + client.app.log({ level: "info", message: `Resolved: ${absolutePath}` }); + markdown = await Bun.file(absolutePath).text(); + } } - if (resolved.kind === "not_found") { - client.app.log({ level: "error", message: `File not found: ${resolved.input}` }); - return; - } - - const absolutePath = resolved.path; - client.app.log({ level: "info", message: `Resolved: ${absolutePath}` }); - const markdown = await Bun.file(absolutePath).text(); const server = await startAnnotateServer({ markdown, filePath: absolutePath, origin: "opencode", + sourceInfo, sharingEnabled: await getSharingEnabled(), shareBaseUrl: getShareBaseUrl(), htmlContent, diff --git a/apps/opencode-plugin/commands/plannotator-annotate.md b/apps/opencode-plugin/commands/plannotator-annotate.md index a8b03084..f2c7d220 100644 --- a/apps/opencode-plugin/commands/plannotator-annotate.md +++ b/apps/opencode-plugin/commands/plannotator-annotate.md @@ -1,5 +1,5 @@ --- -description: Open interactive annotation UI for a markdown file +description: Open interactive annotation UI for a markdown file, HTML file, or URL --- The Plannotator Annotate UI has been triggered. Opening the annotation UI... diff --git a/apps/pi-extension/index.ts b/apps/pi-extension/index.ts index 38ed6cfe..afe9f472 100644 --- a/apps/pi-extension/index.ts +++ b/apps/pi-extension/index.ts @@ -19,7 +19,7 @@ */ import { existsSync, readFileSync, statSync } from "node:fs"; -import { resolve } from "node:path"; +import { resolve, basename } from "node:path"; import type { ThinkingLevel } from "@mariozechner/pi-agent-core"; import { Type } from "@mariozechner/pi-ai"; import type { @@ -36,6 +36,9 @@ import { import { planDenyFeedback } from "./generated/feedback-templates.js"; import { hasMarkdownFiles } from "./generated/resolve-file.js"; import { FILE_BROWSER_EXCLUDED } from "./generated/reference-common.js"; +import { htmlToMarkdown } from "./generated/html-to-markdown.js"; +import { urlToMarkdown } from "./generated/url-to-markdown.js"; +import { loadConfig, resolveUseJina } from "./generated/config.js"; import { getLastAssistantMessageText, hasPlanBrowserHtml, @@ -380,7 +383,7 @@ export default function plannotator(pi: ExtensionAPI): void { handler: async (args, ctx) => { const filePath = args?.trim(); if (!filePath) { - ctx.ui.notify("Usage: /plannotator-annotate ", "error"); + ctx.ui.notify("Usage: /plannotator-annotate ", "error"); return; } if (!hasPlanBrowserHtml()) { @@ -391,41 +394,70 @@ export default function plannotator(pi: ExtensionAPI): void { return; } - const absolutePath = resolve(ctx.cwd, filePath); - if (!existsSync(absolutePath)) { - ctx.ui.notify(`File not found: ${absolutePath}`, "error"); - return; - } - - // Check if the argument is a directory (folder annotation mode) - let isFolder = false; - try { - isFolder = statSync(absolutePath).isDirectory(); - } catch { - ctx.ui.notify(`Cannot access: ${absolutePath}`, "error"); - return; - } - let markdown: string; + let absolutePath: string; let folderPath: string | undefined; let mode: "annotate" | "annotate-folder" | undefined; + let sourceInfo: string | undefined; + let isFolder = false; - if (isFolder) { - if (!hasMarkdownFiles(absolutePath, FILE_BROWSER_EXCLUDED)) { - ctx.ui.notify(`No markdown files found in ${absolutePath}`, "error"); + // --- URL annotation --- + const isUrl = /^https?:\/\//i.test(filePath); + + if (isUrl) { + const useJina = resolveUseJina(false, loadConfig()); + ctx.ui.notify(`Fetching: ${filePath}${useJina ? " (via Jina Reader)" : " (via fetch+Turndown)"}...`, "info"); + try { + const result = await urlToMarkdown(filePath, { useJina }); + markdown = result.markdown; + } catch (err) { + ctx.ui.notify(`Failed to fetch URL: ${err instanceof Error ? err.message : String(err)}`, "error"); return; } - markdown = ""; - folderPath = absolutePath; - mode = "annotate-folder"; - ctx.ui.notify(`Opening annotation UI for folder ${filePath}...`, "info"); + absolutePath = filePath; + sourceInfo = filePath; } else { - markdown = readFileSync(absolutePath, "utf-8"); - ctx.ui.notify(`Opening annotation UI for ${filePath}...`, "info"); + absolutePath = resolve(ctx.cwd, filePath); + if (!existsSync(absolutePath)) { + ctx.ui.notify(`File not found: ${absolutePath}`, "error"); + return; + } + + try { + isFolder = statSync(absolutePath).isDirectory(); + } catch { + ctx.ui.notify(`Cannot access: ${absolutePath}`, "error"); + return; + } + + if (isFolder) { + if (!hasMarkdownFiles(absolutePath, FILE_BROWSER_EXCLUDED, /\.(mdx?|html?)$/i)) { + ctx.ui.notify(`No markdown or HTML files found in ${absolutePath}`, "error"); + return; + } + markdown = ""; + folderPath = absolutePath; + mode = "annotate-folder"; + ctx.ui.notify(`Opening annotation UI for folder ${filePath}...`, "info"); + } else if (/\.html?$/i.test(absolutePath)) { + // HTML file annotation — convert to markdown via Turndown + const fileSize = statSync(absolutePath).size; + if (fileSize > 10 * 1024 * 1024) { + ctx.ui.notify(`File too large (${Math.round(fileSize / 1024 / 1024)}MB, max 10MB)`, "error"); + return; + } + const html = readFileSync(absolutePath, "utf-8"); + markdown = htmlToMarkdown(html); + sourceInfo = basename(absolutePath); + ctx.ui.notify(`Opening annotation UI for ${filePath}...`, "info"); + } else { + markdown = readFileSync(absolutePath, "utf-8"); + ctx.ui.notify(`Opening annotation UI for ${filePath}...`, "info"); + } } try { - const result = await openMarkdownAnnotation(ctx, absolutePath, markdown, mode ?? "annotate", folderPath); + const result = await openMarkdownAnnotation(ctx, absolutePath, markdown, mode ?? "annotate", folderPath, sourceInfo); if (result.exit) { ctx.ui.notify("Annotation session closed.", "info"); } else if (result.feedback) { diff --git a/apps/pi-extension/package.json b/apps/pi-extension/package.json index 3e144233..22e9f627 100644 --- a/apps/pi-extension/package.json +++ b/apps/pi-extension/package.json @@ -38,6 +38,10 @@ "build": "cp ../hook/dist/index.html plannotator.html && cp ../hook/dist/review.html review-editor.html && rm -rf skills && cp -r ../skills skills && bash vendor.sh", "prepublishOnly": "cd ../.. && bun run build:pi" }, + "dependencies": { + "turndown": "^7.2.4", + "@joplin/turndown-plugin-gfm": "^1.0.64" + }, "peerDependencies": { "@mariozechner/pi-coding-agent": ">=0.53.0" } diff --git a/apps/pi-extension/plannotator-browser.ts b/apps/pi-extension/plannotator-browser.ts index 3ee5bba7..8292c655 100644 --- a/apps/pi-extension/plannotator-browser.ts +++ b/apps/pi-extension/plannotator-browser.ts @@ -368,6 +368,7 @@ export async function openMarkdownAnnotation( markdown: string, mode: AnnotateMode, folderPath?: string, + sourceInfo?: string, ): Promise<{ feedback: string; exit?: boolean }> { if (!ctx.hasUI || !planHtmlContent) { throw new Error("Plannotator annotation browser is unavailable in this session."); @@ -391,6 +392,7 @@ export async function openMarkdownAnnotation( origin: "pi", mode, folderPath, + sourceInfo, htmlContent: planHtmlContent, sharingEnabled: process.env.PLANNOTATOR_SHARE !== "disabled", shareBaseUrl: process.env.PLANNOTATOR_SHARE_URL || undefined, diff --git a/apps/pi-extension/server/reference.ts b/apps/pi-extension/server/reference.ts index 89a965a7..04f76963 100644 --- a/apps/pi-extension/server/reference.ts +++ b/apps/pi-extension/server/reference.ts @@ -22,12 +22,13 @@ import { FILE_BROWSER_EXCLUDED, } from "../generated/reference-common.js"; import { detectObsidianVaults } from "../generated/integrations-common.js"; -import { resolveMarkdownFile } from "../generated/resolve-file.js"; +import { resolveMarkdownFile, isWithinProjectRoot } from "../generated/resolve-file.js"; +import { htmlToMarkdown } from "../generated/html-to-markdown.js"; type Res = ServerResponse; -/** Recursively walk a directory collecting markdown files, skipping ignored dirs. */ -function walkMarkdownFiles(dir: string, root: string, results: string[]): void { +/** Recursively walk a directory collecting files by extension, skipping ignored dirs. */ +function walkMarkdownFiles(dir: string, root: string, results: string[], extensions: RegExp = /\.(mdx?|html?)$/i): void { let entries: Dirent[]; try { entries = readdirSync(dir, { withFileTypes: true }) as Dirent[]; @@ -37,8 +38,8 @@ function walkMarkdownFiles(dir: string, root: string, results: string[]): void { for (const entry of entries) { if (entry.isDirectory()) { if (FILE_BROWSER_EXCLUDED.includes(entry.name + "/")) continue; - walkMarkdownFiles(join(dir, entry.name), root, results); - } else if (entry.isFile() && /\.mdx?$/i.test(entry.name)) { + walkMarkdownFiles(join(dir, entry.name), root, results, extensions); + } else if (entry.isFile() && extensions.test(entry.name)) { const relative = join(dir, entry.name) .slice(root.length + 1) .replace(/\\/g, "/"); @@ -55,17 +56,22 @@ export function handleDocRequest(res: Res, url: URL): void { return; } - // Try resolving relative to base directory first (used by annotate mode) + // Try resolving relative to base directory first (used by annotate mode). + // No isWithinProjectRoot check here — intentional, matches pre-existing + // markdown behavior. The base param is set server-side by the annotate + // server (see serverAnnotate.ts /api/doc route). The standalone HTML + // block below (no base) retains its cwd-based containment check. const base = url.searchParams.get("base"); if ( base && !requestedPath.startsWith("/") && - /\.mdx?$/i.test(requestedPath) + /\.(mdx?|html?)$/i.test(requestedPath) ) { const fromBase = resolvePath(base, requestedPath); try { if (existsSync(fromBase)) { - const markdown = readFileSync(fromBase, "utf-8"); + const raw = readFileSync(fromBase, "utf-8"); + const markdown = /\.html?$/i.test(requestedPath) ? htmlToMarkdown(raw) : raw; json(res, { markdown, filepath: fromBase }); return; } @@ -74,7 +80,25 @@ export function handleDocRequest(res: Res, url: URL): void { } } + // HTML files: resolve directly (not via resolveMarkdownFile which only handles .md/.mdx) const projectRoot = process.cwd(); + if (/\.html?$/i.test(requestedPath)) { + const resolvedHtml = resolvePath(base || projectRoot, requestedPath); + if (!isWithinProjectRoot(resolvedHtml, projectRoot)) { + json(res, { error: "Access denied: path is outside project root" }, 403); + return; + } + try { + if (existsSync(resolvedHtml)) { + const html = readFileSync(resolvedHtml, "utf-8"); + json(res, { markdown: htmlToMarkdown(html), filepath: resolvedHtml }); + return; + } + } catch { /* fall through to 404 */ } + json(res, { error: `File not found: ${requestedPath}` }, 404); + return; + } + const result = resolveMarkdownFile(requestedPath, projectRoot); if (result.kind === "ambiguous") { @@ -119,7 +143,7 @@ export function handleObsidianFilesRequest(res: Res, url: URL): void { } try { const files: string[] = []; - walkMarkdownFiles(resolvedVault, resolvedVault, files); + walkMarkdownFiles(resolvedVault, resolvedVault, files, /\.mdx?$/i); files.sort(); json(res, { tree: buildFileTree(files) }); } catch { @@ -144,7 +168,7 @@ export function handleObsidianDocRequest(res: Res, url: URL): void { // Bare filename search within vault if (!existsSync(resolvedFile) && !filePath.includes("/")) { const files: string[] = []; - walkMarkdownFiles(resolvedVault, resolvedVault, files); + walkMarkdownFiles(resolvedVault, resolvedVault, files, /\.mdx?$/i); const matches = files.filter( (f) => f.split("/").pop()!.toLowerCase() === filePath.toLowerCase(), ); diff --git a/apps/pi-extension/server/serverAnnotate.ts b/apps/pi-extension/server/serverAnnotate.ts index 7699aa12..7b1e1177 100644 --- a/apps/pi-extension/server/serverAnnotate.ts +++ b/apps/pi-extension/server/serverAnnotate.ts @@ -42,6 +42,7 @@ export async function startAnnotateServer(options: { sharingEnabled?: boolean; shareBaseUrl?: string; pasteApiUrl?: string; + sourceInfo?: string; }): Promise { const gitUser = detectGitUser(); const sharingEnabled = @@ -83,6 +84,7 @@ export async function startAnnotateServer(options: { origin: options.origin ?? "pi", mode: options.mode || "annotate", filePath: options.filePath, + sourceInfo: options.sourceInfo, sharingEnabled, shareBaseUrl, pasteApiUrl, @@ -109,8 +111,9 @@ export async function startAnnotateServer(options: { } else if (url.pathname === "/api/draft") { await handleDraftRequest(req, res, draftKey); } else if (url.pathname === "/api/doc" && req.method === "GET") { - // Inject source file's directory as base for relative path resolution - if (!url.searchParams.has("base") && options.filePath) { + // Inject source file's directory as base for relative path resolution. + // Skip for URL annotations — there's no local directory to resolve against. + if (!url.searchParams.has("base") && options.filePath && !/^https?:\/\//i.test(options.filePath)) { url.searchParams.set("base", dirname(resolvePath(options.filePath))); } handleDocRequest(res, url); diff --git a/apps/pi-extension/vendor.sh b/apps/pi-extension/vendor.sh index 579a6ce3..e7dfce03 100755 --- a/apps/pi-extension/vendor.sh +++ b/apps/pi-extension/vendor.sh @@ -6,7 +6,7 @@ cd "$(dirname "$0")" mkdir -p generated generated/ai/providers -for f in feedback-templates review-core storage draft project pr-provider pr-github pr-gitlab checklist integrations-common repo reference-common favicon resolve-file config external-annotation agent-jobs worktree; do +for f in feedback-templates review-core storage draft project pr-provider pr-github pr-gitlab checklist integrations-common repo reference-common favicon resolve-file config external-annotation agent-jobs worktree html-to-markdown url-to-markdown; do src="../../packages/shared/$f.ts" printf '// @generated — DO NOT EDIT. Source: packages/shared/%s.ts\n' "$f" | cat - "$src" > "generated/$f.ts" done diff --git a/bun.lock b/bun.lock index 786f3433..e29705ab 100644 --- a/bun.lock +++ b/bun.lock @@ -17,6 +17,7 @@ "devDependencies": { "@types/dompurify": "^3.2.0", "@types/node": "^25.5.0", + "@types/turndown": "^5.0.6", "bun-types": "^1.3.11", }, }, @@ -63,7 +64,7 @@ }, "apps/opencode-plugin": { "name": "@plannotator/opencode", - "version": "0.17.7", + "version": "0.17.9", "dependencies": { "@opencode-ai/plugin": "^1.1.10", }, @@ -85,7 +86,11 @@ }, "apps/pi-extension": { "name": "@plannotator/pi-extension", - "version": "0.17.7", + "version": "0.17.9", + "dependencies": { + "@joplin/turndown-plugin-gfm": "^1.0.64", + "turndown": "^7.2.4", + }, "peerDependencies": { "@mariozechner/pi-coding-agent": ">=0.53.0", }, @@ -171,7 +176,7 @@ }, "packages/server": { "name": "@plannotator/server", - "version": "0.17.7", + "version": "0.17.9", "dependencies": { "@plannotator/ai": "workspace:*", "@plannotator/shared": "workspace:*", @@ -183,6 +188,10 @@ "packages/shared": { "name": "@plannotator/shared", "version": "0.0.1", + "dependencies": { + "@joplin/turndown-plugin-gfm": "^1.0.64", + "turndown": "^7.2.4", + }, }, "packages/ui": { "name": "@plannotator/ui", @@ -516,6 +525,8 @@ "@isaacs/cliui": ["@isaacs/cliui@9.0.0", "", {}, "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg=="], + "@joplin/turndown-plugin-gfm": ["@joplin/turndown-plugin-gfm@1.0.64", "", {}, "sha512-8GJ7f9OenE3zkSVII5B6qzIkvgF7C/a20gaASEjM6jWPLPJFFQ2nQ3Ou/kXH1mPUTs9dC9VYs8QXVPvZabKXBQ=="], + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], @@ -564,6 +575,8 @@ "@mistralai/mistralai": ["@mistralai/mistralai@1.10.0", "", { "dependencies": { "zod": "^3.20.0", "zod-to-json-schema": "^3.24.1" } }, "sha512-tdIgWs4Le8vpvPiUEWne6tK0qbVc+jMenujnvTqOjogrJUsCSQhus0tHTU1avDDh5//Rq2dFgP9mWRAdIEoBqg=="], + "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.28.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-gmloF+i+flI8ouQK7MWW4mOwuMh4RePBuPFAEPC6+pdqyWOUMDOixb6qZ69owLJpz6XmyllCouc4t8YWO+E2Nw=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -1010,6 +1023,8 @@ "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + "@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/vscode": ["@types/vscode@1.109.0", "", {}, "sha512-0Pf95rnwEIwDbmXGC08r0B4TQhAbsHQ5UyTIgVgoieDe4cOnf92usuR5dEczb6bTKEp7ziZH4TV1TRGPPCExtw=="], @@ -2350,6 +2365,8 @@ "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], + "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="], + "type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], diff --git a/package.json b/package.json index 9175a034..442315fc 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "devDependencies": { "@types/dompurify": "^3.2.0", "@types/node": "^25.5.0", + "@types/turndown": "^5.0.6", "bun-types": "^1.3.11" } } diff --git a/packages/editor/App.tsx b/packages/editor/App.tsx index ca1e6304..5425d145 100644 --- a/packages/editor/App.tsx +++ b/packages/editor/App.tsx @@ -134,6 +134,7 @@ const App: React.FC = () => { const [globalAttachments, setGlobalAttachments] = useState([]); const [annotateMode, setAnnotateMode] = useState(false); const [annotateSource, setAnnotateSource] = useState<'file' | 'message' | 'folder' | null>(null); + const [sourceInfo, setSourceInfo] = useState(); const [sourceFilePath, setSourceFilePath] = useState(); const [imageBaseDir, setImageBaseDir] = useState(undefined); const [isLoading, setIsLoading] = useState(true); @@ -525,7 +526,7 @@ const App: React.FC = () => { if (!res.ok) throw new Error('Not in API mode'); return res.json(); }) - .then((data: { plan: string; origin?: Origin; mode?: 'annotate' | 'annotate-last' | 'annotate-folder' | 'archive'; filePath?: string; sharingEnabled?: boolean; shareBaseUrl?: string; pasteApiUrl?: string; repoInfo?: { display: string; branch?: string }; previousPlan?: string | null; versionInfo?: { version: number; totalVersions: number; project: string }; archivePlans?: ArchivedPlan[]; projectRoot?: string; isWSL?: boolean; serverConfig?: { displayName?: string; gitUser?: string } }) => { + .then((data: { plan: string; origin?: Origin; mode?: 'annotate' | 'annotate-last' | 'annotate-folder' | 'archive'; filePath?: string; sourceInfo?: string; sharingEnabled?: boolean; shareBaseUrl?: string; pasteApiUrl?: string; repoInfo?: { display: string; branch?: string }; previousPlan?: string | null; versionInfo?: { version: number; totalVersions: number; project: string }; archivePlans?: ArchivedPlan[]; projectRoot?: string; isWSL?: boolean; serverConfig?: { displayName?: string; gitUser?: string } }) => { // Initialize config store with server-provided values (config file > cookie > default) configStore.init(data.serverConfig); // gitUser drives the "Use git name" button in Settings; stays undefined (button hidden) when unavailable @@ -553,6 +554,7 @@ const App: React.FC = () => { if (data.mode && data.mode !== 'archive') { setAnnotateSource(data.mode === 'annotate-last' ? 'message' : data.mode === 'annotate-folder' ? 'folder' : 'file'); } + setSourceInfo(data.sourceInfo ?? undefined); if (data.filePath) { setImageBaseDir(data.mode === 'annotate-folder' ? data.filePath : data.filePath.replace(/\/[^/]+$/, '')); if (data.mode === 'annotate') { @@ -1625,6 +1627,7 @@ const App: React.FC = () => { imageBaseDir={imageBaseDir} copyLabel={annotateSource === 'message' ? 'Copy message' : annotateSource === 'file' || annotateSource === 'folder' ? 'Copy file' : undefined} archiveInfo={archive.currentInfo} + sourceInfo={sourceInfo} onToggleCheckbox={checkbox.toggle} checkboxOverrides={checkbox.overrides} actionsLabelMode={actionsLabelMode} diff --git a/packages/server/annotate.ts b/packages/server/annotate.ts index c61de8d7..7d913c55 100644 --- a/packages/server/annotate.ts +++ b/packages/server/annotate.ts @@ -48,6 +48,8 @@ export interface AnnotateServerOptions { shareBaseUrl?: string; /** Base URL of the paste service API for short URL sharing */ pasteApiUrl?: string; + /** Source attribution: original URL or filename (e.g. "https://..." or "index.html") */ + sourceInfo?: string; /** Called when server starts with the URL, remote status, and port */ onReady?: (url: string, isRemote: boolean, port: number) => void; } @@ -92,6 +94,7 @@ export async function startAnnotateServer( origin, mode = "annotate", folderPath, + sourceInfo, sharingEnabled = true, shareBaseUrl, pasteApiUrl, @@ -140,6 +143,7 @@ export async function startAnnotateServer( origin, mode, filePath, + sourceInfo, sharingEnabled, shareBaseUrl, pasteApiUrl, @@ -172,9 +176,10 @@ export async function startAnnotateServer( } // API: Serve a linked markdown document - // Inject source file's directory as base for relative path resolution + // Inject source file's directory as base for relative path resolution. + // Skip base injection for URL annotations — there's no local directory to resolve against. if (url.pathname === "/api/doc" && req.method === "GET") { - if (!url.searchParams.has("base")) { + if (!url.searchParams.has("base") && !/^https?:\/\//i.test(filePath)) { const docUrl = new URL(req.url); docUrl.searchParams.set("base", dirname(filePath)); return handleDoc(new Request(docUrl.toString())); diff --git a/packages/server/reference-handlers.ts b/packages/server/reference-handlers.ts index 79140142..b3968e3d 100644 --- a/packages/server/reference-handlers.ts +++ b/packages/server/reference-handlers.ts @@ -9,7 +9,8 @@ import { existsSync, statSync } from "fs"; import { resolve } from "path"; import { buildFileTree, FILE_BROWSER_EXCLUDED } from "@plannotator/shared/reference-common"; import { detectObsidianVaults } from "./integrations"; -import { resolveMarkdownFile } from "@plannotator/shared/resolve-file"; +import { resolveMarkdownFile, isWithinProjectRoot } from "@plannotator/shared/resolve-file"; +import { htmlToMarkdown } from "@plannotator/shared/html-to-markdown"; // --- Route handlers --- @@ -22,18 +23,23 @@ export async function handleDoc(req: Request): Promise { } // If a base directory is provided, try resolving relative to it first - // (used by annotate mode to resolve paths relative to the source file) + // (used by annotate mode to resolve paths relative to the source file). + // No isWithinProjectRoot check here — intentional, matches pre-existing + // markdown behavior. The base param is set server-side by the annotate + // server (see annotate.ts /api/doc route). The standalone HTML block + // below (no base) retains its cwd-based containment check. const base = url.searchParams.get("base"); if ( base && !requestedPath.startsWith("/") && - /\.mdx?$/i.test(requestedPath) + /\.(mdx?|html?)$/i.test(requestedPath) ) { const fromBase = resolve(base, requestedPath); try { const file = Bun.file(fromBase); if (await file.exists()) { - const markdown = await file.text(); + const raw = await file.text(); + const markdown = /\.html?$/i.test(requestedPath) ? htmlToMarkdown(raw) : raw; return Response.json({ markdown, filepath: fromBase }); } } catch { @@ -41,7 +47,24 @@ export async function handleDoc(req: Request): Promise { } } + // HTML files: resolve directly (not via resolveMarkdownFile which only handles .md/.mdx) const projectRoot = process.cwd(); + if (/\.html?$/i.test(requestedPath)) { + const resolvedHtml = resolve(base || projectRoot, requestedPath); + if (!isWithinProjectRoot(resolvedHtml, projectRoot)) { + return Response.json({ error: "Access denied: path is outside project root" }, { status: 403 }); + } + try { + const file = Bun.file(resolvedHtml); + if (await file.exists()) { + const html = await file.text(); + const markdown = htmlToMarkdown(html); + return Response.json({ markdown, filepath: resolvedHtml }); + } + } catch { /* fall through */ } + return Response.json({ error: `File not found: ${requestedPath}` }, { status: 404 }); + } + const result = resolveMarkdownFile(requestedPath, projectRoot); if (result.kind === "ambiguous") { @@ -203,7 +226,7 @@ export async function handleFileBrowserFiles(req: Request): Promise { } try { - const glob = new Bun.Glob("**/*.{md,mdx}"); + const glob = new Bun.Glob("**/*.{md,mdx,html,htm}"); const files: string[] = []; for await (const match of glob.scan({ cwd: resolvedDir, diff --git a/packages/shared/config.ts b/packages/shared/config.ts index a5c98b90..eeba1b95 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -45,6 +45,13 @@ export interface PlannotatorConfig { * (`gh auth login`). OS-level opt-in only — no UI surface. Default: false. */ verifyAttestation?: boolean; + /** + * Enable Jina Reader for URL-to-markdown conversion during annotation. + * When true (default), `plannotator annotate ` routes through + * r.jina.ai for better JS-rendered page support and reader-mode extraction. + * Set to false to always use plain fetch + Turndown. + */ + jina?: boolean; } const CONFIG_DIR = join(homedir(), ".plannotator"); @@ -125,3 +132,26 @@ export function resolveDefaultDiffType(cfg?: PlannotatorConfig): DefaultDiffType const v = cfg?.diffOptions?.defaultDiffType; return v === 'uncommitted' || v === 'unstaged' || v === 'staged' ? v : 'unstaged'; } + +/** + * Resolve whether to use Jina Reader for URL annotation. + * + * Priority (highest wins): + * --no-jina CLI flag → PLANNOTATOR_JINA env var → config.jina → default true + */ +export function resolveUseJina(cliNoJina: boolean, config: PlannotatorConfig): boolean { + // CLI flag has highest priority + if (cliNoJina) return false; + + // Environment variable + const envVal = process.env.PLANNOTATOR_JINA; + if (envVal !== undefined) { + return envVal === "1" || envVal.toLowerCase() === "true"; + } + + // Config file + if (config.jina !== undefined) return config.jina; + + // Default: enabled + return true; +} diff --git a/packages/shared/html-to-markdown.test.ts b/packages/shared/html-to-markdown.test.ts new file mode 100644 index 00000000..f9645bcc --- /dev/null +++ b/packages/shared/html-to-markdown.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from "bun:test"; +import { htmlToMarkdown } from "./html-to-markdown"; + +describe("htmlToMarkdown", () => { + test("converts basic HTML to markdown", () => { + const md = htmlToMarkdown("

Hello

World

"); + expect(md).toContain("# Hello"); + expect(md).toContain("World"); + }); + + test("converts tables with explicit thead", () => { + const html = "
AB
12
"; + const md = htmlToMarkdown(html); + expect(md).toContain("| A"); + expect(md).toContain("| ---"); + expect(md).toContain("| 1"); + }); + + test("converts tables WITHOUT thead (common HTML pattern)", () => { + const html = "
NameValue
foobar
"; + const md = htmlToMarkdown(html); + expect(md).toContain("| Name"); + expect(md).toContain("| ---"); + expect(md).toContain("| foo"); + }); + + test("strips script, style, and noscript tags", () => { + const html = '

Visible

'; + const md = htmlToMarkdown(html); + expect(md).toContain("Visible"); + expect(md).not.toContain("alert"); + expect(md).not.toContain(".x{}"); + expect(md).not.toContain("Hidden"); + }); + + test("converts strikethrough (GFM)", () => { + const md = htmlToMarkdown("

This is deleted text

"); + expect(md).toContain("~deleted~"); + }); + + test("preserves code blocks as fenced markdown", () => { + const md = htmlToMarkdown('
const x = 1;
'); + expect(md).toContain("```"); + expect(md).toContain("const x = 1;"); + }); + + test("converts links", () => { + const md = htmlToMarkdown('Click'); + expect(md).toContain("[Click](https://example.com)"); + }); + + test("preserves dangerous link hrefs (sanitization is in the renderer, not here)", () => { + const md = htmlToMarkdown('XSS'); + // Turndown preserves the link — the UI renderer blocks it + expect(md).toContain("javascript:"); + }); + + test("handles empty HTML gracefully", () => { + expect(htmlToMarkdown("")).toBe(""); + expect(htmlToMarkdown("")).toBe(""); + }); +}); diff --git a/packages/shared/html-to-markdown.ts b/packages/shared/html-to-markdown.ts new file mode 100644 index 00000000..c13189d8 --- /dev/null +++ b/packages/shared/html-to-markdown.ts @@ -0,0 +1,32 @@ +/** + * HTML-to-Markdown conversion via Turndown. + * + * Shared between the CLI (single HTML file / URL) and the server + * (on-demand conversion for HTML files in folder mode). + */ + +import TurndownService from "turndown"; +// @ts-expect-error — @joplin/turndown-plugin-gfm ships JS only, no .d.ts (see declarations.d.ts for local types) +import { gfm } from "@joplin/turndown-plugin-gfm"; + +const td = new TurndownService({ + headingStyle: "atx", + codeBlockStyle: "fenced", + bulletListMarker: "-", +}); + +td.use(gfm); + +// Strip