mirror of
https://github.com/jackwener/OpenCLI.git
synced 2026-09-14 18:25:42 +08:00
feat: add weixin article download adapter & abstract download helpers (#280)
- New: src/clis/weixin/download.ts — WeChat article to Markdown adapter - New: src/download/article-download.ts — shared article download helper (TurndownService, image localization, frontmatter, customizable labels) - New: src/download/media-download.ts — shared media download helper (batch download, ProgressTracker, yt-dlp routing, auto cookie export) - Refactor: migrate zhihu/download to use downloadArticle() - Refactor: migrate xiaohongshu/download to use downloadMedia() - Refactor: migrate twitter/download to use downloadMedia() - Refactor: migrate bilibili/download to use downloadMedia() - Docs: add weixin to README, README.zh-CN, download docs, adapter docs
This commit is contained in:
jakevin
@@ -115,6 +115,7 @@ Run `opencli list` for the live registry.
|
||||
| **apple-podcasts** | `search` `episodes` `top` | Public |
|
||||
| **xiaoyuzhou** | `podcast` `podcast-episodes` `episode` | Public |
|
||||
| **zhihu** | `hot` `search` `question` `download` | Browser |
|
||||
| **weixin** | `download` | Browser |
|
||||
| **youtube** | `search` `video` `transcript` | Browser |
|
||||
| **boss** | `search` `detail` `recommend` `joblist` `greet` `batchgreet` `send` `chatlist` `chatmsg` `invite` `mark` `exchange` `resume` `stats` | Browser |
|
||||
| **coupang** | `search` `add-to-cart` | Browser |
|
||||
@@ -192,6 +193,7 @@ OpenCLI supports downloading images, videos, and articles from supported platfor
|
||||
| **bilibili** | Videos | Requires `yt-dlp` installed |
|
||||
| **twitter** | Images, Videos | Downloads from user media tab or single tweet |
|
||||
| **zhihu** | Articles (Markdown) | Exports articles with optional image download |
|
||||
| **weixin** | Articles (Markdown) | Exports WeChat Official Account articles |
|
||||
|
||||
### Prerequisites
|
||||
|
||||
@@ -225,6 +227,9 @@ opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
||||
|
||||
# Export with local images
|
||||
opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --download-images
|
||||
|
||||
# Export WeChat article to Markdown
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ npm install -g @jackwener/opencli@latest
|
||||
| **apple-podcasts** | `search` `episodes` `top` | 公开 |
|
||||
| **xiaoyuzhou** | `podcast` `podcast-episodes` `episode` | 公开 |
|
||||
| **zhihu** | `hot` `search` `question` `download` | 浏览器 |
|
||||
| **weixin** | `download` | 浏览器 |
|
||||
| **youtube** | `search` `video` `transcript` | 浏览器 |
|
||||
| **boss** | `search` `detail` `recommend` `joblist` `greet` `batchgreet` `send` `chatlist` `chatmsg` `invite` `mark` `exchange` `resume` `stats` | 浏览器 |
|
||||
| **coupang** | `search` `add-to-cart` | 浏览器 |
|
||||
@@ -194,6 +195,7 @@ OpenCLI 支持从各平台下载图片、视频和文章。
|
||||
| **B站** | 视频 | 需要安装 `yt-dlp` |
|
||||
| **Twitter/X** | 图片、视频 | 从用户媒体页或单条推文下载 |
|
||||
| **知乎** | 文章（Markdown） | 导出文章，可选下载图片到本地 |
|
||||
| **微信公众号** | 文章（Markdown） | 导出微信公众号文章为 Markdown |
|
||||
|
||||
### 前置依赖
|
||||
|
||||
@@ -227,6 +229,9 @@ opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
||||
|
||||
# 导出并下载图片
|
||||
opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --download-images
|
||||
|
||||
# 导出微信公众号文章为 Markdown
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# WeChat (微信公众号)
|
||||
|
||||
**Mode**: 🔐 Browser · **Domain**: `mp.weixin.qq.com`
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `opencli weixin download` | 下载微信公众号文章为 Markdown 格式 |
|
||||
|
||||
## Usage Examples
|
||||
|
||||
```bash
|
||||
# Export article to Markdown
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
|
||||
# Export with locally downloaded images
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --download-images
|
||||
|
||||
# Export without images
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --no-download-images
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
Downloads to `<output>/<article-title>/`:
|
||||
- `<article-title>.md` — Markdown with frontmatter (title, author, publish time, source URL)
|
||||
- `images/` — Downloaded images (if `--download-images` is enabled, default: true)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Chrome running and **logged into** mp.weixin.qq.com (for articles behind login wall)
|
||||
- [Browser Bridge extension](/guide/browser-bridge) installed
|
||||
@@ -10,6 +10,7 @@ OpenCLI supports downloading images, videos, and articles from supported platfor
|
||||
| **bilibili** | Videos | Requires `yt-dlp` installed |
|
||||
| **twitter** | Images, Videos | Downloads from user media tab or single tweet |
|
||||
| **zhihu** | Articles (Markdown) | Exports articles with optional image download |
|
||||
| **weixin** | Articles (Markdown) | Exports WeChat Official Account articles |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -43,6 +44,9 @@ opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
||||
|
||||
# Export with local images
|
||||
opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --download-images
|
||||
|
||||
# Export WeChat article to Markdown
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
```
|
||||
|
||||
## Pipeline Step (YAML Adapters)
|
||||
|
||||
@@ -8,18 +8,9 @@
|
||||
* - yt-dlp must be installed: pip install yt-dlp
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import {
|
||||
ytdlpDownload,
|
||||
checkYtdlp,
|
||||
sanitizeFilename,
|
||||
getTempDir,
|
||||
exportCookiesToNetscape,
|
||||
formatCookieHeader,
|
||||
} from '../../download/index.js';
|
||||
import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
|
||||
import { checkYtdlp, sanitizeFilename } from '../../download/index.js';
|
||||
import { downloadMedia } from '../../download/media-download.js';
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
@@ -63,21 +54,8 @@ cli({
|
||||
|
||||
const title = sanitizeFilename(data?.title || 'video');
|
||||
|
||||
// Extract cookies for authenticated downloads
|
||||
const cookies = await page.getCookies({ domain: 'bilibili.com' });
|
||||
const cookieString = formatCookieHeader(cookies);
|
||||
|
||||
// Create output directory
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
|
||||
// Export cookies to Netscape format for yt-dlp
|
||||
let cookiesFile: string | undefined;
|
||||
if (cookies.length > 0) {
|
||||
const tempDir = getTempDir();
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
cookiesFile = path.join(tempDir, `bilibili_cookies_${Date.now()}.txt`);
|
||||
exportCookiesToNetscape(cookies, cookiesFile);
|
||||
}
|
||||
// Extract cookies for yt-dlp
|
||||
const browserCookies = await page.getCookies({ domain: 'bilibili.com' });
|
||||
|
||||
// Build yt-dlp format string based on quality
|
||||
let format = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best';
|
||||
@@ -89,62 +67,26 @@ cli({
|
||||
format = 'bestvideo[height<=480][ext=mp4]+bestaudio[ext=m4a]/best[height<=480]';
|
||||
}
|
||||
|
||||
const destPath = path.join(output, `${bvid}_${title}.mp4`);
|
||||
const videoUrl = `https://www.bilibili.com/video/${bvid}`;
|
||||
const filename = `${bvid}_${title}.mp4`;
|
||||
|
||||
const tracker = new DownloadProgressTracker(1, true);
|
||||
const progressBar = tracker.onFileStart(`${bvid}.mp4`, 0);
|
||||
const results = await downloadMedia(
|
||||
[{ type: 'video-ytdlp', url: videoUrl, filename }],
|
||||
{
|
||||
output,
|
||||
browserCookies,
|
||||
filenamePrefix: bvid,
|
||||
ytdlpExtraArgs: ['-f', format, '--merge-output-format', 'mp4', '--embed-thumbnail'],
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await ytdlpDownload(
|
||||
`https://www.bilibili.com/video/${bvid}`,
|
||||
destPath,
|
||||
{
|
||||
cookiesFile,
|
||||
format,
|
||||
extraArgs: [
|
||||
'--merge-output-format', 'mp4',
|
||||
'--embed-thumbnail',
|
||||
],
|
||||
onProgress: (percent) => {
|
||||
if (progressBar) progressBar.update(percent, 100);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (progressBar) {
|
||||
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
||||
}
|
||||
|
||||
tracker.onFileComplete(result.success);
|
||||
tracker.finish();
|
||||
|
||||
// Cleanup cookies file
|
||||
if (cookiesFile && fs.existsSync(cookiesFile)) {
|
||||
fs.unlinkSync(cookiesFile);
|
||||
}
|
||||
|
||||
return [{
|
||||
bvid,
|
||||
title: data?.title || 'video',
|
||||
status: result.success ? 'success' : 'failed',
|
||||
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
||||
}];
|
||||
} catch (err: any) {
|
||||
if (progressBar) progressBar.fail(err.message);
|
||||
tracker.onFileComplete(false);
|
||||
tracker.finish();
|
||||
|
||||
// Cleanup cookies file
|
||||
if (cookiesFile && fs.existsSync(cookiesFile)) {
|
||||
fs.unlinkSync(cookiesFile);
|
||||
}
|
||||
|
||||
return [{
|
||||
bvid,
|
||||
title: data?.title || 'video',
|
||||
status: 'failed',
|
||||
size: err.message,
|
||||
}];
|
||||
}
|
||||
// Map results to bilibili-specific columns
|
||||
const r = results[0] || { status: 'failed', size: '-' };
|
||||
return [{
|
||||
bvid,
|
||||
title: data?.title || 'video',
|
||||
status: r.status,
|
||||
size: r.size,
|
||||
}];
|
||||
},
|
||||
});
|
||||
|
||||
+13
-111
@@ -6,19 +6,9 @@
|
||||
* opencli twitter download --tweet-url https://x.com/xxx/status/123 --output ./twitter
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import {
|
||||
httpDownload,
|
||||
ytdlpDownload,
|
||||
checkYtdlp,
|
||||
sanitizeFilename,
|
||||
getTempDir,
|
||||
exportCookiesToNetscape,
|
||||
formatCookieHeader,
|
||||
} from '../../download/index.js';
|
||||
import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
|
||||
import { formatCookieHeader } from '../../download/index.js';
|
||||
import { downloadMedia } from '../../download/media-download.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
@@ -101,32 +91,11 @@ cli({
|
||||
`);
|
||||
|
||||
if (!data || data.length === 0) {
|
||||
return [{
|
||||
index: 0,
|
||||
type: '-',
|
||||
status: 'failed',
|
||||
size: 'No media found',
|
||||
}];
|
||||
return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
|
||||
}
|
||||
|
||||
// Extract cookies
|
||||
const cookies = await page.getCookies({ domain: 'x.com' });
|
||||
const cookieString = formatCookieHeader(cookies);
|
||||
|
||||
// Create output directory
|
||||
const outputDir = tweetUrl
|
||||
? path.join(output, 'tweets')
|
||||
: path.join(output, username || 'media');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
// Export cookies for yt-dlp
|
||||
let cookiesFile: string | undefined;
|
||||
if (cookies.length > 0) {
|
||||
const tempDir = getTempDir();
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
cookiesFile = path.join(tempDir, `twitter_cookies_${Date.now()}.txt`);
|
||||
exportCookiesToNetscape(cookies, cookiesFile);
|
||||
}
|
||||
const browserCookies = await page.getCookies({ domain: 'x.com' });
|
||||
|
||||
// Deduplicate media
|
||||
const seen = new Set<string>();
|
||||
@@ -136,81 +105,14 @@ cli({
|
||||
return true;
|
||||
}).slice(0, limit);
|
||||
|
||||
const tracker = new DownloadProgressTracker(uniqueMedia.length, true);
|
||||
const results: any[] = [];
|
||||
|
||||
for (let i = 0; i < uniqueMedia.length; i++) {
|
||||
const media = uniqueMedia[i];
|
||||
const ext = media.type === 'image' ? 'jpg' : 'mp4';
|
||||
const filename = `${username || 'tweet'}_${i + 1}.${ext}`;
|
||||
const destPath = path.join(outputDir, filename);
|
||||
|
||||
const progressBar = tracker.onFileStart(filename, i);
|
||||
|
||||
try {
|
||||
let result: { success: boolean; size: number; error?: string };
|
||||
|
||||
if (media.type === 'video-tweet' && checkYtdlp()) {
|
||||
// Use yt-dlp for video tweets
|
||||
result = await ytdlpDownload(media.url, destPath, {
|
||||
cookiesFile,
|
||||
extraArgs: ['--merge-output-format', 'mp4'],
|
||||
onProgress: (percent) => {
|
||||
if (progressBar) progressBar.update(percent, 100);
|
||||
},
|
||||
});
|
||||
} else if (media.type === 'image') {
|
||||
// Direct HTTP download for images
|
||||
result = await httpDownload(media.url, destPath, {
|
||||
cookies: cookieString,
|
||||
timeout: 30000,
|
||||
onProgress: (received, total) => {
|
||||
if (progressBar) progressBar.update(received, total);
|
||||
},
|
||||
});
|
||||
} else {
|
||||
// Direct HTTP download for direct video URLs
|
||||
result = await httpDownload(media.url, destPath, {
|
||||
cookies: cookieString,
|
||||
timeout: 60000,
|
||||
onProgress: (received, total) => {
|
||||
if (progressBar) progressBar.update(received, total);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (progressBar) {
|
||||
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
||||
}
|
||||
|
||||
tracker.onFileComplete(result.success);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type === 'video-tweet' ? 'video' : media.type,
|
||||
status: result.success ? 'success' : 'failed',
|
||||
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (progressBar) progressBar.fail(err.message);
|
||||
tracker.onFileComplete(false);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type,
|
||||
status: 'failed',
|
||||
size: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tracker.finish();
|
||||
|
||||
// Cleanup cookies file
|
||||
if (cookiesFile && fs.existsSync(cookiesFile)) {
|
||||
fs.unlinkSync(cookiesFile);
|
||||
}
|
||||
|
||||
return results;
|
||||
const subdir = tweetUrl ? 'tweets' : (username || 'media');
|
||||
return downloadMedia(uniqueMedia, {
|
||||
output,
|
||||
subdir,
|
||||
cookies: formatCookieHeader(browserCookies),
|
||||
browserCookies,
|
||||
filenamePrefix: username || 'tweet',
|
||||
ytdlpExtraArgs: ['--merge-output-format', 'mp4'],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* WeChat article download — export WeChat Official Account articles to Markdown.
|
||||
*
|
||||
* Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
|
||||
*
|
||||
* Usage:
|
||||
* opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { downloadArticle } from '../../download/article-download.js';
|
||||
|
||||
// ============================================================
|
||||
// URL Normalization
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Normalize a pasted WeChat article URL.
|
||||
*/
|
||||
export function normalizeWechatUrl(raw: string): string {
|
||||
let s = (raw || '').trim();
|
||||
if (!s) return s;
|
||||
|
||||
// Strip wrapping quotes / angle brackets
|
||||
if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
|
||||
s = s.slice(1, -1).trim();
|
||||
}
|
||||
if (s.startsWith('<') && s.endsWith('>')) {
|
||||
s = s.slice(1, -1).trim();
|
||||
}
|
||||
|
||||
// Remove backslash escapes before URL-significant characters
|
||||
s = s.replace(/\\+([:/&?=#%])/g, '$1');
|
||||
|
||||
// Decode HTML entities
|
||||
s = s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
||||
|
||||
// Allow bare hostnames
|
||||
if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
|
||||
s = 'https://' + s.replace(/^\/+/, '');
|
||||
}
|
||||
|
||||
// Force https for mp.weixin.qq.com
|
||||
try {
|
||||
const parsed = new URL(s);
|
||||
if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
|
||||
parsed.protocol = 'https:';
|
||||
s = parsed.toString();
|
||||
}
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CLI Registration
|
||||
// ============================================================
|
||||
|
||||
cli({
|
||||
site: 'weixin',
|
||||
name: 'download',
|
||||
description: '下载微信公众号文章为 Markdown 格式',
|
||||
domain: 'mp.weixin.qq.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
|
||||
{ name: 'output', default: './weixin-articles', help: 'Output directory' },
|
||||
{ name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
|
||||
],
|
||||
columns: ['title', 'author', 'publish_time', 'status', 'size'],
|
||||
func: async (page, kwargs) => {
|
||||
const rawUrl = kwargs.url;
|
||||
const url = normalizeWechatUrl(rawUrl);
|
||||
|
||||
if (!url.startsWith('https://mp.weixin.qq.com/')) {
|
||||
return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
|
||||
}
|
||||
|
||||
// Navigate and wait for content to load
|
||||
await page.goto(url);
|
||||
await page.wait(5);
|
||||
|
||||
// Extract article data in browser context
|
||||
const data = await page.evaluate(`
|
||||
(() => {
|
||||
const result = {
|
||||
title: '',
|
||||
author: '',
|
||||
publishTime: '',
|
||||
contentHtml: '',
|
||||
codeBlocks: [],
|
||||
imageUrls: []
|
||||
};
|
||||
|
||||
// Title: #activity-name
|
||||
const titleEl = document.querySelector('#activity-name');
|
||||
result.title = titleEl ? titleEl.textContent.trim() : '';
|
||||
|
||||
// Author (WeChat Official Account name): #js_name
|
||||
const authorEl = document.querySelector('#js_name');
|
||||
result.author = authorEl ? authorEl.textContent.trim() : '';
|
||||
|
||||
// Publish time: extract create_time from script tags
|
||||
const htmlStr = document.documentElement.innerHTML;
|
||||
let timeMatch = htmlStr.match(/create_time\\s*:\\s*JsDecode\\('([^']+)'\\)/);
|
||||
if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*:\\s*'(\\d+)'/);
|
||||
if (!timeMatch) timeMatch = htmlStr.match(/create_time\\s*[:=]\\s*["']?(\\d+)["']?/);
|
||||
if (timeMatch) {
|
||||
const ts = parseInt(timeMatch[1], 10);
|
||||
if (ts > 0) {
|
||||
const d = new Date(ts * 1000);
|
||||
const pad = n => String(n).padStart(2, '0');
|
||||
const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
|
||||
result.publishTime =
|
||||
utc8.getUTCFullYear() + '-' +
|
||||
pad(utc8.getUTCMonth() + 1) + '-' +
|
||||
pad(utc8.getUTCDate()) + ' ' +
|
||||
pad(utc8.getUTCHours()) + ':' +
|
||||
pad(utc8.getUTCMinutes()) + ':' +
|
||||
pad(utc8.getUTCSeconds());
|
||||
}
|
||||
}
|
||||
|
||||
// Content processing
|
||||
const contentEl = document.querySelector('#js_content');
|
||||
if (!contentEl) return result;
|
||||
|
||||
// Fix lazy-loaded images: data-src -> src
|
||||
contentEl.querySelectorAll('img').forEach(img => {
|
||||
const dataSrc = img.getAttribute('data-src');
|
||||
if (dataSrc) img.setAttribute('src', dataSrc);
|
||||
});
|
||||
|
||||
// Extract code blocks with placeholder replacement
|
||||
const codeBlocks = [];
|
||||
contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
|
||||
el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
|
||||
const pre = el.querySelector('pre[data-lang]');
|
||||
const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
|
||||
const lines = [];
|
||||
el.querySelectorAll('code').forEach(codeTag => {
|
||||
const text = codeTag.textContent;
|
||||
if (/^[ce]?ounter\\(line/.test(text)) return;
|
||||
lines.push(text);
|
||||
});
|
||||
if (lines.length === 0) lines.push(el.textContent);
|
||||
const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
|
||||
codeBlocks.push({ lang, code: lines.join('\\n') });
|
||||
const p = document.createElement('p');
|
||||
p.textContent = placeholder;
|
||||
el.replaceWith(p);
|
||||
});
|
||||
result.codeBlocks = codeBlocks;
|
||||
|
||||
// Remove noise elements
|
||||
['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
|
||||
contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
|
||||
});
|
||||
|
||||
// Collect image URLs (deduplicated)
|
||||
const seen = new Set();
|
||||
contentEl.querySelectorAll('img[src]').forEach(img => {
|
||||
const src = img.getAttribute('src');
|
||||
if (src && !seen.has(src)) {
|
||||
seen.add(src);
|
||||
result.imageUrls.push(src);
|
||||
}
|
||||
});
|
||||
|
||||
result.contentHtml = contentEl.innerHTML;
|
||||
return result;
|
||||
})()
|
||||
`);
|
||||
|
||||
return downloadArticle(
|
||||
{
|
||||
title: data?.title || '',
|
||||
author: data?.author,
|
||||
publishTime: data?.publishTime,
|
||||
sourceUrl: url,
|
||||
contentHtml: data?.contentHtml || '',
|
||||
codeBlocks: data?.codeBlocks,
|
||||
imageUrls: data?.imageUrls,
|
||||
},
|
||||
{
|
||||
output: kwargs.output,
|
||||
downloadImages: kwargs['download-images'],
|
||||
imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
|
||||
frontmatterLabels: { author: '公众号' },
|
||||
detectImageExt: (url) => {
|
||||
const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
|
||||
return m ? m[1] : 'png';
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
@@ -5,15 +5,9 @@
|
||||
* opencli xiaohongshu download --note_id abc123 --output ./xhs
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import {
|
||||
httpDownload,
|
||||
sanitizeFilename,
|
||||
formatCookieHeader,
|
||||
} from '../../download/index.js';
|
||||
import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
|
||||
import { formatCookieHeader } from '../../download/index.js';
|
||||
import { downloadMedia } from '../../download/media-download.js';
|
||||
|
||||
cli({
|
||||
site: 'xiaohongshu',
|
||||
@@ -69,7 +63,6 @@ cli({
|
||||
if (src && (src.includes('xhscdn') || src.includes('xiaohongshu'))) {
|
||||
// Convert to high quality URL (remove resize parameters)
|
||||
src = src.split('?')[0];
|
||||
// Try to get original size
|
||||
src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
|
||||
imageUrls.add(src);
|
||||
}
|
||||
@@ -88,20 +81,14 @@ cli({
|
||||
document.querySelectorAll(selector).forEach(v => {
|
||||
const src = v.src || v.getAttribute('src') || '';
|
||||
if (src) {
|
||||
result.media.push({
|
||||
type: 'video',
|
||||
url: src
|
||||
});
|
||||
result.media.push({ type: 'video', url: src });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Add images to media
|
||||
imageUrls.forEach(url => {
|
||||
result.media.push({
|
||||
type: 'image',
|
||||
url: url
|
||||
});
|
||||
result.media.push({ type: 'image', url: url });
|
||||
});
|
||||
|
||||
return result;
|
||||
@@ -115,58 +102,12 @@ cli({
|
||||
// Extract cookies for authenticated downloads
|
||||
const cookies = formatCookieHeader(await page.getCookies({ domain: 'xiaohongshu.com' }));
|
||||
|
||||
// Create output directory
|
||||
const outputDir = path.join(output, noteId);
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
// Download all media files
|
||||
const tracker = new DownloadProgressTracker(data.media.length, true);
|
||||
const results: any[] = [];
|
||||
|
||||
for (let i = 0; i < data.media.length; i++) {
|
||||
const media = data.media[i];
|
||||
const ext = media.type === 'video' ? 'mp4' : 'jpg';
|
||||
const filename = `${noteId}_${i + 1}.${ext}`;
|
||||
const destPath = path.join(outputDir, filename);
|
||||
|
||||
const progressBar = tracker.onFileStart(filename, i);
|
||||
|
||||
try {
|
||||
const result = await httpDownload(media.url, destPath, {
|
||||
cookies,
|
||||
timeout: 60000,
|
||||
onProgress: (received, total) => {
|
||||
if (progressBar) progressBar.update(received, total);
|
||||
},
|
||||
});
|
||||
|
||||
if (progressBar) {
|
||||
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
||||
}
|
||||
|
||||
tracker.onFileComplete(result.success);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type,
|
||||
status: result.success ? 'success' : 'failed',
|
||||
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (progressBar) progressBar.fail(err.message);
|
||||
tracker.onFileComplete(false);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type,
|
||||
status: 'failed',
|
||||
size: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tracker.finish();
|
||||
|
||||
return results;
|
||||
return downloadMedia(data.media, {
|
||||
output,
|
||||
subdir: noteId,
|
||||
cookies,
|
||||
filenamePrefix: noteId,
|
||||
timeout: 60000,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { htmlToMarkdown } from './download.js';
|
||||
import TurndownService from 'turndown';
|
||||
|
||||
describe('htmlToMarkdown', () => {
|
||||
describe('article markdown conversion', () => {
|
||||
it('renders ordered lists with the original list item content', () => {
|
||||
const html = '<ol><li>First item</li><li>Second item</li></ol>';
|
||||
const td = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-' });
|
||||
const md = td.turndown(html);
|
||||
|
||||
expect(htmlToMarkdown(html)).toContain('1. First item');
|
||||
expect(htmlToMarkdown(html)).toContain('2. Second item');
|
||||
expect(htmlToMarkdown(html)).not.toContain('$1');
|
||||
expect(md).toMatch(/1\.\s+First item/);
|
||||
expect(md).toMatch(/2\.\s+Second item/);
|
||||
expect(md).not.toContain('$1');
|
||||
});
|
||||
});
|
||||
|
||||
+23
-158
@@ -5,85 +5,8 @@
|
||||
* opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { sanitizeFilename, httpDownload, formatCookieHeader } from '../../download/index.js';
|
||||
import { formatBytes } from '../../download/progress.js';
|
||||
|
||||
/**
|
||||
* Convert HTML content to Markdown.
|
||||
* This is a simplified converter for Zhihu article content.
|
||||
*/
|
||||
export function htmlToMarkdown(html: string): string {
|
||||
let md = html;
|
||||
|
||||
// Remove script and style tags
|
||||
md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
|
||||
md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
|
||||
|
||||
// Convert headers
|
||||
md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
|
||||
md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
|
||||
md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
|
||||
md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n');
|
||||
|
||||
// Convert paragraphs
|
||||
md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n');
|
||||
|
||||
// Convert links
|
||||
md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
|
||||
|
||||
// Convert images
|
||||
md = md.replace(/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '');
|
||||
md = md.replace(/<img[^>]*src="([^"]*)"[^>]*\/?>/gi, '');
|
||||
|
||||
// Convert lists
|
||||
md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
|
||||
return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
|
||||
});
|
||||
md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
|
||||
let index = 0;
|
||||
return content.replace(
|
||||
/<li[^>]*>([\s\S]*?)<\/li>/gi,
|
||||
(_itemMatch: string, itemContent: string) => `${++index}. ${itemContent}\n`,
|
||||
) + '\n';
|
||||
});
|
||||
|
||||
// Convert bold and italic
|
||||
md = md.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
|
||||
md = md.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
|
||||
md = md.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
|
||||
md = md.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
|
||||
|
||||
// Convert code blocks
|
||||
md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '```\n$1\n```\n\n');
|
||||
md = md.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
|
||||
|
||||
// Convert blockquotes
|
||||
md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (match, content) => {
|
||||
return content.split('\n').map((line: string) => `> ${line}`).join('\n') + '\n\n';
|
||||
});
|
||||
|
||||
// Convert line breaks
|
||||
md = md.replace(/<br\s*\/?>/gi, '\n');
|
||||
|
||||
// Remove remaining HTML tags
|
||||
md = md.replace(/<[^>]+>/g, '');
|
||||
|
||||
// Decode HTML entities
|
||||
md = md.replace(/ /g, ' ');
|
||||
md = md.replace(/</g, '<');
|
||||
md = md.replace(/>/g, '>');
|
||||
md = md.replace(/&/g, '&');
|
||||
md = md.replace(/"/g, '"');
|
||||
|
||||
// Clean up extra whitespace
|
||||
md = md.replace(/\n{3,}/g, '\n\n');
|
||||
md = md.trim();
|
||||
|
||||
return md;
|
||||
}
|
||||
import { downloadArticle } from '../../download/article-download.js';
|
||||
|
||||
cli({
|
||||
site: 'zhihu',
|
||||
@@ -92,18 +15,17 @@ cli({
|
||||
domain: 'zhuanlan.zhihu.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'url', required: true, positional: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
|
||||
{ name: 'url', required: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
|
||||
{ name: 'output', default: './zhihu-articles', help: 'Output directory' },
|
||||
{ name: 'download-images', type: 'boolean', default: false, help: 'Download images locally' },
|
||||
],
|
||||
columns: ['title', 'author', 'status', 'size'],
|
||||
columns: ['title', 'author', 'publish_time', 'status', 'size'],
|
||||
func: async (page, kwargs) => {
|
||||
const url = kwargs.url;
|
||||
const output = kwargs.output;
|
||||
const downloadImages = kwargs['download-images'];
|
||||
|
||||
// Navigate to article page
|
||||
await page.goto(url);
|
||||
await page.wait(3);
|
||||
|
||||
// Extract article content
|
||||
const data = await page.evaluate(`
|
||||
@@ -111,9 +33,9 @@ cli({
|
||||
const result = {
|
||||
title: '',
|
||||
author: '',
|
||||
content: '',
|
||||
publishTime: '',
|
||||
images: []
|
||||
contentHtml: '',
|
||||
imageUrls: []
|
||||
};
|
||||
|
||||
// Get title
|
||||
@@ -131,13 +53,13 @@ cli({
|
||||
// Get content HTML
|
||||
const contentEl = document.querySelector('.Post-RichTextContainer, .RichText, .ArticleContent');
|
||||
if (contentEl) {
|
||||
result.content = contentEl.innerHTML;
|
||||
result.contentHtml = contentEl.innerHTML;
|
||||
|
||||
// Extract image URLs
|
||||
contentEl.querySelectorAll('img').forEach(img => {
|
||||
const src = img.getAttribute('data-original') || img.getAttribute('data-actualsrc') || img.src;
|
||||
if (src && !src.includes('data:image')) {
|
||||
result.images.push(src);
|
||||
result.imageUrls.push(src);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -146,77 +68,20 @@ cli({
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!data || !data.content) {
|
||||
return [{
|
||||
title: 'Error',
|
||||
author: '-',
|
||||
status: 'failed',
|
||||
size: 'Could not extract article content',
|
||||
}];
|
||||
}
|
||||
|
||||
// Create output directory
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
|
||||
// Convert HTML to Markdown
|
||||
let markdown = htmlToMarkdown(data.content);
|
||||
|
||||
// Create frontmatter
|
||||
const frontmatter = [
|
||||
'---',
|
||||
`title: "${data.title.replace(/"/g, '\\"')}"`,
|
||||
`author: "${data.author.replace(/"/g, '\\"')}"`,
|
||||
`source: "${url}"`,
|
||||
data.publishTime ? `date: "${data.publishTime}"` : '',
|
||||
'---',
|
||||
'',
|
||||
].filter(Boolean).join('\n');
|
||||
|
||||
// Download images if requested
|
||||
if (downloadImages && data.images && data.images.length > 0) {
|
||||
const imagesDir = path.join(output, 'images');
|
||||
fs.mkdirSync(imagesDir, { recursive: true });
|
||||
|
||||
const cookies = formatCookieHeader(await page.getCookies({ domain: 'zhihu.com' }));
|
||||
|
||||
for (let i = 0; i < data.images.length; i++) {
|
||||
const imgUrl = data.images[i];
|
||||
const ext = imgUrl.match(/\.(jpg|jpeg|png|gif|webp)/i)?.[1] || 'jpg';
|
||||
const imgFilename = `img_${i + 1}.${ext}`;
|
||||
const imgPath = path.join(imagesDir, imgFilename);
|
||||
|
||||
try {
|
||||
await httpDownload(imgUrl, imgPath, {
|
||||
cookies,
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
// Replace image URL in markdown with local path
|
||||
markdown = markdown.replace(
|
||||
new RegExp(imgUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'),
|
||||
`./images/${imgFilename}`,
|
||||
);
|
||||
} catch {
|
||||
// Keep original URL if download fails
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write markdown file
|
||||
const safeTitle = sanitizeFilename(data.title, 100);
|
||||
const filename = `${safeTitle}.md`;
|
||||
const filePath = path.join(output, filename);
|
||||
|
||||
const fullContent = frontmatter + '\n' + markdown;
|
||||
fs.writeFileSync(filePath, fullContent, 'utf-8');
|
||||
|
||||
const size = Buffer.byteLength(fullContent, 'utf-8');
|
||||
|
||||
return [{
|
||||
title: data.title,
|
||||
author: data.author,
|
||||
status: 'success',
|
||||
size: formatBytes(size),
|
||||
}];
|
||||
return downloadArticle(
|
||||
{
|
||||
title: data?.title || '',
|
||||
author: data?.author,
|
||||
publishTime: data?.publishTime,
|
||||
sourceUrl: url,
|
||||
contentHtml: data?.contentHtml || '',
|
||||
imageUrls: data?.imageUrls,
|
||||
},
|
||||
{
|
||||
output: kwargs.output,
|
||||
downloadImages: kwargs['download-images'],
|
||||
imageHeaders: { Referer: 'https://zhuanlan.zhihu.com/' },
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
/**
|
||||
* Article download helper — shared logic for downloading articles as Markdown.
|
||||
*
|
||||
* Used by: zhihu/download, weixin/download, and future article adapters.
|
||||
*
|
||||
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import TurndownService from 'turndown';
|
||||
import { httpDownload, sanitizeFilename } from './index.js';
|
||||
import { formatBytes } from './progress.js';
|
||||
|
||||
const IMAGE_CONCURRENCY = 5;
|
||||
|
||||
// ============================================================
|
||||
// Types
|
||||
// ============================================================
|
||||
|
||||
export interface ArticleData {
|
||||
title: string;
|
||||
author?: string;
|
||||
publishTime?: string;
|
||||
sourceUrl?: string;
|
||||
contentHtml: string;
|
||||
/** Pre-extracted code blocks to restore after Markdown conversion */
|
||||
codeBlocks?: Array<{ lang: string; code: string }>;
|
||||
/** Image URLs found in the article (pre-collected from DOM) */
|
||||
imageUrls?: string[];
|
||||
}
|
||||
|
||||
export interface FrontmatterLabels {
|
||||
author?: string;
|
||||
publishTime?: string;
|
||||
sourceUrl?: string;
|
||||
}
|
||||
|
||||
export interface ArticleDownloadOptions {
|
||||
output: string;
|
||||
downloadImages?: boolean;
|
||||
/** Extra headers for image downloads (e.g. { Referer: '...' }) */
|
||||
imageHeaders?: Record<string, string>;
|
||||
maxTitleLength?: number;
|
||||
/** Custom TurndownService configuration callback */
|
||||
configureTurndown?: (td: TurndownService) => void;
|
||||
/** Custom image extension detector (default: infer from URL extension) */
|
||||
detectImageExt?: (url: string) => string;
|
||||
/** Custom frontmatter labels (default: Chinese labels) */
|
||||
frontmatterLabels?: FrontmatterLabels;
|
||||
}
|
||||
|
||||
export interface ArticleDownloadResult {
|
||||
title: string;
|
||||
author: string;
|
||||
publish_time: string;
|
||||
status: string;
|
||||
size: string;
|
||||
}
|
||||
|
||||
const DEFAULT_LABELS: Required<FrontmatterLabels> = {
|
||||
author: '作者',
|
||||
publishTime: '发布时间',
|
||||
sourceUrl: '原文链接',
|
||||
};
|
||||
|
||||
// ============================================================
|
||||
// Markdown Conversion
|
||||
// ============================================================
|
||||
|
||||
function createTurndown(configure?: (td: TurndownService) => void): TurndownService {
|
||||
const td = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
codeBlockStyle: 'fenced',
|
||||
bulletListMarker: '-',
|
||||
});
|
||||
td.addRule('linebreak', {
|
||||
filter: 'br',
|
||||
replacement: () => '\n',
|
||||
});
|
||||
if (configure) configure(td);
|
||||
return td;
|
||||
}
|
||||
|
||||
function convertToMarkdown(
|
||||
contentHtml: string,
|
||||
codeBlocks: Array<{ lang: string; code: string }>,
|
||||
configure?: (td: TurndownService) => void,
|
||||
): string {
|
||||
const td = createTurndown(configure);
|
||||
let md = td.turndown(contentHtml);
|
||||
|
||||
// Restore code block placeholders
|
||||
codeBlocks.forEach((block, i) => {
|
||||
const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
|
||||
const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
|
||||
md = md.replace(placeholder, fenced);
|
||||
});
|
||||
|
||||
// Clean up
|
||||
md = md.replace(/\u00a0/g, ' ');
|
||||
md = md.replace(/\n{4,}/g, '\n\n\n');
|
||||
md = md.replace(/[ \t]+$/gm, '');
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
function replaceImageUrls(md: string, urlMap: Record<string, string>): string {
|
||||
return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
|
||||
const local = urlMap[imgUrl];
|
||||
return local ? `` : match;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Image Downloading
|
||||
// ============================================================
|
||||
|
||||
function defaultDetectImageExt(url: string): string {
|
||||
const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
|
||||
return extMatch ? extMatch[1] : 'jpg';
|
||||
}
|
||||
|
||||
async function downloadImages(
|
||||
imgUrls: string[],
|
||||
imgDir: string,
|
||||
headers?: Record<string, string>,
|
||||
detectExt?: (url: string) => string,
|
||||
): Promise<Record<string, string>> {
|
||||
const urlMap: Record<string, string> = {};
|
||||
if (imgUrls.length === 0) return urlMap;
|
||||
|
||||
const detect = detectExt || defaultDetectImageExt;
|
||||
|
||||
// Deduplicate image URLs
|
||||
const seen = new Set<string>();
|
||||
const uniqueUrls = imgUrls.filter(url => {
|
||||
if (seen.has(url)) return false;
|
||||
seen.add(url);
|
||||
return true;
|
||||
});
|
||||
|
||||
for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
|
||||
const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
|
||||
const results = await Promise.all(
|
||||
batch.map(async (rawUrl, j) => {
|
||||
const index = i + j + 1;
|
||||
let imgUrl = rawUrl;
|
||||
if (imgUrl.startsWith('//')) imgUrl = `https:${imgUrl}`;
|
||||
|
||||
const ext = detect(imgUrl);
|
||||
const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
|
||||
const filepath = path.join(imgDir, filename);
|
||||
|
||||
try {
|
||||
const result = await httpDownload(imgUrl, filepath, {
|
||||
headers,
|
||||
timeout: 15000,
|
||||
});
|
||||
if (result.success) {
|
||||
return { remoteUrl: rawUrl, localPath: `images/${filename}` };
|
||||
}
|
||||
} catch {
|
||||
// Skip failed downloads
|
||||
}
|
||||
return null;
|
||||
}),
|
||||
);
|
||||
|
||||
for (const r of results) {
|
||||
if (r) urlMap[r.remoteUrl] = r.localPath;
|
||||
}
|
||||
}
|
||||
return urlMap;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Main API
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Download an article to Markdown with optional image localization.
|
||||
*
|
||||
* Handles the full pipeline:
|
||||
* 1. HTML → Markdown (via TurndownService)
|
||||
* 2. Code block placeholder restoration
|
||||
* 3. Batch image downloading with concurrency + deduplication
|
||||
* 4. Image URL replacement in Markdown
|
||||
* 5. Frontmatter generation (customizable labels)
|
||||
* 6. File write
|
||||
*/
|
||||
export async function downloadArticle(
|
||||
data: ArticleData,
|
||||
options: ArticleDownloadOptions,
|
||||
): Promise<ArticleDownloadResult[]> {
|
||||
const {
|
||||
output,
|
||||
downloadImages: shouldDownloadImages = true,
|
||||
imageHeaders,
|
||||
maxTitleLength = 80,
|
||||
configureTurndown,
|
||||
detectImageExt,
|
||||
frontmatterLabels,
|
||||
} = options;
|
||||
|
||||
const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
|
||||
|
||||
if (!data.title) {
|
||||
return [{
|
||||
title: 'Error',
|
||||
author: '-',
|
||||
publish_time: '-',
|
||||
status: 'failed — no title',
|
||||
size: '-',
|
||||
}];
|
||||
}
|
||||
|
||||
if (!data.contentHtml) {
|
||||
return [{
|
||||
title: data.title,
|
||||
author: data.author || '-',
|
||||
publish_time: data.publishTime || '-',
|
||||
status: 'failed — no content',
|
||||
size: '-',
|
||||
}];
|
||||
}
|
||||
|
||||
// Convert HTML to Markdown
|
||||
let markdown = convertToMarkdown(
|
||||
data.contentHtml,
|
||||
data.codeBlocks || [],
|
||||
configureTurndown,
|
||||
);
|
||||
|
||||
// Prepare output directory
|
||||
const safeTitle = sanitizeFilename(data.title, maxTitleLength);
|
||||
const articleDir = path.join(output, safeTitle);
|
||||
fs.mkdirSync(articleDir, { recursive: true });
|
||||
|
||||
// Download images
|
||||
if (shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
|
||||
const imagesDir = path.join(articleDir, 'images');
|
||||
fs.mkdirSync(imagesDir, { recursive: true });
|
||||
|
||||
const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
|
||||
markdown = replaceImageUrls(markdown, urlMap);
|
||||
}
|
||||
|
||||
// Build frontmatter with customizable labels
|
||||
const headerLines = [`# ${data.title}`, ''];
|
||||
if (data.author) headerLines.push(`> ${labels.author}: ${data.author}`);
|
||||
if (data.publishTime) headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
|
||||
if (data.sourceUrl) headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
|
||||
headerLines.push('', '---', '');
|
||||
|
||||
const fullContent = headerLines.join('\n') + markdown;
|
||||
|
||||
// Write file
|
||||
const filename = `${safeTitle}.md`;
|
||||
const filePath = path.join(articleDir, filename);
|
||||
fs.writeFileSync(filePath, fullContent, 'utf-8');
|
||||
|
||||
const size = Buffer.byteLength(fullContent, 'utf-8');
|
||||
|
||||
return [{
|
||||
title: data.title,
|
||||
author: data.author || '-',
|
||||
publish_time: data.publishTime || '-',
|
||||
status: 'success',
|
||||
size: formatBytes(size),
|
||||
}];
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Media download helper — shared logic for batch downloading images/videos.
|
||||
*
|
||||
* Used by: xiaohongshu/download, twitter/download, bilibili/download,
|
||||
* and future media adapters.
|
||||
*
|
||||
* Flow: MediaItem[] → DownloadProgressTracker → httpDownload/ytdlpDownload → results
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
httpDownload,
|
||||
ytdlpDownload,
|
||||
checkYtdlp,
|
||||
getTempDir,
|
||||
exportCookiesToNetscape,
|
||||
} from './index.js';
|
||||
import type { BrowserCookie } from '../types.js';
|
||||
import { DownloadProgressTracker, formatBytes } from './progress.js';
|
||||
|
||||
// ============================================================
|
||||
// Types
|
||||
// ============================================================
|
||||
|
||||
export interface MediaItem {
|
||||
type: 'image' | 'video' | 'video-tweet' | 'video-ytdlp';
|
||||
url: string;
|
||||
/** Optional custom filename (without directory) */
|
||||
filename?: string;
|
||||
}
|
||||
|
||||
export interface MediaDownloadOptions {
|
||||
output: string;
|
||||
/** Subdirectory inside output */
|
||||
subdir?: string;
|
||||
/** Cookie string for HTTP downloads */
|
||||
cookies?: string;
|
||||
/** Raw browser cookies — auto-exported to Netscape for yt-dlp, auto-cleaned up */
|
||||
browserCookies?: BrowserCookie[];
|
||||
/** Timeout in ms (default: 30000 for images, 60000 for videos) */
|
||||
timeout?: number;
|
||||
/** File name prefix (default: 'download') */
|
||||
filenamePrefix?: string;
|
||||
/** Extra yt-dlp args */
|
||||
ytdlpExtraArgs?: string[];
|
||||
/** Whether to show progress (default: true) */
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
export interface MediaDownloadResult {
|
||||
index: number;
|
||||
type: string;
|
||||
status: string;
|
||||
size: string;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Main API
|
||||
// ============================================================
|
||||
|
||||
/**
|
||||
* Batch download media files with progress tracking.
|
||||
*
|
||||
* Handles:
|
||||
* - DownloadProgressTracker for terminal UX
|
||||
* - Automatic httpDownload vs ytdlpDownload routing via MediaItem.type
|
||||
* - Cookie export to Netscape format for yt-dlp (auto-cleanup)
|
||||
* - Directory creation
|
||||
* - Error handling with per-file results
|
||||
*/
|
||||
export async function downloadMedia(
|
||||
items: MediaItem[],
|
||||
options: MediaDownloadOptions,
|
||||
): Promise<MediaDownloadResult[]> {
|
||||
const {
|
||||
output,
|
||||
subdir,
|
||||
cookies,
|
||||
browserCookies,
|
||||
timeout,
|
||||
filenamePrefix = 'download',
|
||||
ytdlpExtraArgs = [],
|
||||
verbose = true,
|
||||
} = options;
|
||||
|
||||
if (!items || items.length === 0) {
|
||||
return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
|
||||
}
|
||||
|
||||
// Create output directory
|
||||
const outputDir = subdir ? path.join(output, subdir) : output;
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
// Pre-check yt-dlp availability (once, not per-item)
|
||||
const hasYtdlp = checkYtdlp();
|
||||
|
||||
// Auto-export browser cookies to Netscape format for yt-dlp
|
||||
let cookiesFile: string | undefined;
|
||||
const needsYtdlp = items.some(m => m.type === 'video-tweet' || m.type === 'video-ytdlp');
|
||||
if (needsYtdlp && browserCookies && browserCookies.length > 0) {
|
||||
const tempDir = getTempDir();
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
cookiesFile = path.join(tempDir, `media_cookies_${Date.now()}.txt`);
|
||||
exportCookiesToNetscape(browserCookies, cookiesFile);
|
||||
}
|
||||
|
||||
const tracker = new DownloadProgressTracker(items.length, verbose);
|
||||
const results: MediaDownloadResult[] = [];
|
||||
|
||||
try {
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const media = items[i];
|
||||
const isVideo = media.type !== 'image';
|
||||
const ext = isVideo ? 'mp4' : 'jpg';
|
||||
const filename = media.filename || `${filenamePrefix}_${i + 1}.${ext}`;
|
||||
const destPath = path.join(outputDir, filename);
|
||||
|
||||
const progressBar = tracker.onFileStart(filename, i);
|
||||
|
||||
try {
|
||||
let result: { success: boolean; size: number; error?: string };
|
||||
const useYtdlp = (media.type === 'video-tweet' || media.type === 'video-ytdlp') && hasYtdlp;
|
||||
|
||||
if (useYtdlp) {
|
||||
result = await ytdlpDownload(media.url, destPath, {
|
||||
cookiesFile,
|
||||
extraArgs: ytdlpExtraArgs,
|
||||
onProgress: (percent) => {
|
||||
if (progressBar) progressBar.update(percent, 100);
|
||||
},
|
||||
});
|
||||
} else {
|
||||
// Direct HTTP download for images and direct video URLs
|
||||
const dlTimeout = timeout || (isVideo ? 60000 : 30000);
|
||||
result = await httpDownload(media.url, destPath, {
|
||||
cookies,
|
||||
timeout: dlTimeout,
|
||||
onProgress: (received, total) => {
|
||||
if (progressBar) progressBar.update(received, total);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (progressBar) {
|
||||
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
||||
}
|
||||
tracker.onFileComplete(result.success);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type === 'video-tweet' || media.type === 'video-ytdlp' ? 'video' : media.type,
|
||||
status: result.success ? 'success' : 'failed',
|
||||
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (progressBar) progressBar.fail(err.message);
|
||||
tracker.onFileComplete(false);
|
||||
|
||||
results.push({
|
||||
index: i + 1,
|
||||
type: media.type,
|
||||
status: 'failed',
|
||||
size: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
tracker.finish();
|
||||
|
||||
// Auto-cleanup exported cookies file
|
||||
if (cookiesFile && fs.existsSync(cookiesFile)) {
|
||||
fs.unlinkSync(cookiesFile);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
Reference in New Issue
Block a user