From 7250de9228e6b2e78d486f9e97c614cfe2b2ce43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=95=85=E7=92=83?= Date: Mon, 27 Jul 2026 16:56:46 +0800 Subject: [PATCH] feat: add changelog sync to oss --- .github/workflows/publish.yml | 2 + tools/release/lib/binary-release.mjs | 15 +- tools/release/lib/oss-direct-upload.mjs | 216 ++++++++++++++++++++---- 3 files changed, 203 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6907d97..5913687 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,6 +79,7 @@ jobs: BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }} BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }} BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }} + BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }} run: node tools/release/publish-stable.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} publish-channel: @@ -135,4 +136,5 @@ jobs: BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }} BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }} BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }} + BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }} run: node tools/release/publish-channel.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} --channel "${{ inputs.channel }}" diff --git a/tools/release/lib/binary-release.mjs b/tools/release/lib/binary-release.mjs index 0b1d37a..13f4da3 100644 --- a/tools/release/lib/binary-release.mjs +++ b/tools/release/lib/binary-release.mjs @@ -27,7 +27,11 @@ import { ROOT, readPackageJson, PACKAGES } from "./packages.mjs"; import { buildBinaryArtifacts, matrixAssetNames } from "./binary-build.mjs"; import { channelManifestFileName, normalizeModeChannel } from "./binary-options.mjs"; import { ensureGh, GITHUB_REPOSITORY, upsertRelease } from "./gh-release.mjs"; -import { maintainReleaseManifest, mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs"; +import { + maintainReleaseManifest, + mirrorReleaseAssetsToOss, + syncStaticFilesToOss, +} from "./oss-direct-upload.mjs"; const DEFAULT_DIR = join(ROOT, "dist-bin"); @@ -179,6 +183,10 @@ export async function releaseBinaryArtifacts(rawOptions = {}) { process.stdout.write(`[dry-run] ${dir} missing; planning expected assets\n`); planDryRunWithoutArtifacts({ version, mode, channel }); const plans = ossMirrorPlans({ dir, version, mode, channel, files: null }); + await syncStaticFilesToOss({ + filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")], + dryRun: true, + }); await mirrorReleaseAssetsToOss({ plans, dryRun: true }); if (mode === "stable") { await maintainReleaseManifest({ @@ -226,6 +234,11 @@ export async function releaseBinaryArtifacts(rawOptions = {}) { uploadChannel({ dir, version, channel, files, dryRun }); } + // Sync changelogs (and other static files) to OSS before the binary mirror. + await syncStaticFilesToOss({ + filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")], + dryRun, + }); // Push the exact Release assets straight to OSS from the runner, then // HEAD-reconcile. Stable releases additionally maintain release/manifest.json // (newer-version guard). Throws on failure — CI is the only OSS writer. diff --git a/tools/release/lib/oss-direct-upload.mjs b/tools/release/lib/oss-direct-upload.mjs index a821a86..c7d024e 100644 --- a/tools/release/lib/oss-direct-upload.mjs +++ b/tools/release/lib/oss-direct-upload.mjs @@ -26,6 +26,8 @@ * oss:GetObject on the release prefix * BAILIAN_OSS_BUCKET / BAILIAN_OSS_REGION / BAILIAN_RELEASE_PREFIX * —— required once the channel is enabled + * BAILIAN_STATIC_PREFIX —— prefix for static files (changelogs, etc.); + * same bucket/creds, separate namespace * BAILIAN_OSS_ENDPOINT —— optional request endpoint override; * public manifest URLs always use the * region endpoint @@ -69,6 +71,7 @@ function ossHost(cfg) { } function contentTypeFor(name) { + if (name.endsWith(".md")) return "text/markdown; charset=utf-8"; if (name.endsWith(".zip")) return "application/zip"; if (name.endsWith(".json")) return "application/json"; return "application/octet-stream"; @@ -204,6 +207,64 @@ async function getObjectJson(key, creds, cfg) { } } +/** + * Run async task factories with a bounded concurrency pool. + * Returns results in the same order as the input tasks array. + * (Same contract as packages/commands/src/commands/skill/shared.ts) + * + * @template T + * @param {Array<() => Promise>} tasks + * @param {number} limit + * @returns {Promise} + */ +async function runWithConcurrency(tasks, limit) { + const results = new Array(tasks.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < tasks.length) { + const currentIndex = nextIndex++; + results[currentIndex] = await tasks[currentIndex](); + } + } + + const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => worker()); + await Promise.all(workers); + return results; +} + +/** + * HEAD-reconcile: verify every uploaded object exists remotely with the same + * byte size as the local file. Runs HEAD requests concurrently. + * + * @param {Array<{ path: string, key: string }>} jobs + * @param {{ ak: string, sk: string }} creds + * @param {object} cfg + * @param {string} label Context for error messages (e.g. "release", "static-files") + */ +async function reconcileUploads(jobs, creds, cfg, label) { + const results = await runWithConcurrency( + jobs.map((job) => async () => { + const remote = await headObjectSize(job.key, creds, cfg); + const local = statSync(job.path).size; + if (remote !== local) { + return { + ok: false, + key: job.key, + error: `OSS ${label} reconcile mismatch for ${job.key}: local ${local}B vs remote ${remote ?? "missing"}`, + }; + } + return { ok: true, key: job.key }; + }), + 4, + ); + const mismatches = results.filter((result) => !result.ok); + if (mismatches.length > 0) { + throw new Error(mismatches.map((item) => item.error).join("\n")); + } + process.stdout.write(`${label} reconcile ok: ${jobs.length}/${jobs.length} object(s) verified\n`); +} + /** * Upload release assets to OSS under `//`, then * HEAD-reconcile every object against the local byte size. @@ -246,48 +307,40 @@ export async function mirrorReleaseAssetsToOss({ plans, dryRun = false }) { return { uploaded: 0, skipped: false }; } - // Bounded worker pool; collect failures, then throw once at the end. - const failed = []; - let cursor = 0; - const worker = async () => { - while (true) { - const index = cursor++; - if (index >= jobs.length) return; - const { path, key } = jobs[index]; + const results = await runWithConcurrency( + jobs.map((job) => async () => { const startedAt = Date.now(); try { - const body = readFileSync(path); - await putWithRetry({ creds, cfg, key, body, contentType: contentTypeFor(key) }); + const body = readFileSync(job.path); + await putWithRetry({ + creds, + cfg, + key: job.key, + body, + contentType: contentTypeFor(job.key), + }); process.stdout.write( - ` [oss] ok ${key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`, + ` [oss] ok ${job.key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`, ); - } catch (err) { - failed.push({ key, error: err.message }); - process.stdout.write(` [oss] FAIL ${key}: ${err.message}\n`); + return { ok: true, key: job.key }; + } catch (error) { + process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`); + return { ok: false, key: job.key, error: error.message }; } - } - }; - await Promise.all(Array.from({ length: Math.min(4, jobs.length) }, () => worker())); + }), + 4, + ); + const failed = results.filter((result) => !result.ok); if (failed.length > 0) { throw new Error( `OSS upload failed for ${failed.length}/${jobs.length} object(s): ${failed - .map((f) => f.key) + .map((item) => item.key) .join(", ")}`, ); } - // Reconcile: every uploaded object must exist remotely with the local byte size. - for (const { path, key } of jobs) { - const remote = await headObjectSize(key, creds, cfg); - const local = statSync(path).size; - if (remote !== local) { - throw new Error( - `OSS reconcile mismatch for ${key}: local ${local}B vs remote ${remote ?? "missing"}`, - ); - } - } - process.stdout.write(`reconcile ok: ${jobs.length}/${jobs.length} object(s) verified on OSS\n`); + await reconcileUploads(jobs, creds, cfg, "release"); return { uploaded: jobs.length, skipped: false }; } @@ -354,3 +407,108 @@ export async function maintainReleaseManifest({ } return { updated: true, latest: tag }; } + +/** + * Resolve the OSS context for the static-files channel. Same bucket/creds as + * the release channel but uses BAILIAN_STATIC_PREFIX instead of + * BAILIAN_RELEASE_PREFIX. Returns null when credentials are absent (channel + * disabled); throws when creds exist but required config is incomplete. + */ +function staticOssContext() { + const ak = process.env.BAILIAN_OSS_AK?.trim(); + const sk = process.env.BAILIAN_OSS_SK?.trim(); + if (!ak || !sk) return null; + const cfg = { + bucket: process.env.BAILIAN_OSS_BUCKET?.trim() || "", + region: process.env.BAILIAN_OSS_REGION?.trim() || "", + endpoint: process.env.BAILIAN_OSS_ENDPOINT?.trim() || "", + prefix: process.env.BAILIAN_STATIC_PREFIX?.trim() || "", + }; + const missing = [ + ["BAILIAN_OSS_BUCKET", cfg.bucket], + ["BAILIAN_OSS_REGION", cfg.region], + ["BAILIAN_STATIC_PREFIX", cfg.prefix], + ] + .filter(([, value]) => !value) + .map(([name]) => name); + if (missing.length > 0) { + throw new Error(`OSS static-files channel misconfigured; missing env: ${missing.join(", ")}`); + } + return { creds: { ak, sk }, cfg }; +} + +/** + * Sync a list of local files to OSS under `/`. + * Generic utility for any repo files that need to be mirrored to the static + * prefix (changelogs today; docs, banners, etc. in the future). + * + * Gating: BAILIAN_OSS_AK/SK unset → warn + no-op. BAILIAN_STATIC_PREFIX unset + * (with creds present) → throw (misconfiguration). + * + * @param {{ + * filePaths: string[], + * dryRun?: boolean, + * }} options + * @returns {Promise<{ uploaded: number, skipped: boolean }>} + */ +export async function syncStaticFilesToOss({ filePaths, dryRun = false }) { + const ctx = staticOssContext(); + if (!ctx) { + process.stdout.write("\n[warn] BAILIAN_OSS_AK/SK unset; skip static-files sync to OSS\n"); + return { uploaded: 0, skipped: true }; + } + const { creds, cfg } = ctx; + + const jobs = filePaths.map((path) => ({ + path, + key: `${cfg.prefix}/${basename(path)}`, + })); + if (jobs.length === 0) return { uploaded: 0, skipped: true }; + + process.stdout.write( + `\n==> OSS static-files sync: ${jobs.length} file(s) → ${cfg.bucket}/${cfg.prefix}/\n`, + ); + + if (dryRun) { + for (const job of jobs) { + process.stdout.write(`[dry-run] PUT oss://${cfg.bucket}/${job.key}\n`); + } + return { uploaded: 0, skipped: false }; + } + + const results = await runWithConcurrency( + jobs.map((job) => async () => { + const startedAt = Date.now(); + try { + const body = readFileSync(job.path); + await putWithRetry({ + creds, + cfg, + key: job.key, + body, + contentType: contentTypeFor(job.key), + }); + process.stdout.write( + ` [oss] ok ${job.key} (${(body.length / 1024).toFixed(1)}KB, ${Date.now() - startedAt}ms)\n`, + ); + return { ok: true, key: job.key }; + } catch (error) { + process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`); + return { ok: false, key: job.key, error: error.message }; + } + }), + 4, + ); + + const failed = results.filter((result) => !result.ok); + if (failed.length > 0) { + throw new Error( + `OSS static-files sync failed for ${failed.length}/${jobs.length} file(s): ${failed + .map((item) => item.key) + .join(", ")}`, + ); + } + + await reconcileUploads(jobs, creds, cfg, "static-files"); + return { uploaded: jobs.length, skipped: false }; +}