diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6907d97..5913687 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,6 +79,7 @@ jobs: BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }} BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }} BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }} + BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }} run: node tools/release/publish-stable.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} publish-channel: @@ -135,4 +136,5 @@ jobs: BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }} BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }} BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }} + BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }} run: node tools/release/publish-channel.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} --channel "${{ inputs.channel }}" diff --git a/packages/core/src/install/cdn.ts b/packages/core/src/install/cdn.ts index d9996f4..8460a1e 100644 --- a/packages/core/src/install/cdn.ts +++ b/packages/core/src/install/cdn.ts @@ -5,7 +5,7 @@ * * Layout under the base: * v/.zip —— immutable per-version binaries + SHA256SUMS - * manifest.json —— stable pointer { latest, releasedAt, assets } + * manifest.json —— stable pointer; same rolling-manifest shape as latest.json * latest.json —— stable rolling manifest (os-arch keyed, sha256) * .json —— per-channel rolling manifests (beta versions) * diff --git a/tools/release/lib/binary-release.mjs b/tools/release/lib/binary-release.mjs index 0b1d37a..7143db8 100644 --- a/tools/release/lib/binary-release.mjs +++ b/tools/release/lib/binary-release.mjs @@ -11,7 +11,8 @@ * * Re-runs are idempotent via `gh release upload --clobber` (see gh-release.mjs). * After the GitHub upload the same assets are pushed straight to OSS from the - * runner, HEAD-reconciled, and (stable only) release/manifest.json is updated — + * runner, HEAD-reconciled, and (stable only) release/manifest.json + latest.json + * are rewritten with the SAME rolling-manifest body as channel `.json` — * all in-process, no external FC (see oss-direct-upload.mjs). * * Called by publish-stable.mjs / publish-channel.mjs. @@ -20,14 +21,18 @@ * node tools/release/lib/binary-release.mjs --mode channel --channel beta --dry-run */ import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { basename, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs as parseCliArgs } from "node:util"; import { ROOT, readPackageJson, PACKAGES } from "./packages.mjs"; import { buildBinaryArtifacts, matrixAssetNames } from "./binary-build.mjs"; import { channelManifestFileName, normalizeModeChannel } from "./binary-options.mjs"; import { ensureGh, GITHUB_REPOSITORY, upsertRelease } from "./gh-release.mjs"; -import { maintainReleaseManifest, mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs"; +import { + maintainReleaseManifest, + mirrorReleaseAssetsToOss, + syncStaticFilesToOss, +} from "./oss-direct-upload.mjs"; const DEFAULT_DIR = join(ROOT, "dist-bin"); @@ -179,11 +184,14 @@ export async function releaseBinaryArtifacts(rawOptions = {}) { process.stdout.write(`[dry-run] ${dir} missing; planning expected assets\n`); planDryRunWithoutArtifacts({ version, mode, channel }); const plans = ossMirrorPlans({ dir, version, mode, channel, files: null }); + await syncStaticFilesToOss({ + filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")], + dryRun: true, + }); await mirrorReleaseAssetsToOss({ plans, dryRun: true }); if (mode === "stable") { await maintainReleaseManifest({ tag: `v${version}`, - assetNames: plans[0].paths.map((path) => basename(path)), channelJsonPath: null, dryRun: true, }); @@ -226,6 +234,11 @@ export async function releaseBinaryArtifacts(rawOptions = {}) { uploadChannel({ dir, version, channel, files, dryRun }); } + // Sync changelogs (and other static files) to OSS before the binary mirror. + await syncStaticFilesToOss({ + filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")], + dryRun, + }); // Push the exact Release assets straight to OSS from the runner, then // HEAD-reconcile. Stable releases additionally maintain release/manifest.json // (newer-version guard). Throws on failure — CI is the only OSS writer. @@ -234,7 +247,6 @@ export async function releaseBinaryArtifacts(rawOptions = {}) { if (mode === "stable" && !mirror.skipped) { await maintainReleaseManifest({ tag: `v${version}`, - assetNames: plans[0].paths.map((path) => basename(path)), channelJsonPath: join(dir, rollingManifest), dryRun, }); diff --git a/tools/release/lib/oss-direct-upload.mjs b/tools/release/lib/oss-direct-upload.mjs index a821a86..ae142d0 100644 --- a/tools/release/lib/oss-direct-upload.mjs +++ b/tools/release/lib/oss-direct-upload.mjs @@ -8,9 +8,11 @@ * channel manifests (`.json`) go to the prefix root (empty tag). * - After upload, every object is HEAD-verified against the local byte size * (reconciliation — the runner has the ground-truth artifacts on disk). - * - Stable only: when the tag is a NEWER version than manifest.latest + * - Stable only: when the tag is a NEWER version than the current manifest * (compareVersions), rewrite `/manifest.json` and the rolling - * `/latest.json`. Channel/prerelease never touches either. + * `/latest.json` — both carry the SAME rolling-manifest body + * written by binary-build.mjs, so manifest.json shares the channel + * `.json` shape. Channel/prerelease never touches either. * * Zero-dependency: OSS V1 header signature (HMAC-SHA1) over plain fetch. * @@ -26,6 +28,8 @@ * oss:GetObject on the release prefix * BAILIAN_OSS_BUCKET / BAILIAN_OSS_REGION / BAILIAN_RELEASE_PREFIX * —— required once the channel is enabled + * BAILIAN_STATIC_PREFIX —— prefix for static files (changelogs, etc.); + * same bucket/creds, separate namespace * BAILIAN_OSS_ENDPOINT —— optional request endpoint override; * public manifest URLs always use the * region endpoint @@ -69,6 +73,7 @@ function ossHost(cfg) { } function contentTypeFor(name) { + if (name.endsWith(".md")) return "text/markdown; charset=utf-8"; if (name.endsWith(".zip")) return "application/zip"; if (name.endsWith(".json")) return "application/json"; return "application/octet-stream"; @@ -98,36 +103,6 @@ export function compareVersions(a, b) { return 0; } -/** Format now (or a given time) as an Asia/Shanghai +08:00 string. */ -function toBeijing(input) { - const d = input ? new Date(input) : new Date(); - const parts = new Intl.DateTimeFormat("en-US", { - timeZone: "Asia/Shanghai", - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - hour12: false, - }).formatToParts(d); - const g = (t) => parts.find((p) => p.type === t)?.value ?? "00"; - return `${g("year")}-${g("month")}-${g("day")}T${g("hour")}:${g("minute")}:${g("second")}+08:00`; -} - -/** - * Build the manifest.json contents. Public URLs always use the durable region - * endpoint (never the acceleration endpoint used for uploads). - */ -function buildManifest(tag, releasedAt, assetNames, cfg) { - const base = `https://${cfg.bucket}.${cfg.region}.aliyuncs.com/${cfg.prefix}/${tag}`; - const assets = {}; - for (const name of [...assetNames].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))) { - assets[name] = `${base}/${encodeURIComponent(name)}`; - } - return { latest: tag, releasedAt, assets }; -} - /** * Signed OSS request (V1 header signature). Keys here are [A-Za-z0-9._/-] only, * so no URL encoding is needed and the signed resource matches the request path. @@ -204,6 +179,64 @@ async function getObjectJson(key, creds, cfg) { } } +/** + * Run async task factories with a bounded concurrency pool. + * Returns results in the same order as the input tasks array. + * (Same contract as packages/commands/src/commands/skill/shared.ts) + * + * @template T + * @param {Array<() => Promise>} tasks + * @param {number} limit + * @returns {Promise} + */ +async function runWithConcurrency(tasks, limit) { + const results = Array.from({ length: tasks.length }); + let nextIndex = 0; + + async function worker() { + while (nextIndex < tasks.length) { + const currentIndex = nextIndex++; + results[currentIndex] = await tasks[currentIndex](); + } + } + + const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => worker()); + await Promise.all(workers); + return results; +} + +/** + * HEAD-reconcile: verify every uploaded object exists remotely with the same + * byte size as the local file. Runs HEAD requests concurrently. + * + * @param {Array<{ path: string, key: string }>} jobs + * @param {{ ak: string, sk: string }} creds + * @param {object} cfg + * @param {string} label Context for error messages (e.g. "release", "static-files") + */ +async function reconcileUploads(jobs, creds, cfg, label) { + const results = await runWithConcurrency( + jobs.map((job) => async () => { + const remote = await headObjectSize(job.key, creds, cfg); + const local = statSync(job.path).size; + if (remote !== local) { + return { + ok: false, + key: job.key, + error: `OSS ${label} reconcile mismatch for ${job.key}: local ${local}B vs remote ${remote ?? "missing"}`, + }; + } + return { ok: true, key: job.key }; + }), + 4, + ); + const mismatches = results.filter((result) => !result.ok); + if (mismatches.length > 0) { + throw new Error(mismatches.map((item) => item.error).join("\n")); + } + process.stdout.write(`${label} reconcile ok: ${jobs.length}/${jobs.length} object(s) verified\n`); +} + /** * Upload release assets to OSS under `//`, then * HEAD-reconcile every object against the local byte size. @@ -246,70 +279,59 @@ export async function mirrorReleaseAssetsToOss({ plans, dryRun = false }) { return { uploaded: 0, skipped: false }; } - // Bounded worker pool; collect failures, then throw once at the end. - const failed = []; - let cursor = 0; - const worker = async () => { - while (true) { - const index = cursor++; - if (index >= jobs.length) return; - const { path, key } = jobs[index]; + const results = await runWithConcurrency( + jobs.map((job) => async () => { const startedAt = Date.now(); try { - const body = readFileSync(path); - await putWithRetry({ creds, cfg, key, body, contentType: contentTypeFor(key) }); + const body = readFileSync(job.path); + await putWithRetry({ + creds, + cfg, + key: job.key, + body, + contentType: contentTypeFor(job.key), + }); process.stdout.write( - ` [oss] ok ${key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`, + ` [oss] ok ${job.key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`, ); - } catch (err) { - failed.push({ key, error: err.message }); - process.stdout.write(` [oss] FAIL ${key}: ${err.message}\n`); + return { ok: true, key: job.key }; + } catch (error) { + process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`); + return { ok: false, key: job.key, error: error.message }; } - } - }; - await Promise.all(Array.from({ length: Math.min(4, jobs.length) }, () => worker())); + }), + 4, + ); + const failed = results.filter((result) => !result.ok); if (failed.length > 0) { throw new Error( `OSS upload failed for ${failed.length}/${jobs.length} object(s): ${failed - .map((f) => f.key) + .map((item) => item.key) .join(", ")}`, ); } - // Reconcile: every uploaded object must exist remotely with the local byte size. - for (const { path, key } of jobs) { - const remote = await headObjectSize(key, creds, cfg); - const local = statSync(path).size; - if (remote !== local) { - throw new Error( - `OSS reconcile mismatch for ${key}: local ${local}B vs remote ${remote ?? "missing"}`, - ); - } - } - process.stdout.write(`reconcile ok: ${jobs.length}/${jobs.length} object(s) verified on OSS\n`); + await reconcileUploads(jobs, creds, cfg, "release"); return { uploaded: jobs.length, skipped: false }; } /** * Maintain the STABLE pointers at the prefix root: rewrite `manifest.json` - * (and, when `channelJsonPath` is given, the rolling `latest.json`) when `tag` - * is a newer version than the current `latest` (first write included). + * and the rolling `latest.json` when `tag` is a newer version than the + * current manifest (first write included). Both objects carry the SAME + * rolling-manifest body produced by binary-build.mjs (`channelJsonPath`): + * `{ name, channel, version, releasedAt, assets: { "-": { file, sha256, inner } } }` + * — identical in shape to the channel `.json` manifests. * * @param {{ * tag: string, - * assetNames: string[], * channelJsonPath?: string | null, * dryRun?: boolean, - * }} options + * }} options `channelJsonPath` is required outside dry-run. * @returns {Promise<{ updated: boolean, latest: string | null }>} */ -export async function maintainReleaseManifest({ - tag, - assetNames, - channelJsonPath = null, - dryRun = false, -}) { +export async function maintainReleaseManifest({ tag, channelJsonPath = null, dryRun = false }) { const ctx = ossContext(); if (!ctx) { process.stdout.write("[info] BAILIAN_OSS_AK/SK unset; skip manifest.json maintenance\n"); @@ -320,37 +342,144 @@ export async function maintainReleaseManifest({ if (dryRun) { process.stdout.write( - `[dry-run] manifest: GET oss://${cfg.bucket}/${key} → rewrite manifest.json + latest.json when ${tag} > latest (assets: ${assetNames.length})\n`, + `[dry-run] manifest: GET oss://${cfg.bucket}/${key} → rewrite manifest.json + latest.json from ${channelJsonPath ?? ""} when ${tag} > latest\n`, ); return { updated: false, latest: null }; } + if (!channelJsonPath) { + throw new Error("maintainReleaseManifest requires channelJsonPath outside dry-run"); + } const current = await getObjectJson(key, creds, cfg); - const currentLatest = typeof current?.latest === "string" ? current.latest : null; + // Rolling-manifest shape carries `version`; fall back to the legacy + // `{ latest }` pointer shape so the first migrated write still compares. + const currentLatest = + typeof current?.version === "string" + ? current.version + : typeof current?.latest === "string" + ? current.latest + : null; const newer = currentLatest == null || compareVersions(tag, currentLatest) > 0; if (!newer) { process.stdout.write(`manifest unchanged: latest=${currentLatest} is not older than ${tag}\n`); return { updated: false, latest: currentLatest }; } - const manifest = buildManifest(tag, toBeijing(), assetNames, cfg); + const body = readFileSync(channelJsonPath); + await putObject({ creds, cfg, key, body, contentType: "application/json" }); + process.stdout.write(`manifest.json → latest=${tag} (was ${currentLatest ?? "none"})\n`); await putObject({ creds, cfg, - key, - body: Buffer.from(JSON.stringify(manifest, null, 2)), + key: `${cfg.prefix}/latest.json`, + body, contentType: "application/json", }); - process.stdout.write(`manifest.json → latest=${tag} (was ${currentLatest ?? "none"})\n`); - if (channelJsonPath) { - await putObject({ - creds, - cfg, - key: `${cfg.prefix}/latest.json`, - body: readFileSync(channelJsonPath), - contentType: "application/json", - }); - process.stdout.write(`latest.json → ${tag}\n`); - } + process.stdout.write(`latest.json → ${tag}\n`); return { updated: true, latest: tag }; } + +/** + * Resolve the OSS context for the static-files channel. Same bucket/creds as + * the release channel but uses BAILIAN_STATIC_PREFIX instead of + * BAILIAN_RELEASE_PREFIX. Returns null when credentials are absent (channel + * disabled); throws when creds exist but required config is incomplete. + */ +function staticOssContext() { + const ak = process.env.BAILIAN_OSS_AK?.trim(); + const sk = process.env.BAILIAN_OSS_SK?.trim(); + if (!ak || !sk) return null; + const cfg = { + bucket: process.env.BAILIAN_OSS_BUCKET?.trim() || "", + region: process.env.BAILIAN_OSS_REGION?.trim() || "", + endpoint: process.env.BAILIAN_OSS_ENDPOINT?.trim() || "", + prefix: process.env.BAILIAN_STATIC_PREFIX?.trim() || "", + }; + const missing = [ + ["BAILIAN_OSS_BUCKET", cfg.bucket], + ["BAILIAN_OSS_REGION", cfg.region], + ["BAILIAN_STATIC_PREFIX", cfg.prefix], + ] + .filter(([, value]) => !value) + .map(([name]) => name); + if (missing.length > 0) { + throw new Error(`OSS static-files channel misconfigured; missing env: ${missing.join(", ")}`); + } + return { creds: { ak, sk }, cfg }; +} + +/** + * Sync a list of local files to OSS under `/`. + * Generic utility for any repo files that need to be mirrored to the static + * prefix (changelogs today; docs, banners, etc. in the future). + * + * Gating: BAILIAN_OSS_AK/SK unset → warn + no-op. BAILIAN_STATIC_PREFIX unset + * (with creds present) → throw (misconfiguration). + * + * @param {{ + * filePaths: string[], + * dryRun?: boolean, + * }} options + * @returns {Promise<{ uploaded: number, skipped: boolean }>} + */ +export async function syncStaticFilesToOss({ filePaths, dryRun = false }) { + const ctx = staticOssContext(); + if (!ctx) { + process.stdout.write("\n[warn] BAILIAN_OSS_AK/SK unset; skip static-files sync to OSS\n"); + return { uploaded: 0, skipped: true }; + } + const { creds, cfg } = ctx; + + const jobs = filePaths.map((path) => ({ + path, + key: `${cfg.prefix}/${basename(path)}`, + })); + if (jobs.length === 0) return { uploaded: 0, skipped: true }; + + process.stdout.write( + `\n==> OSS static-files sync: ${jobs.length} file(s) → ${cfg.bucket}/${cfg.prefix}/\n`, + ); + + if (dryRun) { + for (const job of jobs) { + process.stdout.write(`[dry-run] PUT oss://${cfg.bucket}/${job.key}\n`); + } + return { uploaded: 0, skipped: false }; + } + + const results = await runWithConcurrency( + jobs.map((job) => async () => { + const startedAt = Date.now(); + try { + const body = readFileSync(job.path); + await putWithRetry({ + creds, + cfg, + key: job.key, + body, + contentType: contentTypeFor(job.key), + }); + process.stdout.write( + ` [oss] ok ${job.key} (${(body.length / 1024).toFixed(1)}KB, ${Date.now() - startedAt}ms)\n`, + ); + return { ok: true, key: job.key }; + } catch (error) { + process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`); + return { ok: false, key: job.key, error: error.message }; + } + }), + 4, + ); + + const failed = results.filter((result) => !result.ok); + if (failed.length > 0) { + throw new Error( + `OSS static-files sync failed for ${failed.length}/${jobs.length} file(s): ${failed + .map((item) => item.key) + .join(", ")}`, + ); + } + + await reconcileUploads(jobs, creds, cfg, "static-files"); + return { uploaded: jobs.length, skipped: false }; +}