diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 875a35e..567c79c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -69,8 +69,15 @@ jobs: - name: publish-stable env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Optional: HTTP trigger for external FC (GitHub Release → OSS). Leave unset to skip. - BAILIAN_OSS_SYNC_WEBHOOK: ${{ secrets.BAILIAN_OSS_SYNC_WEBHOOK }} + # OSS release channel runs fully in CI: upload + reconcile + manifest.json. + # All values come from repo Settings: credentials via Secrets, the rest + # via Variables. Leave the secrets unset to skip the OSS channel entirely. + BAILIAN_OSS_AK: ${{ secrets.BAILIAN_OSS_AK }} + BAILIAN_OSS_SK: ${{ secrets.BAILIAN_OSS_SK }} + BAILIAN_OSS_BUCKET: ${{ vars.BAILIAN_OSS_BUCKET }} + BAILIAN_OSS_REGION: ${{ vars.BAILIAN_OSS_REGION }} + BAILIAN_OSS_ENDPOINT: ${{ vars.BAILIAN_OSS_ENDPOINT }} + BAILIAN_RELEASE_PREFIX: ${{ vars.BAILIAN_RELEASE_PREFIX }} run: node tools/release/publish-stable.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} publish-channel: @@ -120,5 +127,11 @@ jobs: - name: publish-channel env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BAILIAN_OSS_SYNC_WEBHOOK: ${{ secrets.BAILIAN_OSS_SYNC_WEBHOOK }} + # OSS release channel — same Settings-injected values as stable. + BAILIAN_OSS_AK: ${{ secrets.BAILIAN_OSS_AK }} + BAILIAN_OSS_SK: ${{ secrets.BAILIAN_OSS_SK }} + BAILIAN_OSS_BUCKET: ${{ vars.BAILIAN_OSS_BUCKET }} + BAILIAN_OSS_REGION: ${{ vars.BAILIAN_OSS_REGION }} + BAILIAN_OSS_ENDPOINT: ${{ vars.BAILIAN_OSS_ENDPOINT }} + BAILIAN_RELEASE_PREFIX: ${{ vars.BAILIAN_RELEASE_PREFIX }} run: node tools/release/publish-channel.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} --channel "${{ inputs.channel }}" diff --git a/tools/release/lib/binary-release.mjs b/tools/release/lib/binary-release.mjs index de646a1..a906a51 100644 --- a/tools/release/lib/binary-release.mjs +++ b/tools/release/lib/binary-release.mjs @@ -10,7 +10,9 @@ * binaries); only the rolling `channel-` manifest differs per channel. * * Re-runs are idempotent via `gh release upload --clobber` (see gh-release.mjs). - * Optionally notifies BAILIAN_OSS_SYNC_WEBHOOK (see oss-sync-webhook.mjs). + * After the GitHub upload the same assets are pushed straight to OSS from the + * runner, HEAD-reconciled, and (stable only) release/manifest.json is updated — + * all in-process, no external FC (see oss-direct-upload.mjs). * * Called by publish-stable.mjs / publish-channel.mjs. * Debug: @@ -18,14 +20,14 @@ * node tools/release/lib/binary-release.mjs --mode channel --channel beta --dry-run */ import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { basename, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs as parseCliArgs } from "node:util"; import { ROOT, readPackageJson, PACKAGES } from "./packages.mjs"; import { buildBinaryArtifacts, matrixAssetNames } from "./binary-build.mjs"; import { channelManifestFileName, normalizeModeChannel } from "./binary-options.mjs"; import { ensureGh, GITHUB_REPOSITORY, upsertRelease } from "./gh-release.mjs"; -import { notifyOssSyncWebhook } from "./oss-sync-webhook.mjs"; +import { maintainReleaseManifest, mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs"; const DEFAULT_DIR = join(ROOT, "dist-bin"); @@ -124,6 +126,23 @@ function planDryRunWithoutArtifacts({ version, mode, channel }) { }); } +/** + * Build the OSS mirror plan — the same tag/asset pairs as the GitHub Release + * upload. `files == null` means dry-run planning without artifacts on disk, + * so bare basenames stand in for real paths. + */ +function ossMirrorPlans({ dir, version, mode, channel, files }) { + const paths = files + ? versionBinaryAssets(dir, version, files) + : [...matrixAssetNames(version), "SHA256SUMS"]; + const plans = [{ tag: `v${version}`, paths }]; + if (mode === "channel") { + const manifest = channelManifestFileName(channel); + plans.push({ tag: `channel-${channel}`, paths: [files ? join(dir, manifest) : manifest] }); + } + return plans; +} + /** * Build (unless skipped / dry-run) and upload binary artifacts to GitHub Releases. * Called by publish-stable / publish-channel orchestrators. @@ -131,7 +150,7 @@ function planDryRunWithoutArtifacts({ version, mode, channel }) { * `--dry-run` never compiles; it plans gh release steps. Prebuilt `dist-bin` is * optional (used only to list real paths when present). */ -export function releaseBinaryArtifacts(rawOptions = {}) { +export async function releaseBinaryArtifacts(rawOptions = {}) { const { mode, channel } = normalizeModeChannel(rawOptions.mode, rawOptions.channel); const dir = rawOptions.dir ? resolve(rawOptions.dir) : DEFAULT_DIR; const dryRun = Boolean(rawOptions.dryRun); @@ -157,7 +176,15 @@ export function releaseBinaryArtifacts(rawOptions = {}) { if (dryRun && !existsSync(dir)) { process.stdout.write(`[dry-run] ${dir} missing; planning expected assets\n`); planDryRunWithoutArtifacts({ version, mode, channel }); - notifyOssSyncWebhook({ version, mode, channel, dryRun }); + const plans = ossMirrorPlans({ dir, version, mode, channel, files: null }); + await mirrorReleaseAssetsToOss({ plans, dryRun: true }); + if (mode === "stable") { + await maintainReleaseManifest({ + tag: `v${version}`, + assetNames: plans[0].paths.map((path) => basename(path)), + dryRun: true, + }); + } return { version, mode, channel, dryRun }; } @@ -198,7 +225,18 @@ export function releaseBinaryArtifacts(rawOptions = {}) { uploadChannel({ dir, version, channel, files, dryRun }); } - notifyOssSyncWebhook({ version, mode, channel, dryRun }); + // Push the exact Release assets straight to OSS from the runner, then + // HEAD-reconcile. Stable releases additionally maintain release/manifest.json + // (newer-version guard). Throws on failure — CI is the only OSS writer. + const plans = ossMirrorPlans({ dir, version, mode, channel, files }); + const mirror = await mirrorReleaseAssetsToOss({ plans, dryRun }); + if (mode === "stable" && !mirror.skipped) { + await maintainReleaseManifest({ + tag: `v${version}`, + assetNames: plans[0].paths.map((path) => basename(path)), + dryRun, + }); + } return { version, mode, channel, dryRun }; } @@ -223,7 +261,7 @@ if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { process.stdout.write(USAGE); process.exit(0); } - releaseBinaryArtifacts({ + await releaseBinaryArtifacts({ dir: values.dir ? resolve(values.dir) : undefined, dryRun: values["dry-run"], mode: values.mode, diff --git a/tools/release/lib/oss-direct-upload.mjs b/tools/release/lib/oss-direct-upload.mjs new file mode 100644 index 0000000..69f3b6b --- /dev/null +++ b/tools/release/lib/oss-direct-upload.mjs @@ -0,0 +1,304 @@ +/** + * Publish binary release assets to OSS entirely from the CI runner: + * upload → HEAD-reconcile byte sizes → maintain release/manifest.json. + * No external FC is involved anymore; CI is the single writer. + * + * Flow: + * - Every mode uploads its assets to `//` (channel + * builds include the rolling `channel-/.json`). + * - After upload, every object is HEAD-verified against the local byte size + * (reconciliation — the runner has the ground-truth artifacts on disk). + * - Stable only: when the tag is a NEWER version than manifest.latest + * (compareVersions), rewrite `/manifest.json`. Channel/prerelease + * never touches it — same semantics the FC sync-release used to enforce. + * + * Zero-dependency: OSS V1 header signature (HMAC-SHA1) over plain fetch. + * + * Gating / failure model: + * - BAILIAN_OSS_AK / BAILIAN_OSS_SK unset → warn + no-op (npm/GitHub publish + * still succeed; set the secrets to enable the OSS channel). + * - Once enabled, any upload/reconcile/manifest failure THROWS and fails the + * release step — re-running the workflow is idempotent (uploads overwrite). + * + * Environment variables (all injected from GitHub repo Settings — Secrets for + * credentials, Variables for the rest; never hardcode values in the workflow): + * BAILIAN_OSS_AK / BAILIAN_OSS_SK —— RAM AccessKey; needs oss:PutObject and + * oss:GetObject on `/*` + * BAILIAN_OSS_BUCKET —— target bucket, default bailian-wiki + * BAILIAN_OSS_REGION —— region, default oss-cn-hangzhou + * BAILIAN_OSS_ENDPOINT —— optional request endpoint override, + * e.g. oss-accelerate.aliyuncs.com; public + * manifest URLs always use the region endpoint + * BAILIAN_RELEASE_PREFIX —— object prefix, default release + */ +import { createHash, createHmac } from "node:crypto"; +import { readFileSync, statSync } from "node:fs"; +import { basename } from "node:path"; + +const OSS_CONFIG = { + bucket: process.env.BAILIAN_OSS_BUCKET || "bailian-wiki", + region: process.env.BAILIAN_OSS_REGION || "oss-cn-hangzhou", + endpoint: process.env.BAILIAN_OSS_ENDPOINT || "", + prefix: process.env.BAILIAN_RELEASE_PREFIX || "release", +}; + +function ossCredentials() { + const ak = process.env.BAILIAN_OSS_AK?.trim(); + const sk = process.env.BAILIAN_OSS_SK?.trim(); + return ak && sk ? { ak, sk } : null; +} + +/** Virtual-hosted-style request host: .. */ +function ossHost(cfg) { + return `${cfg.bucket}.${cfg.endpoint || `${cfg.region}.aliyuncs.com`}`; +} + +function contentTypeFor(name) { + if (name.endsWith(".zip")) return "application/zip"; + if (name.endsWith(".json")) return "application/json"; + return "application/octet-stream"; +} + +/** + * Compare two version strings (strip a leading v/V, split on `.`, numeric + * per-segment; non-numeric / missing segments count as 0). + * @returns {number} 1 if a>b, -1 if a + String(v ?? "") + .trim() + .replace(/^[vV]/, "") + .split(".") + .map((s) => parseInt(s, 10) || 0); + const pa = norm(a); + const pb = norm(b); + const len = Math.max(pa.length, pb.length); + for (let i = 0; i < len; i++) { + const x = pa[i] ?? 0; + const y = pb[i] ?? 0; + if (x > y) return 1; + if (x < y) return -1; + } + return 0; +} + +/** Format now (or a given time) as an Asia/Shanghai +08:00 string. */ +function toBeijing(input) { + const d = input ? new Date(input) : new Date(); + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: "Asia/Shanghai", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + }).formatToParts(d); + const g = (t) => parts.find((p) => p.type === t)?.value ?? "00"; + return `${g("year")}-${g("month")}-${g("day")}T${g("hour")}:${g("minute")}:${g("second")}+08:00`; +} + +/** + * Build the manifest.json contents. Public URLs always use the durable region + * endpoint (never the acceleration endpoint used for uploads). + */ +function buildManifest(tag, releasedAt, assetNames, cfg) { + const base = `https://${cfg.bucket}.${cfg.region}.aliyuncs.com/${cfg.prefix}/${tag}`; + const assets = {}; + for (const name of [...assetNames].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))) { + assets[name] = `${base}/${encodeURIComponent(name)}`; + } + return { latest: tag, releasedAt, assets }; +} + +/** + * Signed OSS request (V1 header signature). Keys here are [A-Za-z0-9._/-] only, + * so no URL encoding is needed and the signed resource matches the request path. + */ +async function ossRequest(method, key, { creds, cfg, body = null, contentType = "" }) { + const date = new Date().toUTCString(); + const contentMd5 = body ? createHash("md5").update(body).digest("base64") : ""; + const canonical = `${method}\n${contentMd5}\n${contentType}\n${date}\n/${cfg.bucket}/${key}`; + const signature = createHmac("sha1", creds.sk).update(canonical).digest("base64"); + const headers = { Date: date, Authorization: `OSS ${creds.ak}:${signature}` }; + if (contentType) headers["Content-Type"] = contentType; + if (contentMd5) headers["Content-MD5"] = contentMd5; + const options = { method, headers }; + if (body) options.body = body; + return fetch(`https://${ossHost(cfg)}/${key}`, options); +} + +async function putObject({ creds, cfg, key, body, contentType }) { + const res = await ossRequest("PUT", key, { creds, cfg, body, contentType }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`OSS PUT ${key} failed: HTTP ${res.status} ${text.slice(0, 200)}`); + } +} + +/** PUT with exponential-backoff retries (runner → OSS can flake too). */ +async function putWithRetry(params, attempts = 3) { + for (let attempt = 1; ; attempt++) { + try { + return await putObject(params); + } catch (err) { + if (attempt >= attempts) throw err; + const delay = 1000 * 2 ** (attempt - 1); + process.stdout.write( + ` [oss] retry ${attempt}/${attempts - 1} for ${params.key} in ${delay}ms (${err.message})\n`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } +} + +/** Remote object byte size; null when the object does not exist. */ +async function headObjectSize(key, creds, cfg) { + const res = await ossRequest("HEAD", key, { creds, cfg }); + if (res.status === 404) return null; + if (!res.ok) throw new Error(`OSS HEAD ${key} failed: HTTP ${res.status}`); + return Number(res.headers.get("content-length")); +} + +/** GET + parse a JSON object; null when missing or corrupt. */ +async function getObjectJson(key, creds, cfg) { + const res = await ossRequest("GET", key, { creds, cfg }); + if (res.status === 404) return null; + if (!res.ok) throw new Error(`OSS GET ${key} failed: HTTP ${res.status}`); + try { + return await res.json(); + } catch { + return null; + } +} + +/** + * Upload release assets to OSS under `//`, then + * HEAD-reconcile every object against the local byte size. + * Throws on any upload or reconcile failure (CI is the only writer now). + * + * @param {{ + * plans: Array<{ tag: string, paths: string[] }>, + * dryRun?: boolean, + * }} options `paths` may be bare basenames in dry-run planning mode. + * @returns {Promise<{ uploaded: number, skipped: boolean }>} + */ +export async function mirrorReleaseAssetsToOss({ plans, dryRun = false }) { + const creds = ossCredentials(); + const cfg = OSS_CONFIG; + + const jobs = plans.flatMap(({ tag, paths }) => + paths.map((path) => ({ path, key: `${cfg.prefix}/${tag}/${basename(path)}` })), + ); + if (jobs.length === 0) return { uploaded: 0, skipped: true }; + + if (!creds) { + process.stdout.write( + "\n[warn] BAILIAN_OSS_AK/SK unset; skip the OSS release channel entirely\n", + ); + return { uploaded: 0, skipped: true }; + } + + process.stdout.write( + `\n==> OSS upload: ${jobs.length} object(s) → ${cfg.bucket} (${cfg.endpoint || cfg.region})\n`, + ); + + if (dryRun) { + for (const job of jobs) { + process.stdout.write(`[dry-run] PUT oss://${cfg.bucket}/${job.key}\n`); + } + process.stdout.write(`[dry-run] reconcile (HEAD size check) ${jobs.length} object(s)\n`); + return { uploaded: 0, skipped: false }; + } + + // Bounded worker pool; collect failures, then throw once at the end. + const failed = []; + let cursor = 0; + const worker = async () => { + while (true) { + const index = cursor++; + if (index >= jobs.length) return; + const { path, key } = jobs[index]; + const startedAt = Date.now(); + try { + const body = readFileSync(path); + await putWithRetry({ creds, cfg, key, body, contentType: contentTypeFor(key) }); + process.stdout.write( + ` [oss] ok ${key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`, + ); + } catch (err) { + failed.push({ key, error: err.message }); + process.stdout.write(` [oss] FAIL ${key}: ${err.message}\n`); + } + } + }; + await Promise.all(Array.from({ length: Math.min(4, jobs.length) }, () => worker())); + + if (failed.length > 0) { + throw new Error( + `OSS upload failed for ${failed.length}/${jobs.length} object(s): ${failed + .map((f) => f.key) + .join(", ")}`, + ); + } + + // Reconcile: every uploaded object must exist remotely with the local byte size. + for (const { path, key } of jobs) { + const remote = await headObjectSize(key, creds, cfg); + const local = statSync(path).size; + if (remote !== local) { + throw new Error( + `OSS reconcile mismatch for ${key}: local ${local}B vs remote ${remote ?? "missing"}`, + ); + } + } + process.stdout.write(`reconcile ok: ${jobs.length}/${jobs.length} object(s) verified on OSS\n`); + return { uploaded: jobs.length, skipped: false }; +} + +/** + * Maintain `/manifest.json` for STABLE releases only: rewrite it when + * `tag` is a newer version than the current `latest` (first write included). + * Same update rule the FC sync-release used to apply. + * + * @param {{ tag: string, assetNames: string[], dryRun?: boolean }} options + * @returns {Promise<{ updated: boolean, latest: string | null }>} + */ +export async function maintainReleaseManifest({ tag, assetNames, dryRun = false }) { + const creds = ossCredentials(); + const cfg = OSS_CONFIG; + const key = `${cfg.prefix}/manifest.json`; + + if (!creds) { + process.stdout.write("[info] BAILIAN_OSS_AK/SK unset; skip manifest.json maintenance\n"); + return { updated: false, latest: null }; + } + + if (dryRun) { + process.stdout.write( + `[dry-run] manifest: GET oss://${cfg.bucket}/${key} → rewrite when ${tag} > latest (assets: ${assetNames.length})\n`, + ); + return { updated: false, latest: null }; + } + + const current = await getObjectJson(key, creds, cfg); + const currentLatest = typeof current?.latest === "string" ? current.latest : null; + const newer = currentLatest == null || compareVersions(tag, currentLatest) > 0; + if (!newer) { + process.stdout.write(`manifest unchanged: latest=${currentLatest} is not older than ${tag}\n`); + return { updated: false, latest: currentLatest }; + } + + const manifest = buildManifest(tag, toBeijing(), assetNames, cfg); + await putObject({ + creds, + cfg, + key, + body: Buffer.from(JSON.stringify(manifest, null, 2)), + contentType: "application/json", + }); + process.stdout.write(`manifest.json → latest=${tag} (was ${currentLatest ?? "none"})\n`); + return { updated: true, latest: tag }; +} diff --git a/tools/release/lib/oss-sync-webhook.mjs b/tools/release/lib/oss-sync-webhook.mjs deleted file mode 100644 index ba9f814..0000000 --- a/tools/release/lib/oss-sync-webhook.mjs +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Optional hook for an external FC that mirrors GitHub Releases → OSS. - * Set BAILIAN_OSS_SYNC_WEBHOOK to an HTTP endpoint; unset → no-op. - * Failure is warn-only — Release publish already succeeded. - */ -import { tryRun } from "./proc.mjs"; -import { GITHUB_REPOSITORY } from "./gh-release.mjs"; - -/** - * @param {{ - * version: string, - * mode: "stable" | "channel", - * channel: string | null, - * dryRun?: boolean, - * repo?: string, - * }} options - */ -export function notifyOssSyncWebhook({ - version, - mode, - channel, - dryRun = false, - repo = GITHUB_REPOSITORY, -}) { - const webhook = process.env.BAILIAN_OSS_SYNC_WEBHOOK?.trim(); - if (!webhook) { - process.stdout.write( - "\n[info] BAILIAN_OSS_SYNC_WEBHOOK unset; skip notifying external OSS sync FC\n", - ); - return; - } - const tag = `v${version}`; - const body = { - repo, - mode, - channel, - version, - tag, - rollingChannelTag: mode === "channel" ? `channel-${channel}` : null, - }; - if (dryRun) { - process.stdout.write(`[dry-run] POST ${webhook}\n${JSON.stringify(body, null, 2)}\n`); - return; - } - process.stdout.write(`\n==> notify OSS sync FC: ${webhook}\n`); - const result = tryRun("curl", [ - "-fsS", - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - JSON.stringify(body), - webhook, - ]); - if (result.status !== 0) { - process.stdout.write( - `[warn] OSS sync webhook failed (release already published): ${result.stderr || result.stdout}\n`, - ); - return; - } - if (result.stdout) process.stdout.write(`${result.stdout}\n`); -} diff --git a/tools/release/publish-channel.mjs b/tools/release/publish-channel.mjs index 420271e..a5e62c6 100644 --- a/tools/release/publish-channel.mjs +++ b/tools/release/publish-channel.mjs @@ -96,7 +96,7 @@ try { step( `publish binary GitHub Release (mode=channel, channel=${channel}, version=${betaVersion})`, ); - releaseBinaryArtifacts({ mode: "channel", channel, dryRun }); + await releaseBinaryArtifacts({ mode: "channel", channel, dryRun }); } const parts = ["npm"]; diff --git a/tools/release/publish-stable.mjs b/tools/release/publish-stable.mjs index 40fa2cb..66e8544 100644 --- a/tools/release/publish-stable.mjs +++ b/tools/release/publish-stable.mjs @@ -86,7 +86,7 @@ try { log("\n[skip-binary] skipping binary GitHub Release"); } else { step(`publish binary GitHub Release (mode=stable, version=${version})`); - releaseBinaryArtifacts({ mode: "stable", dryRun }); + await releaseBinaryArtifacts({ mode: "stable", dryRun }); } const parts = ["npm"];