feat: add changelog sync to oss

This commit is contained in:
故璃
2026-07-27 16:56:46 +08:00
parent ebbd173b79
commit 7250de9228
3 changed files with 203 additions and 30 deletions
+2
View File
@@ -79,6 +79,7 @@ jobs:
BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }}
BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }}
BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }}
BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }}
run: node tools/release/publish-stable.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }}
publish-channel:
@@ -135,4 +136,5 @@ jobs:
BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }}
BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }}
BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }}
BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }}
run: node tools/release/publish-channel.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} --channel "${{ inputs.channel }}"
+14 -1
View File
@@ -27,7 +27,11 @@ import { ROOT, readPackageJson, PACKAGES } from "./packages.mjs";
import { buildBinaryArtifacts, matrixAssetNames } from "./binary-build.mjs";
import { channelManifestFileName, normalizeModeChannel } from "./binary-options.mjs";
import { ensureGh, GITHUB_REPOSITORY, upsertRelease } from "./gh-release.mjs";
import { maintainReleaseManifest, mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs";
import {
maintainReleaseManifest,
mirrorReleaseAssetsToOss,
syncStaticFilesToOss,
} from "./oss-direct-upload.mjs";
const DEFAULT_DIR = join(ROOT, "dist-bin");
@@ -179,6 +183,10 @@ export async function releaseBinaryArtifacts(rawOptions = {}) {
process.stdout.write(`[dry-run] ${dir} missing; planning expected assets\n`);
planDryRunWithoutArtifacts({ version, mode, channel });
const plans = ossMirrorPlans({ dir, version, mode, channel, files: null });
await syncStaticFilesToOss({
filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")],
dryRun: true,
});
await mirrorReleaseAssetsToOss({ plans, dryRun: true });
if (mode === "stable") {
await maintainReleaseManifest({
@@ -226,6 +234,11 @@ export async function releaseBinaryArtifacts(rawOptions = {}) {
uploadChannel({ dir, version, channel, files, dryRun });
}
// Sync changelogs (and other static files) to OSS before the binary mirror.
await syncStaticFilesToOss({
filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")],
dryRun,
});
// Push the exact Release assets straight to OSS from the runner, then
// HEAD-reconcile. Stable releases additionally maintain release/manifest.json
// (newer-version guard). Throws on failure — CI is the only OSS writer.
+187 -29
View File
@@ -26,6 +26,8 @@
* oss:GetObject on the release prefix
* BAILIAN_OSS_BUCKET / BAILIAN_OSS_REGION / BAILIAN_RELEASE_PREFIX
* —— required once the channel is enabled
* BAILIAN_STATIC_PREFIX —— prefix for static files (changelogs, etc.);
* same bucket/creds, separate namespace
* BAILIAN_OSS_ENDPOINT —— optional request endpoint override;
* public manifest URLs always use the
* region endpoint
@@ -69,6 +71,7 @@ function ossHost(cfg) {
}
function contentTypeFor(name) {
if (name.endsWith(".md")) return "text/markdown; charset=utf-8";
if (name.endsWith(".zip")) return "application/zip";
if (name.endsWith(".json")) return "application/json";
return "application/octet-stream";
@@ -204,6 +207,64 @@ async function getObjectJson(key, creds, cfg) {
}
}
/**
* Run async task factories with a bounded concurrency pool.
* Returns results in the same order as the input tasks array.
* (Same contract as packages/commands/src/commands/skill/shared.ts)
*
* @template T
* @param {Array<() => Promise<T>>} tasks
* @param {number} limit
* @returns {Promise<T[]>}
*/
async function runWithConcurrency(tasks, limit) {
const results = new Array(tasks.length);
let nextIndex = 0;
async function worker() {
while (nextIndex < tasks.length) {
const currentIndex = nextIndex++;
results[currentIndex] = await tasks[currentIndex]();
}
}
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => worker());
await Promise.all(workers);
return results;
}
/**
* HEAD-reconcile: verify every uploaded object exists remotely with the same
* byte size as the local file. Runs HEAD requests concurrently.
*
* @param {Array<{ path: string, key: string }>} jobs
* @param {{ ak: string, sk: string }} creds
* @param {object} cfg
* @param {string} label Context for error messages (e.g. "release", "static-files")
*/
async function reconcileUploads(jobs, creds, cfg, label) {
const results = await runWithConcurrency(
jobs.map((job) => async () => {
const remote = await headObjectSize(job.key, creds, cfg);
const local = statSync(job.path).size;
if (remote !== local) {
return {
ok: false,
key: job.key,
error: `OSS ${label} reconcile mismatch for ${job.key}: local ${local}B vs remote ${remote ?? "missing"}`,
};
}
return { ok: true, key: job.key };
}),
4,
);
const mismatches = results.filter((result) => !result.ok);
if (mismatches.length > 0) {
throw new Error(mismatches.map((item) => item.error).join("\n"));
}
process.stdout.write(`${label} reconcile ok: ${jobs.length}/${jobs.length} object(s) verified\n`);
}
/**
* Upload release assets to OSS under `<prefix>/<tag>/<basename>`, then
* HEAD-reconcile every object against the local byte size.
@@ -246,48 +307,40 @@ export async function mirrorReleaseAssetsToOss({ plans, dryRun = false }) {
return { uploaded: 0, skipped: false };
}
// Bounded worker pool; collect failures, then throw once at the end.
const failed = [];
let cursor = 0;
const worker = async () => {
while (true) {
const index = cursor++;
if (index >= jobs.length) return;
const { path, key } = jobs[index];
const results = await runWithConcurrency(
jobs.map((job) => async () => {
const startedAt = Date.now();
try {
const body = readFileSync(path);
await putWithRetry({ creds, cfg, key, body, contentType: contentTypeFor(key) });
const body = readFileSync(job.path);
await putWithRetry({
creds,
cfg,
key: job.key,
body,
contentType: contentTypeFor(job.key),
});
process.stdout.write(
` [oss] ok ${key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`,
` [oss] ok ${job.key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`,
);
} catch (err) {
failed.push({ key, error: err.message });
process.stdout.write(` [oss] FAIL ${key}: ${err.message}\n`);
return { ok: true, key: job.key };
} catch (error) {
process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`);
return { ok: false, key: job.key, error: error.message };
}
}
};
await Promise.all(Array.from({ length: Math.min(4, jobs.length) }, () => worker()));
}),
4,
);
const failed = results.filter((result) => !result.ok);
if (failed.length > 0) {
throw new Error(
`OSS upload failed for ${failed.length}/${jobs.length} object(s): ${failed
.map((f) => f.key)
.map((item) => item.key)
.join(", ")}`,
);
}
// Reconcile: every uploaded object must exist remotely with the local byte size.
for (const { path, key } of jobs) {
const remote = await headObjectSize(key, creds, cfg);
const local = statSync(path).size;
if (remote !== local) {
throw new Error(
`OSS reconcile mismatch for ${key}: local ${local}B vs remote ${remote ?? "missing"}`,
);
}
}
process.stdout.write(`reconcile ok: ${jobs.length}/${jobs.length} object(s) verified on OSS\n`);
await reconcileUploads(jobs, creds, cfg, "release");
return { uploaded: jobs.length, skipped: false };
}
@@ -354,3 +407,108 @@ export async function maintainReleaseManifest({
}
return { updated: true, latest: tag };
}
/**
* Resolve the OSS context for the static-files channel. Same bucket/creds as
* the release channel but uses BAILIAN_STATIC_PREFIX instead of
* BAILIAN_RELEASE_PREFIX. Returns null when credentials are absent (channel
* disabled); throws when creds exist but required config is incomplete.
*/
function staticOssContext() {
const ak = process.env.BAILIAN_OSS_AK?.trim();
const sk = process.env.BAILIAN_OSS_SK?.trim();
if (!ak || !sk) return null;
const cfg = {
bucket: process.env.BAILIAN_OSS_BUCKET?.trim() || "",
region: process.env.BAILIAN_OSS_REGION?.trim() || "",
endpoint: process.env.BAILIAN_OSS_ENDPOINT?.trim() || "",
prefix: process.env.BAILIAN_STATIC_PREFIX?.trim() || "",
};
const missing = [
["BAILIAN_OSS_BUCKET", cfg.bucket],
["BAILIAN_OSS_REGION", cfg.region],
["BAILIAN_STATIC_PREFIX", cfg.prefix],
]
.filter(([, value]) => !value)
.map(([name]) => name);
if (missing.length > 0) {
throw new Error(`OSS static-files channel misconfigured; missing env: ${missing.join(", ")}`);
}
return { creds: { ak, sk }, cfg };
}
/**
* Sync a list of local files to OSS under `<BAILIAN_STATIC_PREFIX>/<basename>`.
* Generic utility for any repo files that need to be mirrored to the static
* prefix (changelogs today; docs, banners, etc. in the future).
*
* Gating: BAILIAN_OSS_AK/SK unset → warn + no-op. BAILIAN_STATIC_PREFIX unset
* (with creds present) → throw (misconfiguration).
*
* @param {{
* filePaths: string[],
* dryRun?: boolean,
* }} options
* @returns {Promise<{ uploaded: number, skipped: boolean }>}
*/
export async function syncStaticFilesToOss({ filePaths, dryRun = false }) {
const ctx = staticOssContext();
if (!ctx) {
process.stdout.write("\n[warn] BAILIAN_OSS_AK/SK unset; skip static-files sync to OSS\n");
return { uploaded: 0, skipped: true };
}
const { creds, cfg } = ctx;
const jobs = filePaths.map((path) => ({
path,
key: `${cfg.prefix}/${basename(path)}`,
}));
if (jobs.length === 0) return { uploaded: 0, skipped: true };
process.stdout.write(
`\n==> OSS static-files sync: ${jobs.length} file(s) → ${cfg.bucket}/${cfg.prefix}/\n`,
);
if (dryRun) {
for (const job of jobs) {
process.stdout.write(`[dry-run] PUT oss://${cfg.bucket}/${job.key}\n`);
}
return { uploaded: 0, skipped: false };
}
const results = await runWithConcurrency(
jobs.map((job) => async () => {
const startedAt = Date.now();
try {
const body = readFileSync(job.path);
await putWithRetry({
creds,
cfg,
key: job.key,
body,
contentType: contentTypeFor(job.key),
});
process.stdout.write(
` [oss] ok ${job.key} (${(body.length / 1024).toFixed(1)}KB, ${Date.now() - startedAt}ms)\n`,
);
return { ok: true, key: job.key };
} catch (error) {
process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`);
return { ok: false, key: job.key, error: error.message };
}
}),
4,
);
const failed = results.filter((result) => !result.ok);
if (failed.length > 0) {
throw new Error(
`OSS static-files sync failed for ${failed.length}/${jobs.length} file(s): ${failed
.map((item) => item.key)
.join(", ")}`,
);
}
await reconcileUploads(jobs, creds, cfg, "static-files");
return { uploaded: jobs.length, skipped: false };
}