mirror of
https://github.com/modelstudioai/cli.git
synced 2026-09-14 19:49:23 +08:00
feat: add changelog sync to oss
This commit is contained in:
@@ -79,6 +79,7 @@ jobs:
|
||||
BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }}
|
||||
BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }}
|
||||
BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }}
|
||||
BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }}
|
||||
run: node tools/release/publish-stable.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }}
|
||||
|
||||
publish-channel:
|
||||
@@ -135,4 +136,5 @@ jobs:
|
||||
BAILIAN_OSS_REGION: ${{ secrets.BAILIAN_OSS_REGION }}
|
||||
BAILIAN_OSS_ENDPOINT: ${{ secrets.BAILIAN_OSS_ENDPOINT }}
|
||||
BAILIAN_RELEASE_PREFIX: ${{ secrets.BAILIAN_RELEASE_PREFIX }}
|
||||
BAILIAN_STATIC_PREFIX: ${{ secrets.BAILIAN_STATIC_PREFIX }}
|
||||
run: node tools/release/publish-channel.mjs ${{ inputs.package == 'knowledge-studio-cli' && '--knowledge' || '' }} --channel "${{ inputs.channel }}"
|
||||
|
||||
@@ -27,7 +27,11 @@ import { ROOT, readPackageJson, PACKAGES } from "./packages.mjs";
|
||||
import { buildBinaryArtifacts, matrixAssetNames } from "./binary-build.mjs";
|
||||
import { channelManifestFileName, normalizeModeChannel } from "./binary-options.mjs";
|
||||
import { ensureGh, GITHUB_REPOSITORY, upsertRelease } from "./gh-release.mjs";
|
||||
import { maintainReleaseManifest, mirrorReleaseAssetsToOss } from "./oss-direct-upload.mjs";
|
||||
import {
|
||||
maintainReleaseManifest,
|
||||
mirrorReleaseAssetsToOss,
|
||||
syncStaticFilesToOss,
|
||||
} from "./oss-direct-upload.mjs";
|
||||
|
||||
const DEFAULT_DIR = join(ROOT, "dist-bin");
|
||||
|
||||
@@ -179,6 +183,10 @@ export async function releaseBinaryArtifacts(rawOptions = {}) {
|
||||
process.stdout.write(`[dry-run] ${dir} missing; planning expected assets\n`);
|
||||
planDryRunWithoutArtifacts({ version, mode, channel });
|
||||
const plans = ossMirrorPlans({ dir, version, mode, channel, files: null });
|
||||
await syncStaticFilesToOss({
|
||||
filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")],
|
||||
dryRun: true,
|
||||
});
|
||||
await mirrorReleaseAssetsToOss({ plans, dryRun: true });
|
||||
if (mode === "stable") {
|
||||
await maintainReleaseManifest({
|
||||
@@ -226,6 +234,11 @@ export async function releaseBinaryArtifacts(rawOptions = {}) {
|
||||
uploadChannel({ dir, version, channel, files, dryRun });
|
||||
}
|
||||
|
||||
// Sync changelogs (and other static files) to OSS before the binary mirror.
|
||||
await syncStaticFilesToOss({
|
||||
filePaths: [join(ROOT, "CHANGELOG.md"), join(ROOT, "CHANGELOG.zh.md")],
|
||||
dryRun,
|
||||
});
|
||||
// Push the exact Release assets straight to OSS from the runner, then
|
||||
// HEAD-reconcile. Stable releases additionally maintain release/manifest.json
|
||||
// (newer-version guard). Throws on failure — CI is the only OSS writer.
|
||||
|
||||
@@ -26,6 +26,8 @@
|
||||
* oss:GetObject on the release prefix
|
||||
* BAILIAN_OSS_BUCKET / BAILIAN_OSS_REGION / BAILIAN_RELEASE_PREFIX
|
||||
* —— required once the channel is enabled
|
||||
* BAILIAN_STATIC_PREFIX —— prefix for static files (changelogs, etc.);
|
||||
* same bucket/creds, separate namespace
|
||||
* BAILIAN_OSS_ENDPOINT —— optional request endpoint override;
|
||||
* public manifest URLs always use the
|
||||
* region endpoint
|
||||
@@ -69,6 +71,7 @@ function ossHost(cfg) {
|
||||
}
|
||||
|
||||
function contentTypeFor(name) {
|
||||
if (name.endsWith(".md")) return "text/markdown; charset=utf-8";
|
||||
if (name.endsWith(".zip")) return "application/zip";
|
||||
if (name.endsWith(".json")) return "application/json";
|
||||
return "application/octet-stream";
|
||||
@@ -204,6 +207,64 @@ async function getObjectJson(key, creds, cfg) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run async task factories with a bounded concurrency pool.
|
||||
* Returns results in the same order as the input tasks array.
|
||||
* (Same contract as packages/commands/src/commands/skill/shared.ts)
|
||||
*
|
||||
* @template T
|
||||
* @param {Array<() => Promise<T>>} tasks
|
||||
* @param {number} limit
|
||||
* @returns {Promise<T[]>}
|
||||
*/
|
||||
async function runWithConcurrency(tasks, limit) {
|
||||
const results = new Array(tasks.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
async function worker() {
|
||||
while (nextIndex < tasks.length) {
|
||||
const currentIndex = nextIndex++;
|
||||
results[currentIndex] = await tasks[currentIndex]();
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => worker());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* HEAD-reconcile: verify every uploaded object exists remotely with the same
|
||||
* byte size as the local file. Runs HEAD requests concurrently.
|
||||
*
|
||||
* @param {Array<{ path: string, key: string }>} jobs
|
||||
* @param {{ ak: string, sk: string }} creds
|
||||
* @param {object} cfg
|
||||
* @param {string} label Context for error messages (e.g. "release", "static-files")
|
||||
*/
|
||||
async function reconcileUploads(jobs, creds, cfg, label) {
|
||||
const results = await runWithConcurrency(
|
||||
jobs.map((job) => async () => {
|
||||
const remote = await headObjectSize(job.key, creds, cfg);
|
||||
const local = statSync(job.path).size;
|
||||
if (remote !== local) {
|
||||
return {
|
||||
ok: false,
|
||||
key: job.key,
|
||||
error: `OSS ${label} reconcile mismatch for ${job.key}: local ${local}B vs remote ${remote ?? "missing"}`,
|
||||
};
|
||||
}
|
||||
return { ok: true, key: job.key };
|
||||
}),
|
||||
4,
|
||||
);
|
||||
const mismatches = results.filter((result) => !result.ok);
|
||||
if (mismatches.length > 0) {
|
||||
throw new Error(mismatches.map((item) => item.error).join("\n"));
|
||||
}
|
||||
process.stdout.write(`${label} reconcile ok: ${jobs.length}/${jobs.length} object(s) verified\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload release assets to OSS under `<prefix>/<tag>/<basename>`, then
|
||||
* HEAD-reconcile every object against the local byte size.
|
||||
@@ -246,48 +307,40 @@ export async function mirrorReleaseAssetsToOss({ plans, dryRun = false }) {
|
||||
return { uploaded: 0, skipped: false };
|
||||
}
|
||||
|
||||
// Bounded worker pool; collect failures, then throw once at the end.
|
||||
const failed = [];
|
||||
let cursor = 0;
|
||||
const worker = async () => {
|
||||
while (true) {
|
||||
const index = cursor++;
|
||||
if (index >= jobs.length) return;
|
||||
const { path, key } = jobs[index];
|
||||
const results = await runWithConcurrency(
|
||||
jobs.map((job) => async () => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const body = readFileSync(path);
|
||||
await putWithRetry({ creds, cfg, key, body, contentType: contentTypeFor(key) });
|
||||
const body = readFileSync(job.path);
|
||||
await putWithRetry({
|
||||
creds,
|
||||
cfg,
|
||||
key: job.key,
|
||||
body,
|
||||
contentType: contentTypeFor(job.key),
|
||||
});
|
||||
process.stdout.write(
|
||||
` [oss] ok ${key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`,
|
||||
` [oss] ok ${job.key} (${(body.length / 1024 / 1024).toFixed(1)}MB, ${Date.now() - startedAt}ms)\n`,
|
||||
);
|
||||
} catch (err) {
|
||||
failed.push({ key, error: err.message });
|
||||
process.stdout.write(` [oss] FAIL ${key}: ${err.message}\n`);
|
||||
return { ok: true, key: job.key };
|
||||
} catch (error) {
|
||||
process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`);
|
||||
return { ok: false, key: job.key, error: error.message };
|
||||
}
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(4, jobs.length) }, () => worker()));
|
||||
}),
|
||||
4,
|
||||
);
|
||||
|
||||
const failed = results.filter((result) => !result.ok);
|
||||
if (failed.length > 0) {
|
||||
throw new Error(
|
||||
`OSS upload failed for ${failed.length}/${jobs.length} object(s): ${failed
|
||||
.map((f) => f.key)
|
||||
.map((item) => item.key)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Reconcile: every uploaded object must exist remotely with the local byte size.
|
||||
for (const { path, key } of jobs) {
|
||||
const remote = await headObjectSize(key, creds, cfg);
|
||||
const local = statSync(path).size;
|
||||
if (remote !== local) {
|
||||
throw new Error(
|
||||
`OSS reconcile mismatch for ${key}: local ${local}B vs remote ${remote ?? "missing"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
process.stdout.write(`reconcile ok: ${jobs.length}/${jobs.length} object(s) verified on OSS\n`);
|
||||
await reconcileUploads(jobs, creds, cfg, "release");
|
||||
return { uploaded: jobs.length, skipped: false };
|
||||
}
|
||||
|
||||
@@ -354,3 +407,108 @@ export async function maintainReleaseManifest({
|
||||
}
|
||||
return { updated: true, latest: tag };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the OSS context for the static-files channel. Same bucket/creds as
|
||||
* the release channel but uses BAILIAN_STATIC_PREFIX instead of
|
||||
* BAILIAN_RELEASE_PREFIX. Returns null when credentials are absent (channel
|
||||
* disabled); throws when creds exist but required config is incomplete.
|
||||
*/
|
||||
function staticOssContext() {
|
||||
const ak = process.env.BAILIAN_OSS_AK?.trim();
|
||||
const sk = process.env.BAILIAN_OSS_SK?.trim();
|
||||
if (!ak || !sk) return null;
|
||||
const cfg = {
|
||||
bucket: process.env.BAILIAN_OSS_BUCKET?.trim() || "",
|
||||
region: process.env.BAILIAN_OSS_REGION?.trim() || "",
|
||||
endpoint: process.env.BAILIAN_OSS_ENDPOINT?.trim() || "",
|
||||
prefix: process.env.BAILIAN_STATIC_PREFIX?.trim() || "",
|
||||
};
|
||||
const missing = [
|
||||
["BAILIAN_OSS_BUCKET", cfg.bucket],
|
||||
["BAILIAN_OSS_REGION", cfg.region],
|
||||
["BAILIAN_STATIC_PREFIX", cfg.prefix],
|
||||
]
|
||||
.filter(([, value]) => !value)
|
||||
.map(([name]) => name);
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`OSS static-files channel misconfigured; missing env: ${missing.join(", ")}`);
|
||||
}
|
||||
return { creds: { ak, sk }, cfg };
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a list of local files to OSS under `<BAILIAN_STATIC_PREFIX>/<basename>`.
|
||||
* Generic utility for any repo files that need to be mirrored to the static
|
||||
* prefix (changelogs today; docs, banners, etc. in the future).
|
||||
*
|
||||
* Gating: BAILIAN_OSS_AK/SK unset → warn + no-op. BAILIAN_STATIC_PREFIX unset
|
||||
* (with creds present) → throw (misconfiguration).
|
||||
*
|
||||
* @param {{
|
||||
* filePaths: string[],
|
||||
* dryRun?: boolean,
|
||||
* }} options
|
||||
* @returns {Promise<{ uploaded: number, skipped: boolean }>}
|
||||
*/
|
||||
export async function syncStaticFilesToOss({ filePaths, dryRun = false }) {
|
||||
const ctx = staticOssContext();
|
||||
if (!ctx) {
|
||||
process.stdout.write("\n[warn] BAILIAN_OSS_AK/SK unset; skip static-files sync to OSS\n");
|
||||
return { uploaded: 0, skipped: true };
|
||||
}
|
||||
const { creds, cfg } = ctx;
|
||||
|
||||
const jobs = filePaths.map((path) => ({
|
||||
path,
|
||||
key: `${cfg.prefix}/${basename(path)}`,
|
||||
}));
|
||||
if (jobs.length === 0) return { uploaded: 0, skipped: true };
|
||||
|
||||
process.stdout.write(
|
||||
`\n==> OSS static-files sync: ${jobs.length} file(s) → ${cfg.bucket}/${cfg.prefix}/\n`,
|
||||
);
|
||||
|
||||
if (dryRun) {
|
||||
for (const job of jobs) {
|
||||
process.stdout.write(`[dry-run] PUT oss://${cfg.bucket}/${job.key}\n`);
|
||||
}
|
||||
return { uploaded: 0, skipped: false };
|
||||
}
|
||||
|
||||
const results = await runWithConcurrency(
|
||||
jobs.map((job) => async () => {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const body = readFileSync(job.path);
|
||||
await putWithRetry({
|
||||
creds,
|
||||
cfg,
|
||||
key: job.key,
|
||||
body,
|
||||
contentType: contentTypeFor(job.key),
|
||||
});
|
||||
process.stdout.write(
|
||||
` [oss] ok ${job.key} (${(body.length / 1024).toFixed(1)}KB, ${Date.now() - startedAt}ms)\n`,
|
||||
);
|
||||
return { ok: true, key: job.key };
|
||||
} catch (error) {
|
||||
process.stdout.write(` [oss] FAIL ${job.key}: ${error.message}\n`);
|
||||
return { ok: false, key: job.key, error: error.message };
|
||||
}
|
||||
}),
|
||||
4,
|
||||
);
|
||||
|
||||
const failed = results.filter((result) => !result.ok);
|
||||
if (failed.length > 0) {
|
||||
throw new Error(
|
||||
`OSS static-files sync failed for ${failed.length}/${jobs.length} file(s): ${failed
|
||||
.map((item) => item.key)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
await reconcileUploads(jobs, creds, cfg, "static-files");
|
||||
return { uploaded: jobs.length, skipped: false };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user