From bd17c27023048eb0a0056d41ef8f94c5e0febf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=95=85=E7=92=83?= Date: Mon, 27 Jul 2026 15:41:04 +0800 Subject: [PATCH] feat: index.json protocol adapter --- packages/cli/postinstall.js | 16 ++++++--- packages/core/src/advisor/sync.ts | 6 ++-- packages/core/src/skills/extract.ts | 36 ++++++++++++++++++- packages/core/src/skills/index.ts | 9 +++-- packages/core/src/skills/installer.ts | 19 ++++++++-- packages/core/src/skills/registry.ts | 27 +++++++------- packages/core/src/skills/types.ts | 13 ++++--- packages/core/tests/skills-installer.test.ts | 38 ++++++++++++++++++++ packages/core/tests/skills-status.test.ts | 1 - 9 files changed, 134 insertions(+), 31 deletions(-) diff --git a/packages/cli/postinstall.js b/packages/cli/postinstall.js index b9b4e96..ee99f66 100644 --- a/packages/cli/postinstall.js +++ b/packages/cli/postinstall.js @@ -5,9 +5,10 @@ * package and overwrites the local directory, ensuring data is in place the first time the user runs * `bl advisor recommend`. * - * Flow (unified skill publishing protocol: skills/index.json + one skill.tar.br per skill): + * Flow (unified skill publishing protocol: skills/index.json + one content-addressed object per skill): * 1. Download skills/index.json from public-read OSS, get the bailian-docs-llm-wiki entry - * 2. Download skills/bailian-docs-llm-wiki/skill.tar.br (brotli q6, ~2.3MB) + * 2. Download skills/bailian-docs-llm-wiki/ (sha256-.tar.br, brotli q6, ~2.3MB); + * legacy fallback to skill.tar.br when the entry has no valid object field * 3. Node built-in brotli decompress + tar-stream extract (per-entry path safety check) to same-volume temp dir * 4. renameSync atomic swap into ~/.bailian/skills/bailian-docs-llm-wiki/ * 5. Write ~/.bailian/wiki-sync-state.json @@ -41,7 +42,10 @@ const CONFIG_DIR_NAME = ".bailian"; const SKILL_DIR_NAME = "skills/bailian-docs-llm-wiki"; const STATE_FILE_NAME = "wiki-sync-state.json"; const INDEX_KEY = "index.json"; -const ASSET_NAME = "skill.tar.br"; +/** Legacy fixed asset key (entries without a valid content-addressed object field) */ +const LEGACY_ASSET_NAME = "skill.tar.br"; +/** Same strict shape check as core registry.ts: only a valid object name may enter the URL */ +const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/; const INDEX_TIMEOUT_MS = 3000; const DOWNLOAD_TIMEOUT_MS = 30000; @@ -153,8 +157,10 @@ async function main() { if (!entry?.contentHash) throw new Error("no bailian-docs-llm-wiki entry (or contentHash) in index.json"); - // 2. Download skill.tar.br (per-entry path safety check during extraction) - const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${ASSET_NAME}`); + // 2. Download the skill archive: content-addressed object first, legacy fixed key as fallback + const assetName = + entry.object && OBJECT_FILE_RE.test(entry.object) ? entry.object : LEGACY_ASSET_NAME; + const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${assetName}`); // 3. Extract to same-volume temp dir + atomic swap const catalogDir = getCatalogDir(); diff --git a/packages/core/src/advisor/sync.ts b/packages/core/src/advisor/sync.ts index 04d80b5..70ff0be 100644 --- a/packages/core/src/advisor/sync.ts +++ b/packages/core/src/advisor/sync.ts @@ -11,7 +11,8 @@ * upsertSkillLockEntry: write lock WITH links so bl skill remove can reclaim correctly) * * Protocol: unified skill publishing protocol (FC publish-skills, all skills are isomorphic), entry point is - * skills/index.json, one skill.tar.br per skill (brotli q6). + * skills/index.json, one content-addressed object per skill (sha256-.tar.br, brotli q6; + * legacy fallback skill.tar.br). * * Complements postinstall.js (layer 1, unconditional overwrite on npm install). Install, extraction, * validation, fan-out and lock writing all reuse the skills/ module (same as bl skill add), symmetric @@ -45,7 +46,6 @@ interface SyncState { } interface SkillsIndex { - version: number; updatedAt?: string; skills: Record; } @@ -121,7 +121,7 @@ async function fetchIndexEntry(): Promise { }); if (!res.ok) return null; const index = (await res.json()) as SkillsIndex; - if (typeof index?.version !== "number" || !index.skills) return null; + if (!index?.skills || typeof index.skills !== "object") return null; return index.skills[WIKI_SKILL_NAME] ?? null; } catch { return null; diff --git a/packages/core/src/skills/extract.ts b/packages/core/src/skills/extract.ts index 9325ec6..48a8a99 100644 --- a/packages/core/src/skills/extract.ts +++ b/packages/core/src/skills/extract.ts @@ -3,7 +3,16 @@ * Symmetric with the publisher (FC skills-publish.mjs: tar.pack + brotli); uses only Node built-in * zlib + tar-stream, no extra decompression dependencies. */ -import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync } from "node:fs"; +import { + createWriteStream, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, +} from "node:fs"; +import { createHash } from "node:crypto"; import { dirname, join } from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -46,6 +55,31 @@ export async function extractTarBr(tarBrBuffer: Buffer, destDir: string): Promis await pipeline(Readable.from(tarBrBuffer), createBrotliDecompress(), extract); } +/** + * Recompute the publisher's deterministic content hash over an extracted directory: + * regular files sorted by "/"-separated relative path (code-unit order, same as the + * publisher's byte-order sort for ASCII paths), sha256 accumulating relPath + bytes. + * Symmetric with computeContentHash in FC skills-publish.mjs. + */ +export function computeDirContentHash(dir: string): string { + const relPaths: string[] = []; + const walk = (sub: string): void => { + for (const dirent of readdirSync(sub ? join(dir, sub) : dir, { withFileTypes: true })) { + const rel = sub ? `${sub}/${dirent.name}` : dirent.name; + if (dirent.isDirectory()) walk(rel); + else if (dirent.isFile()) relPaths.push(rel); + } + }; + walk(""); + relPaths.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); + const hash = createHash("sha256"); + for (const rel of relPaths) { + hash.update(rel); + hash.update(readFileSync(join(dir, rel))); + } + return `sha256:${hash.digest("hex")}`; +} + /** * Atomic swap: replace destDir with the extracted content from tmpDir. * tmpDir must be on the same volume as destDir (same parent) for renameSync to be atomic. diff --git a/packages/core/src/skills/index.ts b/packages/core/src/skills/index.ts index 91300b9..fd2ec5c 100644 --- a/packages/core/src/skills/index.ts +++ b/packages/core/src/skills/index.ts @@ -7,7 +7,12 @@ export type { SkillStatus, SkillStatusRow, } from "./types.ts"; -export { getSkillRegistryBaseUrl, fetchSkillsIndex, downloadSkillAsset } from "./registry.ts"; +export { + getSkillRegistryBaseUrl, + fetchSkillsIndex, + downloadSkillAsset, + resolveAssetFileName, +} from "./registry.ts"; export { getSkillsDir, getSkillLockPath, @@ -18,7 +23,7 @@ export { } from "./lock.ts"; export { sanitizeSkillName, isSafeSkillName } from "./sanitize.ts"; export { validateSkillDir, type SkillMeta } from "./validate.ts"; -export { extractTarBr, atomicSwap, isSafeEntryName } from "./extract.ts"; +export { extractTarBr, atomicSwap, isSafeEntryName, computeDirContentHash } from "./extract.ts"; export { getAgentTargets, detectInstalledAgents, diff --git a/packages/core/src/skills/installer.ts b/packages/core/src/skills/installer.ts index 333609f..5d9d42c 100644 --- a/packages/core/src/skills/installer.ts +++ b/packages/core/src/skills/installer.ts @@ -2,7 +2,7 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { BailianError } from "../errors/base.ts"; import { ExitCode } from "../errors/codes.ts"; -import { atomicSwap, extractTarBr } from "./extract.ts"; +import { atomicSwap, computeDirContentHash, extractTarBr } from "./extract.ts"; import { getSkillsDir } from "./lock.ts"; import { downloadSkillAsset } from "./registry.ts"; import { isSafeSkillName } from "./sanitize.ts"; @@ -34,6 +34,7 @@ function assertSafeName(name: string): void { export async function installSkillFromBuffer( name: string, tarBrBuffer: Buffer, + expectedContentHash?: string, ): Promise { assertSafeName(name); const skillsDir = getSkillsDir(); @@ -43,6 +44,18 @@ export async function installSkillFromBuffer( try { mkdirSync(tmpDir, { recursive: true }); await extractTarBr(tarBrBuffer, tmpDir); + // Integrity check before touching canonical: recompute the publisher fingerprint over + // the extracted files; on mismatch the current installation is left untouched + if (expectedContentHash?.startsWith("sha256:")) { + const actualContentHash = computeDirContentHash(tmpDir); + if (actualContentHash !== expectedContentHash) { + throw new BailianError( + `Skill ${name} failed integrity check: index says ${expectedContentHash}, archive is ${actualContentHash}`, + ExitCode.GENERAL, + "Downloaded archive does not match the index fingerprint (registry may be mid-publish); retry later", + ); + } + } const meta = validateSkillDir(tmpDir, name); atomicSwap(tmpDir, dest); return { name, path: dest, meta }; @@ -60,8 +73,8 @@ export async function installSkill(name: string, entry: SkillIndexEntry): Promis "Upgrade bailian-cli to the latest version and retry", ); } - const buffer = await downloadSkillAsset(name); - return installSkillFromBuffer(name, buffer); + const buffer = await downloadSkillAsset(name, entry); + return installSkillFromBuffer(name, buffer, entry.contentHash); } /** Remove the skill directory under canonical; returns whether it was actually deleted (dir absent → false) */ diff --git a/packages/core/src/skills/registry.ts b/packages/core/src/skills/registry.ts index 942899f..534403d 100644 --- a/packages/core/src/skills/registry.ts +++ b/packages/core/src/skills/registry.ts @@ -1,6 +1,6 @@ import { BailianError } from "../errors/base.ts"; import { ExitCode } from "../errors/codes.ts"; -import type { SkillsIndex } from "./types.ts"; +import type { SkillIndexEntry, SkillsIndex } from "./types.ts"; /** * Skill registry client: public-read OSS, pure HTTPS GET, zero credentials (usable with auth: "none"). @@ -8,8 +8,6 @@ import type { SkillsIndex } from "./types.ts"; * for canary/private mirror scenarios. */ const DEFAULT_REGISTRY_BASE_URL = "https://bailian-wiki.oss-cn-hangzhou.aliyuncs.com/skills"; -/** index.json protocol version supported by this client */ -const SUPPORTED_INDEX_VERSION = 1; const INDEX_TIMEOUT_MS = 10_000; const ASSET_TIMEOUT_MS = 120_000; @@ -69,19 +67,24 @@ export async function fetchSkillsIndex(): Promise { "Retry later or contact the publisher", ); } - if (index.version !== SUPPORTED_INDEX_VERSION) { - throw new BailianError( - `Skill index protocol version ${index.version} is not supported by this CLI`, - ExitCode.GENERAL, - "Upgrade bailian-cli to the latest version and retry", - ); - } return index; } +/** + * Strict shape check for entry.object (defense against a hostile/corrupted index — + * anything not matching falls back to the legacy fixed key, never into the URL path). + */ +const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/; + +/** Resolve which file to download for a skill: content-addressed object, else legacy fixed key */ +export function resolveAssetFileName(entry?: SkillIndexEntry): string { + const object = entry?.object; + return object && OBJECT_FILE_RE.test(object) ? object : "skill.tar.br"; +} + /** Download the tar.br archive for a single skill (one skill = one GET) */ -export async function downloadSkillAsset(name: string): Promise { - const url = `${getSkillRegistryBaseUrl()}/${name}/skill.tar.br`; +export async function downloadSkillAsset(name: string, entry?: SkillIndexEntry): Promise { + const url = `${getSkillRegistryBaseUrl()}/${name}/${resolveAssetFileName(entry)}`; let res: Response; try { res = await fetch(url, { signal: AbortSignal.timeout(ASSET_TIMEOUT_MS) }); diff --git a/packages/core/src/skills/types.ts b/packages/core/src/skills/types.ts index d8b865a..b6fa9aa 100644 --- a/packages/core/src/skills/types.ts +++ b/packages/core/src/skills/types.ts @@ -2,8 +2,10 @@ * Data structures for the unified skill publishing protocol (symmetric with FC publisher skills-publish.mjs). * * Remote layout (public-read OSS, the sole data source for `bl skill`): - * /index.json — skill catalog (SkillsIndex) - * //skill.tar.br — one object per skill (tar + brotli, atomic publish) + * /index.json — skill catalog (SkillsIndex) + * //sha256-.tar.br — content-addressed skill object (tar + brotli); + * entry.object names the exact file, so index.json is the single atomic commit point. + * Legacy fallback: //skill.tar.br (entries without object) * * Local layout: * ~/.bailian/skills// — canonical install directory @@ -22,11 +24,14 @@ export interface SkillIndexEntry { contentHash?: string; /** Compression format identifier, currently always "tar.br" */ compression?: string; + /** + * Content-addressed object file name under //, e.g. "sha256-.tar.br". + * Absent on legacy entries — client falls back to the fixed key "skill.tar.br". + */ + object?: string; } export interface SkillsIndex { - /** Protocol schema version, currently 1; client should error and prompt upgrade on unrecognized versions */ - version: number; updatedAt?: string; /** key = skill name (i.e. OSS directory name, download path, local install dir name) */ skills: Record; diff --git a/packages/core/tests/skills-installer.test.ts b/packages/core/tests/skills-installer.test.ts index b7ac94b..63dd3d0 100644 --- a/packages/core/tests/skills-installer.test.ts +++ b/packages/core/tests/skills-installer.test.ts @@ -1,4 +1,5 @@ import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "fs"; +import { createHash } from "crypto"; import { tmpdir } from "os"; import { join } from "path"; import { brotliCompressSync } from "zlib"; @@ -98,3 +99,40 @@ test("installer: invalid skill name rejected outright", async () => { await expect(installSkillFromBuffer("../escape", buf)).rejects.toThrow(/Invalid skill name/); }); }); + +/** Same accumulation as publisher computeContentHash: sorted rel path + bytes */ +function expectedHashOf(files: Record): string { + const hash = createHash("sha256"); + for (const rel of Object.keys(files).sort()) { + hash.update(rel); + hash.update(Buffer.from(files[rel])); + } + return `sha256:${hash.digest("hex")}`; +} + +test("installer: matching contentHash passes integrity check", async () => { + await inTempConfigDir(async () => { + const files = { "SKILL.md": VALID_SKILL_MD, "references/usage.md": "# usage\n" }; + const installed = await installSkillFromBuffer( + "demo", + await buildTarBr(files), + expectedHashOf(files), + ); + expect(installed.name).toBe("demo"); + expect(existsSync(join(getSkillsDir(), "demo", "SKILL.md"))).toBe(true); + }); +}); + +test("installer: contentHash mismatch → rejected, previous install preserved", async () => { + await inTempConfigDir(async () => { + await installSkillFromBuffer("demo", await buildTarBr({ "SKILL.md": VALID_SKILL_MD })); + const tampered = await buildTarBr({ "SKILL.md": VALID_SKILL_MD, "extra.md": "tampered\n" }); + await expect( + installSkillFromBuffer("demo", tampered, expectedHashOf({ "SKILL.md": VALID_SKILL_MD })), + ).rejects.toThrow(/integrity check/); + // Old version untouched, temp dir cleaned up + expect(readFileSync(join(getSkillsDir(), "demo", "SKILL.md"), "utf-8")).toBe(VALID_SKILL_MD); + expect(existsSync(join(getSkillsDir(), "demo", "extra.md"))).toBe(false); + expect(readdirSync(getSkillsDir()).filter((e) => e !== "demo")).toEqual([]); + }); +}); diff --git a/packages/core/tests/skills-status.test.ts b/packages/core/tests/skills-status.test.ts index 31edd32..c84cec8 100644 --- a/packages/core/tests/skills-status.test.ts +++ b/packages/core/tests/skills-status.test.ts @@ -7,7 +7,6 @@ const PUB = "2026-07-23T00:00:00+08:00"; function makeIndex(skills: Record): SkillsIndex { return { - version: 1, skills: Object.fromEntries( Object.entries(skills).map(([name, contentHash]) => [ name,