feat: index.json protocol adapter

This commit is contained in:
故璃
2026-07-27 15:41:04 +08:00
parent 87c37994f2
commit bd17c27023
9 changed files with 134 additions and 31 deletions
+11 -5
View File
@@ -5,9 +5,10 @@
* package and overwrites the local directory, ensuring data is in place the first time the user runs
* `bl advisor recommend`.
*
* Flow (unified skill publishing protocol: skills/index.json + one skill.tar.br per skill):
* Flow (unified skill publishing protocol: skills/index.json + one content-addressed object per skill):
* 1. Download skills/index.json from public-read OSS, get the bailian-docs-llm-wiki entry
* 2. Download skills/bailian-docs-llm-wiki/skill.tar.br (brotli q6, ~2.3MB)
* 2. Download skills/bailian-docs-llm-wiki/<entry.object> (sha256-<hex>.tar.br, brotli q6, ~2.3MB);
* legacy fallback to skill.tar.br when the entry has no valid object field
* 3. Node built-in brotli decompress + tar-stream extract (per-entry path safety check) to same-volume temp dir
* 4. renameSync atomic swap into ~/.bailian/skills/bailian-docs-llm-wiki/
* 5. Write ~/.bailian/wiki-sync-state.json
@@ -41,7 +42,10 @@ const CONFIG_DIR_NAME = ".bailian";
const SKILL_DIR_NAME = "skills/bailian-docs-llm-wiki";
const STATE_FILE_NAME = "wiki-sync-state.json";
const INDEX_KEY = "index.json";
const ASSET_NAME = "skill.tar.br";
/** Legacy fixed asset key (entries without a valid content-addressed object field) */
const LEGACY_ASSET_NAME = "skill.tar.br";
/** Same strict shape check as core registry.ts: only a valid object name may enter the URL */
const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/;
const INDEX_TIMEOUT_MS = 3000;
const DOWNLOAD_TIMEOUT_MS = 30000;
@@ -153,8 +157,10 @@ async function main() {
if (!entry?.contentHash)
throw new Error("no bailian-docs-llm-wiki entry (or contentHash) in index.json");
// 2. Download skill.tar.br (per-entry path safety check during extraction)
const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${ASSET_NAME}`);
// 2. Download the skill archive: content-addressed object first, legacy fixed key as fallback
const assetName =
entry.object && OBJECT_FILE_RE.test(entry.object) ? entry.object : LEGACY_ASSET_NAME;
const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${assetName}`);
// 3. Extract to same-volume temp dir + atomic swap
const catalogDir = getCatalogDir();
+3 -3
View File
@@ -11,7 +11,8 @@
* upsertSkillLockEntry: write lock WITH links so bl skill remove can reclaim correctly)
*
* Protocol: unified skill publishing protocol (FC publish-skills, all skills are isomorphic), entry point is
* skills/index.json, one skill.tar.br per skill (brotli q6).
* skills/index.json, one content-addressed object per skill (sha256-<hex>.tar.br, brotli q6;
* legacy fallback skill.tar.br).
*
* Complements postinstall.js (layer 1, unconditional overwrite on npm install). Install, extraction,
* validation, fan-out and lock writing all reuse the skills/ module (same as bl skill add), symmetric
@@ -45,7 +46,6 @@ interface SyncState {
}
interface SkillsIndex {
version: number;
updatedAt?: string;
skills: Record<string, SkillIndexEntry>;
}
@@ -121,7 +121,7 @@ async function fetchIndexEntry(): Promise<SkillIndexEntry | null> {
});
if (!res.ok) return null;
const index = (await res.json()) as SkillsIndex;
if (typeof index?.version !== "number" || !index.skills) return null;
if (!index?.skills || typeof index.skills !== "object") return null;
return index.skills[WIKI_SKILL_NAME] ?? null;
} catch {
return null;
+35 -1
View File
@@ -3,7 +3,16 @@
* Symmetric with the publisher (FC skills-publish.mjs: tar.pack + brotli); uses only Node built-in
* zlib + tar-stream, no extra decompression dependencies.
*/
import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
import {
createWriteStream,
existsSync,
mkdirSync,
readdirSync,
readFileSync,
renameSync,
rmSync,
} from "node:fs";
import { createHash } from "node:crypto";
import { dirname, join } from "node:path";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
@@ -46,6 +55,31 @@ export async function extractTarBr(tarBrBuffer: Buffer, destDir: string): Promis
await pipeline(Readable.from(tarBrBuffer), createBrotliDecompress(), extract);
}
/**
* Recompute the publisher's deterministic content hash over an extracted directory:
* regular files sorted by "/"-separated relative path (code-unit order, same as the
* publisher's byte-order sort for ASCII paths), sha256 accumulating relPath + bytes.
* Symmetric with computeContentHash in FC skills-publish.mjs.
*/
export function computeDirContentHash(dir: string): string {
const relPaths: string[] = [];
const walk = (sub: string): void => {
for (const dirent of readdirSync(sub ? join(dir, sub) : dir, { withFileTypes: true })) {
const rel = sub ? `${sub}/${dirent.name}` : dirent.name;
if (dirent.isDirectory()) walk(rel);
else if (dirent.isFile()) relPaths.push(rel);
}
};
walk("");
relPaths.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
const hash = createHash("sha256");
for (const rel of relPaths) {
hash.update(rel);
hash.update(readFileSync(join(dir, rel)));
}
return `sha256:${hash.digest("hex")}`;
}
/**
* Atomic swap: replace destDir with the extracted content from tmpDir.
* tmpDir must be on the same volume as destDir (same parent) for renameSync to be atomic.
+7 -2
View File
@@ -7,7 +7,12 @@ export type {
SkillStatus,
SkillStatusRow,
} from "./types.ts";
export { getSkillRegistryBaseUrl, fetchSkillsIndex, downloadSkillAsset } from "./registry.ts";
export {
getSkillRegistryBaseUrl,
fetchSkillsIndex,
downloadSkillAsset,
resolveAssetFileName,
} from "./registry.ts";
export {
getSkillsDir,
getSkillLockPath,
@@ -18,7 +23,7 @@ export {
} from "./lock.ts";
export { sanitizeSkillName, isSafeSkillName } from "./sanitize.ts";
export { validateSkillDir, type SkillMeta } from "./validate.ts";
export { extractTarBr, atomicSwap, isSafeEntryName } from "./extract.ts";
export { extractTarBr, atomicSwap, isSafeEntryName, computeDirContentHash } from "./extract.ts";
export {
getAgentTargets,
detectInstalledAgents,
+16 -3
View File
@@ -2,7 +2,7 @@ import { existsSync, mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { BailianError } from "../errors/base.ts";
import { ExitCode } from "../errors/codes.ts";
import { atomicSwap, extractTarBr } from "./extract.ts";
import { atomicSwap, computeDirContentHash, extractTarBr } from "./extract.ts";
import { getSkillsDir } from "./lock.ts";
import { downloadSkillAsset } from "./registry.ts";
import { isSafeSkillName } from "./sanitize.ts";
@@ -34,6 +34,7 @@ function assertSafeName(name: string): void {
export async function installSkillFromBuffer(
name: string,
tarBrBuffer: Buffer,
expectedContentHash?: string,
): Promise<InstalledSkill> {
assertSafeName(name);
const skillsDir = getSkillsDir();
@@ -43,6 +44,18 @@ export async function installSkillFromBuffer(
try {
mkdirSync(tmpDir, { recursive: true });
await extractTarBr(tarBrBuffer, tmpDir);
// Integrity check before touching canonical: recompute the publisher fingerprint over
// the extracted files; on mismatch the current installation is left untouched
if (expectedContentHash?.startsWith("sha256:")) {
const actualContentHash = computeDirContentHash(tmpDir);
if (actualContentHash !== expectedContentHash) {
throw new BailianError(
`Skill ${name} failed integrity check: index says ${expectedContentHash}, archive is ${actualContentHash}`,
ExitCode.GENERAL,
"Downloaded archive does not match the index fingerprint (registry may be mid-publish); retry later",
);
}
}
const meta = validateSkillDir(tmpDir, name);
atomicSwap(tmpDir, dest);
return { name, path: dest, meta };
@@ -60,8 +73,8 @@ export async function installSkill(name: string, entry: SkillIndexEntry): Promis
"Upgrade bailian-cli to the latest version and retry",
);
}
const buffer = await downloadSkillAsset(name);
return installSkillFromBuffer(name, buffer);
const buffer = await downloadSkillAsset(name, entry);
return installSkillFromBuffer(name, buffer, entry.contentHash);
}
/** Remove the skill directory under canonical; returns whether it was actually deleted (dir absent → false) */
+15 -12
View File
@@ -1,6 +1,6 @@
import { BailianError } from "../errors/base.ts";
import { ExitCode } from "../errors/codes.ts";
import type { SkillsIndex } from "./types.ts";
import type { SkillIndexEntry, SkillsIndex } from "./types.ts";
/**
* Skill registry client: public-read OSS, pure HTTPS GET, zero credentials (usable with auth: "none").
@@ -8,8 +8,6 @@ import type { SkillsIndex } from "./types.ts";
* for canary/private mirror scenarios.
*/
const DEFAULT_REGISTRY_BASE_URL = "https://bailian-wiki.oss-cn-hangzhou.aliyuncs.com/skills";
/** index.json protocol version supported by this client */
const SUPPORTED_INDEX_VERSION = 1;
const INDEX_TIMEOUT_MS = 10_000;
const ASSET_TIMEOUT_MS = 120_000;
@@ -69,19 +67,24 @@ export async function fetchSkillsIndex(): Promise<SkillsIndex> {
"Retry later or contact the publisher",
);
}
if (index.version !== SUPPORTED_INDEX_VERSION) {
throw new BailianError(
`Skill index protocol version ${index.version} is not supported by this CLI`,
ExitCode.GENERAL,
"Upgrade bailian-cli to the latest version and retry",
);
}
return index;
}
/**
* Strict shape check for entry.object (defense against a hostile/corrupted index —
* anything not matching falls back to the legacy fixed key, never into the URL path).
*/
const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/;
/** Resolve which file to download for a skill: content-addressed object, else legacy fixed key */
export function resolveAssetFileName(entry?: SkillIndexEntry): string {
const object = entry?.object;
return object && OBJECT_FILE_RE.test(object) ? object : "skill.tar.br";
}
/** Download the tar.br archive for a single skill (one skill = one GET) */
export async function downloadSkillAsset(name: string): Promise<Buffer> {
const url = `${getSkillRegistryBaseUrl()}/${name}/skill.tar.br`;
export async function downloadSkillAsset(name: string, entry?: SkillIndexEntry): Promise<Buffer> {
const url = `${getSkillRegistryBaseUrl()}/${name}/${resolveAssetFileName(entry)}`;
let res: Response;
try {
res = await fetch(url, { signal: AbortSignal.timeout(ASSET_TIMEOUT_MS) });
+9 -4
View File
@@ -2,8 +2,10 @@
* Data structures for the unified skill publishing protocol (symmetric with FC publisher skills-publish.mjs).
*
* Remote layout (public-read OSS, the sole data source for `bl skill`):
* <registry>/index.json — skill catalog (SkillsIndex)
* <registry>/<name>/skill.tar.br — one object per skill (tar + brotli, atomic publish)
* <registry>/index.json — skill catalog (SkillsIndex)
* <registry>/<name>/sha256-<hex>.tar.br — content-addressed skill object (tar + brotli);
* entry.object names the exact file, so index.json is the single atomic commit point.
* Legacy fallback: <registry>/<name>/skill.tar.br (entries without object)
*
* Local layout:
* ~/.bailian/skills/<name>/ — canonical install directory
@@ -22,11 +24,14 @@ export interface SkillIndexEntry {
contentHash?: string;
/** Compression format identifier, currently always "tar.br" */
compression?: string;
/**
* Content-addressed object file name under <registry>/<name>/, e.g. "sha256-<hex>.tar.br".
* Absent on legacy entries — client falls back to the fixed key "skill.tar.br".
*/
object?: string;
}
export interface SkillsIndex {
/** Protocol schema version, currently 1; client should error and prompt upgrade on unrecognized versions */
version: number;
updatedAt?: string;
/** key = skill name (i.e. OSS directory name, download path, local install dir name) */
skills: Record<string, SkillIndexEntry>;
@@ -1,4 +1,5 @@
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "fs";
import { createHash } from "crypto";
import { tmpdir } from "os";
import { join } from "path";
import { brotliCompressSync } from "zlib";
@@ -98,3 +99,40 @@ test("installer: invalid skill name rejected outright", async () => {
await expect(installSkillFromBuffer("../escape", buf)).rejects.toThrow(/Invalid skill name/);
});
});
/** Same accumulation as publisher computeContentHash: sorted rel path + bytes */
function expectedHashOf(files: Record<string, string>): string {
const hash = createHash("sha256");
for (const rel of Object.keys(files).sort()) {
hash.update(rel);
hash.update(Buffer.from(files[rel]));
}
return `sha256:${hash.digest("hex")}`;
}
test("installer: matching contentHash passes integrity check", async () => {
await inTempConfigDir(async () => {
const files = { "SKILL.md": VALID_SKILL_MD, "references/usage.md": "# usage\n" };
const installed = await installSkillFromBuffer(
"demo",
await buildTarBr(files),
expectedHashOf(files),
);
expect(installed.name).toBe("demo");
expect(existsSync(join(getSkillsDir(), "demo", "SKILL.md"))).toBe(true);
});
});
test("installer: contentHash mismatch → rejected, previous install preserved", async () => {
await inTempConfigDir(async () => {
await installSkillFromBuffer("demo", await buildTarBr({ "SKILL.md": VALID_SKILL_MD }));
const tampered = await buildTarBr({ "SKILL.md": VALID_SKILL_MD, "extra.md": "tampered\n" });
await expect(
installSkillFromBuffer("demo", tampered, expectedHashOf({ "SKILL.md": VALID_SKILL_MD })),
).rejects.toThrow(/integrity check/);
// Old version untouched, temp dir cleaned up
expect(readFileSync(join(getSkillsDir(), "demo", "SKILL.md"), "utf-8")).toBe(VALID_SKILL_MD);
expect(existsSync(join(getSkillsDir(), "demo", "extra.md"))).toBe(false);
expect(readdirSync(getSkillsDir()).filter((e) => e !== "demo")).toEqual([]);
});
});
@@ -7,7 +7,6 @@ const PUB = "2026-07-23T00:00:00+08:00";
function makeIndex(skills: Record<string, string>): SkillsIndex {
return {
version: 1,
skills: Object.fromEntries(
Object.entries(skills).map(([name, contentHash]) => [
name,