mirror of
https://github.com/modelstudioai/cli.git
synced 2026-09-14 19:49:23 +08:00
feat: index.json protocol adapter
This commit is contained in:
@@ -5,9 +5,10 @@
|
||||
* package and overwrites the local directory, ensuring data is in place the first time the user runs
|
||||
* `bl advisor recommend`.
|
||||
*
|
||||
* Flow (unified skill publishing protocol: skills/index.json + one skill.tar.br per skill):
|
||||
* Flow (unified skill publishing protocol: skills/index.json + one content-addressed object per skill):
|
||||
* 1. Download skills/index.json from public-read OSS, get the bailian-docs-llm-wiki entry
|
||||
* 2. Download skills/bailian-docs-llm-wiki/skill.tar.br (brotli q6, ~2.3MB)
|
||||
* 2. Download skills/bailian-docs-llm-wiki/<entry.object> (sha256-<hex>.tar.br, brotli q6, ~2.3MB);
|
||||
* legacy fallback to skill.tar.br when the entry has no valid object field
|
||||
* 3. Node built-in brotli decompress + tar-stream extract (per-entry path safety check) to same-volume temp dir
|
||||
* 4. renameSync atomic swap into ~/.bailian/skills/bailian-docs-llm-wiki/
|
||||
* 5. Write ~/.bailian/wiki-sync-state.json
|
||||
@@ -41,7 +42,10 @@ const CONFIG_DIR_NAME = ".bailian";
|
||||
const SKILL_DIR_NAME = "skills/bailian-docs-llm-wiki";
|
||||
const STATE_FILE_NAME = "wiki-sync-state.json";
|
||||
const INDEX_KEY = "index.json";
|
||||
const ASSET_NAME = "skill.tar.br";
|
||||
/** Legacy fixed asset key (entries without a valid content-addressed object field) */
|
||||
const LEGACY_ASSET_NAME = "skill.tar.br";
|
||||
/** Same strict shape check as core registry.ts: only a valid object name may enter the URL */
|
||||
const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/;
|
||||
|
||||
const INDEX_TIMEOUT_MS = 3000;
|
||||
const DOWNLOAD_TIMEOUT_MS = 30000;
|
||||
@@ -153,8 +157,10 @@ async function main() {
|
||||
if (!entry?.contentHash)
|
||||
throw new Error("no bailian-docs-llm-wiki entry (or contentHash) in index.json");
|
||||
|
||||
// 2. Download skill.tar.br (per-entry path safety check during extraction)
|
||||
const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${ASSET_NAME}`);
|
||||
// 2. Download the skill archive: content-addressed object first, legacy fixed key as fallback
|
||||
const assetName =
|
||||
entry.object && OBJECT_FILE_RE.test(entry.object) ? entry.object : LEGACY_ASSET_NAME;
|
||||
const tarBuf = await downloadBuffer(`${REGISTRY_BASE_URL}/${WIKI_SKILL_NAME}/${assetName}`);
|
||||
|
||||
// 3. Extract to same-volume temp dir + atomic swap
|
||||
const catalogDir = getCatalogDir();
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
* upsertSkillLockEntry: write lock WITH links so bl skill remove can reclaim correctly)
|
||||
*
|
||||
* Protocol: unified skill publishing protocol (FC publish-skills, all skills are isomorphic), entry point is
|
||||
* skills/index.json, one skill.tar.br per skill (brotli q6).
|
||||
* skills/index.json, one content-addressed object per skill (sha256-<hex>.tar.br, brotli q6;
|
||||
* legacy fallback skill.tar.br).
|
||||
*
|
||||
* Complements postinstall.js (layer 1, unconditional overwrite on npm install). Install, extraction,
|
||||
* validation, fan-out and lock writing all reuse the skills/ module (same as bl skill add), symmetric
|
||||
@@ -45,7 +46,6 @@ interface SyncState {
|
||||
}
|
||||
|
||||
interface SkillsIndex {
|
||||
version: number;
|
||||
updatedAt?: string;
|
||||
skills: Record<string, SkillIndexEntry>;
|
||||
}
|
||||
@@ -121,7 +121,7 @@ async function fetchIndexEntry(): Promise<SkillIndexEntry | null> {
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const index = (await res.json()) as SkillsIndex;
|
||||
if (typeof index?.version !== "number" || !index.skills) return null;
|
||||
if (!index?.skills || typeof index.skills !== "object") return null;
|
||||
return index.skills[WIKI_SKILL_NAME] ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
|
||||
@@ -3,7 +3,16 @@
|
||||
* Symmetric with the publisher (FC skills-publish.mjs: tar.pack + brotli); uses only Node built-in
|
||||
* zlib + tar-stream, no extra decompression dependencies.
|
||||
*/
|
||||
import { createWriteStream, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
||||
import {
|
||||
createWriteStream,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
} from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { dirname, join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
@@ -46,6 +55,31 @@ export async function extractTarBr(tarBrBuffer: Buffer, destDir: string): Promis
|
||||
await pipeline(Readable.from(tarBrBuffer), createBrotliDecompress(), extract);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute the publisher's deterministic content hash over an extracted directory:
|
||||
* regular files sorted by "/"-separated relative path (code-unit order, same as the
|
||||
* publisher's byte-order sort for ASCII paths), sha256 accumulating relPath + bytes.
|
||||
* Symmetric with computeContentHash in FC skills-publish.mjs.
|
||||
*/
|
||||
export function computeDirContentHash(dir: string): string {
|
||||
const relPaths: string[] = [];
|
||||
const walk = (sub: string): void => {
|
||||
for (const dirent of readdirSync(sub ? join(dir, sub) : dir, { withFileTypes: true })) {
|
||||
const rel = sub ? `${sub}/${dirent.name}` : dirent.name;
|
||||
if (dirent.isDirectory()) walk(rel);
|
||||
else if (dirent.isFile()) relPaths.push(rel);
|
||||
}
|
||||
};
|
||||
walk("");
|
||||
relPaths.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
||||
const hash = createHash("sha256");
|
||||
for (const rel of relPaths) {
|
||||
hash.update(rel);
|
||||
hash.update(readFileSync(join(dir, rel)));
|
||||
}
|
||||
return `sha256:${hash.digest("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic swap: replace destDir with the extracted content from tmpDir.
|
||||
* tmpDir must be on the same volume as destDir (same parent) for renameSync to be atomic.
|
||||
|
||||
@@ -7,7 +7,12 @@ export type {
|
||||
SkillStatus,
|
||||
SkillStatusRow,
|
||||
} from "./types.ts";
|
||||
export { getSkillRegistryBaseUrl, fetchSkillsIndex, downloadSkillAsset } from "./registry.ts";
|
||||
export {
|
||||
getSkillRegistryBaseUrl,
|
||||
fetchSkillsIndex,
|
||||
downloadSkillAsset,
|
||||
resolveAssetFileName,
|
||||
} from "./registry.ts";
|
||||
export {
|
||||
getSkillsDir,
|
||||
getSkillLockPath,
|
||||
@@ -18,7 +23,7 @@ export {
|
||||
} from "./lock.ts";
|
||||
export { sanitizeSkillName, isSafeSkillName } from "./sanitize.ts";
|
||||
export { validateSkillDir, type SkillMeta } from "./validate.ts";
|
||||
export { extractTarBr, atomicSwap, isSafeEntryName } from "./extract.ts";
|
||||
export { extractTarBr, atomicSwap, isSafeEntryName, computeDirContentHash } from "./extract.ts";
|
||||
export {
|
||||
getAgentTargets,
|
||||
detectInstalledAgents,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { existsSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { BailianError } from "../errors/base.ts";
|
||||
import { ExitCode } from "../errors/codes.ts";
|
||||
import { atomicSwap, extractTarBr } from "./extract.ts";
|
||||
import { atomicSwap, computeDirContentHash, extractTarBr } from "./extract.ts";
|
||||
import { getSkillsDir } from "./lock.ts";
|
||||
import { downloadSkillAsset } from "./registry.ts";
|
||||
import { isSafeSkillName } from "./sanitize.ts";
|
||||
@@ -34,6 +34,7 @@ function assertSafeName(name: string): void {
|
||||
export async function installSkillFromBuffer(
|
||||
name: string,
|
||||
tarBrBuffer: Buffer,
|
||||
expectedContentHash?: string,
|
||||
): Promise<InstalledSkill> {
|
||||
assertSafeName(name);
|
||||
const skillsDir = getSkillsDir();
|
||||
@@ -43,6 +44,18 @@ export async function installSkillFromBuffer(
|
||||
try {
|
||||
mkdirSync(tmpDir, { recursive: true });
|
||||
await extractTarBr(tarBrBuffer, tmpDir);
|
||||
// Integrity check before touching canonical: recompute the publisher fingerprint over
|
||||
// the extracted files; on mismatch the current installation is left untouched
|
||||
if (expectedContentHash?.startsWith("sha256:")) {
|
||||
const actualContentHash = computeDirContentHash(tmpDir);
|
||||
if (actualContentHash !== expectedContentHash) {
|
||||
throw new BailianError(
|
||||
`Skill ${name} failed integrity check: index says ${expectedContentHash}, archive is ${actualContentHash}`,
|
||||
ExitCode.GENERAL,
|
||||
"Downloaded archive does not match the index fingerprint (registry may be mid-publish); retry later",
|
||||
);
|
||||
}
|
||||
}
|
||||
const meta = validateSkillDir(tmpDir, name);
|
||||
atomicSwap(tmpDir, dest);
|
||||
return { name, path: dest, meta };
|
||||
@@ -60,8 +73,8 @@ export async function installSkill(name: string, entry: SkillIndexEntry): Promis
|
||||
"Upgrade bailian-cli to the latest version and retry",
|
||||
);
|
||||
}
|
||||
const buffer = await downloadSkillAsset(name);
|
||||
return installSkillFromBuffer(name, buffer);
|
||||
const buffer = await downloadSkillAsset(name, entry);
|
||||
return installSkillFromBuffer(name, buffer, entry.contentHash);
|
||||
}
|
||||
|
||||
/** Remove the skill directory under canonical; returns whether it was actually deleted (dir absent → false) */
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { BailianError } from "../errors/base.ts";
|
||||
import { ExitCode } from "../errors/codes.ts";
|
||||
import type { SkillsIndex } from "./types.ts";
|
||||
import type { SkillIndexEntry, SkillsIndex } from "./types.ts";
|
||||
|
||||
/**
|
||||
* Skill registry client: public-read OSS, pure HTTPS GET, zero credentials (usable with auth: "none").
|
||||
@@ -8,8 +8,6 @@ import type { SkillsIndex } from "./types.ts";
|
||||
* for canary/private mirror scenarios.
|
||||
*/
|
||||
const DEFAULT_REGISTRY_BASE_URL = "https://bailian-wiki.oss-cn-hangzhou.aliyuncs.com/skills";
|
||||
/** index.json protocol version supported by this client */
|
||||
const SUPPORTED_INDEX_VERSION = 1;
|
||||
|
||||
const INDEX_TIMEOUT_MS = 10_000;
|
||||
const ASSET_TIMEOUT_MS = 120_000;
|
||||
@@ -69,19 +67,24 @@ export async function fetchSkillsIndex(): Promise<SkillsIndex> {
|
||||
"Retry later or contact the publisher",
|
||||
);
|
||||
}
|
||||
if (index.version !== SUPPORTED_INDEX_VERSION) {
|
||||
throw new BailianError(
|
||||
`Skill index protocol version ${index.version} is not supported by this CLI`,
|
||||
ExitCode.GENERAL,
|
||||
"Upgrade bailian-cli to the latest version and retry",
|
||||
);
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict shape check for entry.object (defense against a hostile/corrupted index —
|
||||
* anything not matching falls back to the legacy fixed key, never into the URL path).
|
||||
*/
|
||||
const OBJECT_FILE_RE = /^sha256-[0-9a-f]{64}\.tar\.br$/;
|
||||
|
||||
/** Resolve which file to download for a skill: content-addressed object, else legacy fixed key */
|
||||
export function resolveAssetFileName(entry?: SkillIndexEntry): string {
|
||||
const object = entry?.object;
|
||||
return object && OBJECT_FILE_RE.test(object) ? object : "skill.tar.br";
|
||||
}
|
||||
|
||||
/** Download the tar.br archive for a single skill (one skill = one GET) */
|
||||
export async function downloadSkillAsset(name: string): Promise<Buffer> {
|
||||
const url = `${getSkillRegistryBaseUrl()}/${name}/skill.tar.br`;
|
||||
export async function downloadSkillAsset(name: string, entry?: SkillIndexEntry): Promise<Buffer> {
|
||||
const url = `${getSkillRegistryBaseUrl()}/${name}/${resolveAssetFileName(entry)}`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, { signal: AbortSignal.timeout(ASSET_TIMEOUT_MS) });
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
* Data structures for the unified skill publishing protocol (symmetric with FC publisher skills-publish.mjs).
|
||||
*
|
||||
* Remote layout (public-read OSS, the sole data source for `bl skill`):
|
||||
* <registry>/index.json — skill catalog (SkillsIndex)
|
||||
* <registry>/<name>/skill.tar.br — one object per skill (tar + brotli, atomic publish)
|
||||
* <registry>/index.json — skill catalog (SkillsIndex)
|
||||
* <registry>/<name>/sha256-<hex>.tar.br — content-addressed skill object (tar + brotli);
|
||||
* entry.object names the exact file, so index.json is the single atomic commit point.
|
||||
* Legacy fallback: <registry>/<name>/skill.tar.br (entries without object)
|
||||
*
|
||||
* Local layout:
|
||||
* ~/.bailian/skills/<name>/ — canonical install directory
|
||||
@@ -22,11 +24,14 @@ export interface SkillIndexEntry {
|
||||
contentHash?: string;
|
||||
/** Compression format identifier, currently always "tar.br" */
|
||||
compression?: string;
|
||||
/**
|
||||
* Content-addressed object file name under <registry>/<name>/, e.g. "sha256-<hex>.tar.br".
|
||||
* Absent on legacy entries — client falls back to the fixed key "skill.tar.br".
|
||||
*/
|
||||
object?: string;
|
||||
}
|
||||
|
||||
export interface SkillsIndex {
|
||||
/** Protocol schema version, currently 1; client should error and prompt upgrade on unrecognized versions */
|
||||
version: number;
|
||||
updatedAt?: string;
|
||||
/** key = skill name (i.e. OSS directory name, download path, local install dir name) */
|
||||
skills: Record<string, SkillIndexEntry>;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "fs";
|
||||
import { createHash } from "crypto";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import { brotliCompressSync } from "zlib";
|
||||
@@ -98,3 +99,40 @@ test("installer: invalid skill name rejected outright", async () => {
|
||||
await expect(installSkillFromBuffer("../escape", buf)).rejects.toThrow(/Invalid skill name/);
|
||||
});
|
||||
});
|
||||
|
||||
/** Same accumulation as publisher computeContentHash: sorted rel path + bytes */
|
||||
function expectedHashOf(files: Record<string, string>): string {
|
||||
const hash = createHash("sha256");
|
||||
for (const rel of Object.keys(files).sort()) {
|
||||
hash.update(rel);
|
||||
hash.update(Buffer.from(files[rel]));
|
||||
}
|
||||
return `sha256:${hash.digest("hex")}`;
|
||||
}
|
||||
|
||||
test("installer: matching contentHash passes integrity check", async () => {
|
||||
await inTempConfigDir(async () => {
|
||||
const files = { "SKILL.md": VALID_SKILL_MD, "references/usage.md": "# usage\n" };
|
||||
const installed = await installSkillFromBuffer(
|
||||
"demo",
|
||||
await buildTarBr(files),
|
||||
expectedHashOf(files),
|
||||
);
|
||||
expect(installed.name).toBe("demo");
|
||||
expect(existsSync(join(getSkillsDir(), "demo", "SKILL.md"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test("installer: contentHash mismatch → rejected, previous install preserved", async () => {
|
||||
await inTempConfigDir(async () => {
|
||||
await installSkillFromBuffer("demo", await buildTarBr({ "SKILL.md": VALID_SKILL_MD }));
|
||||
const tampered = await buildTarBr({ "SKILL.md": VALID_SKILL_MD, "extra.md": "tampered\n" });
|
||||
await expect(
|
||||
installSkillFromBuffer("demo", tampered, expectedHashOf({ "SKILL.md": VALID_SKILL_MD })),
|
||||
).rejects.toThrow(/integrity check/);
|
||||
// Old version untouched, temp dir cleaned up
|
||||
expect(readFileSync(join(getSkillsDir(), "demo", "SKILL.md"), "utf-8")).toBe(VALID_SKILL_MD);
|
||||
expect(existsSync(join(getSkillsDir(), "demo", "extra.md"))).toBe(false);
|
||||
expect(readdirSync(getSkillsDir()).filter((e) => e !== "demo")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,7 +7,6 @@ const PUB = "2026-07-23T00:00:00+08:00";
|
||||
|
||||
function makeIndex(skills: Record<string, string>): SkillsIndex {
|
||||
return {
|
||||
version: 1,
|
||||
skills: Object.fromEntries(
|
||||
Object.entries(skills).map(([name, contentHash]) => [
|
||||
name,
|
||||
|
||||
Reference in New Issue
Block a user