packages: add remote facts for community entries (#1372)

Generate and validate a bounded facts index, then use it to render
package details.

.github/workflows/deploy.yml

@@ -30,7 +30,7 @@ jobs:
         with:
           path: ./packages/web
           package-manager: bun@1.3.14
-          node-version: 22
+          node-version: 26
 
   deploy:
     needs: build

packages/web/package.json

@@ -18,7 +18,7 @@
     "validate:docs:examples": "bun scripts/verify-doc-examples.ts",
     "validate:docs": "bun run validate:docs:metadata && bun run validate:docs:links && bun run validate:docs:skill && bun run validate:docs:examples",
     "validate:packages": "bun scripts/validate-packages.ts src/content/packages",
-    "test:packages": "bun test src/lib/package-facts.test.ts"
+    "test:packages": "bun test src/lib/package-facts.test.ts src/lib/package-schema.test.ts src/lib/remote-package-facts.test.ts"
   },
   "dependencies": {
     "@astrojs/mdx": "^7.0.5",

packages/web/scripts/package-entry-file.ts

@@ -0,0 +1,34 @@
+import { promises as fs } from "node:fs"
+import { basename } from "node:path"
+import { packageEntrySchema, type PackageEntry } from "../src/lib/package-schema"
+
+const FILE_SIZE_MAX = 256 * 1024
+
+export async function parsePackageEntryFile(path: string): Promise<PackageEntry> {
+  const fileSize = (await fs.stat(path)).size
+  if (fileSize > FILE_SIZE_MAX) throw new Error(`${path} is ${fileSize} bytes; maximum is ${FILE_SIZE_MAX}`)
+
+  const contents = await fs.readFile(path, "utf8")
+  const match = contents.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)
+  if (!match) throw new Error(`${path}: missing YAML frontmatter`)
+
+  let data: unknown
+  try {
+    data = Bun.YAML.parse(match[1])
+  } catch (error) {
+    throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`)
+  }
+
+  const result = packageEntrySchema.safeParse(data)
+  if (!result.success) {
+    const issues = result.error.issues.map((issue) => {
+      const field = issue.path.length ? `${issue.path.join(".")}: ` : ""
+      return `${field}${issue.message}`
+    })
+    throw new Error(`${path}: ${issues.join("; ")}`)
+  }
+
+  const filename = basename(path, ".mdx")
+  if (result.data.id !== filename) throw new Error(`${path}: frontmatter id must match filename \`${filename}\``)
+  return result.data
+}

packages/web/scripts/update-package-facts.ts

@@ -0,0 +1,307 @@
+#!/usr/bin/env bun
+
+import { promises as fs } from "node:fs"
+import { resolve } from "node:path"
+import { githubRepository, githubRepositoryIdentifier, type PackageEntry } from "../src/lib/package-schema"
+import {
+  PACKAGE_COUNT_MAX,
+  remotePackageFactsSchema,
+  type RemotePackageFact,
+  type RemotePackageFacts,
+} from "../src/lib/remote-package-facts"
+import { parsePackageEntryFile } from "./package-entry-file"
+
+const REQUEST_TIMEOUT_MS = 10_000
+const RESPONSE_SIZE_MAX = 1024 * 1024
+const GITHUB_REQUEST_MAX = 500
+
+interface Requester {
+  json(url: string): Promise<unknown>
+  exists(url: string, method?: "GET" | "HEAD"): Promise<void>
+}
+
+export interface UpdatePackageFactsOptions {
+  firstPartyDirectory?: string
+  fetch?: typeof fetch
+  githubToken?: string
+}
+
+export async function updatePackageFacts(
+  communityDirectory: string,
+  options: UpdatePackageFactsOptions = {},
+): Promise<RemotePackageFacts> {
+  const targetDirectory = resolve(communityDirectory)
+  const firstPartyDirectory = resolve(
+    options.firstPartyDirectory ?? resolve(import.meta.dir, "../src/content/packages"),
+  )
+  const directories = [...new Set([firstPartyDirectory, targetDirectory])]
+  const files = (await Promise.all(directories.map(readEntryDirectory))).flat().toSorted()
+  if (files.length > PACKAGE_COUNT_MAX) {
+    throw new Error(`Package entry count ${files.length} exceeds maximum ${PACKAGE_COUNT_MAX}`)
+  }
+
+  const entries = await Promise.all(files.map(async (path) => ({ entry: await parsePackageEntryFile(path), path })))
+  const pathsById = new Map<string, string>()
+  for (const { entry, path } of entries) {
+    const previous = pathsById.get(entry.id)
+    if (previous) throw new Error(`Duplicate package id \`${entry.id}\`: ${previous}, ${path}`)
+    pathsById.set(entry.id, path)
+  }
+
+  const request = createRequester(options.fetch ?? fetch, options.githubToken)
+  const packages: RemotePackageFact[] = []
+  for (const { entry } of entries) {
+    const fact = await generateFact(entry, request)
+    if (fact) packages.push(fact)
+  }
+  packages.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0))
+  const facts = remotePackageFactsSchema.parse({ schemaVersion: 1, packages })
+  const contents = `${JSON.stringify(facts, null, 2)}\n`
+  const outputPath = resolve(targetDirectory, "facts.json")
+  const temporaryPath = `${outputPath}.${process.pid}.tmp`
+
+  try {
+    await fs.writeFile(temporaryPath, contents, "utf8")
+    await fs.rename(temporaryPath, outputPath)
+  } catch (error) {
+    await fs.rm(temporaryPath, { force: true })
+    throw error
+  }
+
+  return facts
+}
+
+async function readEntryDirectory(directory: string): Promise<string[]> {
+  const entries = await fs.readdir(directory, { withFileTypes: true })
+  return entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith(".mdx"))
+    .map((entry) => resolve(directory, entry.name))
+}
+
+async function generateFact(entry: PackageEntry, request: Requester): Promise<RemotePackageFact | undefined> {
+  const fact: RemotePackageFact = { id: entry.id }
+  const sourceRepository = githubRepository(entry.source.url)
+  let repositoryMetadata: unknown
+
+  if (sourceRepository) {
+    repositoryMetadata = await request.json(githubApiUrl(sourceRepository))
+    fact.stars = requiredSafeInteger(repositoryMetadata, "stargazers_count", `GitHub repository ${sourceRepository}`)
+  } else if (/^https?:/i.test(entry.source.url)) {
+    await request.exists(entry.source.url)
+  } else {
+    throw new Error(`Cannot verify non-HTTP source URL: ${entry.source.url}`)
+  }
+
+  const npm = !entry.official
+    ? entry.distributions.find((distribution) => distribution.type === "npm" && distribution.identifier !== undefined)
+    : undefined
+  const release =
+    !entry.official && !npm
+      ? entry.distributions.find(
+          (distribution) => distribution.type === "github-release" && distribution.identifier !== undefined,
+        )
+      : undefined
+  let npmMetadata: unknown
+  let releaseMetadata: unknown
+  for (const distribution of entry.distributions) {
+    if (distribution.type === "npm" && distribution.identifier) {
+      const url = `https://registry.npmjs.org/${encodeURIComponent(distribution.identifier)}`
+      if (distribution === npm) npmMetadata = await request.json(`${url}/latest`)
+      else await request.exists(url, "HEAD")
+    } else if (distribution.type === "github-release" && distribution.identifier) {
+      const repository = githubRepositoryIdentifier(distribution.identifier)
+      if (!repository) throw new Error(`Invalid GitHub repository identifier: ${distribution.identifier}`)
+      if (distribution === release) releaseMetadata = await request.json(`${githubApiUrl(repository)}/releases/latest`)
+      else await request.json(githubApiUrl(repository))
+    }
+  }
+
+  if (entry.official) return fact
+
+  if (npm?.identifier) {
+    const version = objectString(npmMetadata, "version")
+    if (!version) throw new Error(`npm package ${npm.identifier} has no latest version`)
+    fact.version = version
+    fact.license = normalizeLicense(nestedValue(npmMetadata, ["license"])) ?? repositoryLicense(repositoryMetadata)
+    return fact
+  }
+
+  if (release?.identifier) {
+    const version = objectString(releaseMetadata, "tag_name")
+    if (!version) throw new Error(`GitHub repository ${release.identifier} has no latest release tag`)
+    fact.version = version
+  }
+  fact.license = repositoryLicense(repositoryMetadata)
+  return fact.version === undefined && fact.license === undefined && fact.stars === undefined ? undefined : fact
+}
+
+function createRequester(fetch_: typeof fetch, githubToken?: string): Requester {
+  if (githubToken !== undefined && (githubToken.length === 0 || /\s/.test(githubToken))) {
+    throw new Error("GITHUB_TOKEN must be a nonempty token without whitespace")
+  }
+  const jsonRequests = new Map<string, Promise<unknown>>()
+  const existenceRequests = new Map<string, Promise<void>>()
+  let githubRequestCount = 0
+
+  const checkGithubBudget = (url: string) => {
+    if (new URL(url).hostname !== "api.github.com") return
+    if (githubRequestCount === GITHUB_REQUEST_MAX) {
+      throw new Error(`GitHub request count exceeds maximum ${GITHUB_REQUEST_MAX}`)
+    }
+    githubRequestCount++
+  }
+
+  return {
+    json(url) {
+      const existing = jsonRequests.get(url)
+      if (existing) return existing
+      checkGithubBudget(url)
+      const result = fetchJson(fetch_, url, githubToken)
+      jsonRequests.set(url, result)
+      return result
+    },
+    exists(url, method = "GET") {
+      const key = `${method} ${url}`
+      const existing = existenceRequests.get(key)
+      if (existing) return existing
+      checkGithubBudget(url)
+      const result = fetchExists(fetch_, url, method, githubToken)
+      existenceRequests.set(key, result)
+      return result
+    },
+  }
+}
+
+async function fetchJson(fetch_: typeof fetch, url: string, githubToken?: string): Promise<unknown> {
+  const response = await fetchResponse(fetch_, url, "GET", githubToken)
+  try {
+    const contentLength = Number(response.headers.get("content-length"))
+    if (Number.isFinite(contentLength) && contentLength > RESPONSE_SIZE_MAX) {
+      throw new Error(`response is ${contentLength} bytes; maximum is ${RESPONSE_SIZE_MAX}`)
+    }
+    const body = await readResponseBody(response)
+    return JSON.parse(body)
+  } catch (error) {
+    throw new Error(`Invalid JSON from ${url}: ${errorMessage(error)}`)
+  }
+}
+
+async function fetchExists(
+  fetch_: typeof fetch,
+  url: string,
+  method: "GET" | "HEAD",
+  githubToken?: string,
+): Promise<void> {
+  const response = await fetchResponse(fetch_, url, method, githubToken)
+  if (method === "GET") await response.body?.cancel()
+}
+
+async function fetchResponse(
+  fetch_: typeof fetch,
+  url: string,
+  method: "GET" | "HEAD",
+  githubToken?: string,
+): Promise<Response> {
+  const github = new URL(url).hostname === "api.github.com"
+  const headers: Record<string, string> = { "User-Agent": "opentui-package-facts" }
+  if (github) {
+    headers.Accept = "application/vnd.github+json"
+    headers["X-GitHub-Api-Version"] = "2022-11-28"
+    if (githubToken) headers.Authorization = `Bearer ${githubToken}`
+  }
+
+  let response: Response
+  try {
+    response = await fetch_(url, { method, headers, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) })
+  } catch (error) {
+    throw new Error(`Request failed for ${url}: ${redactToken(errorMessage(error), githubToken)}`)
+  }
+  if (response.ok) return response
+
+  const remaining = response.headers.get("x-ratelimit-remaining")
+  const retryAfter = response.headers.get("retry-after")
+  const reset = response.headers.get("x-ratelimit-reset")
+  const rateLimited = (response.status === 403 || response.status === 429) && (remaining === "0" || retryAfter !== null)
+  const rateLimit = rateLimited
+    ? `; rate limited${retryAfter ? `; retry after ${retryAfter} seconds` : reset ? `; resets at ${reset}` : ""}`
+    : ""
+  throw new Error(`Request failed for ${url}: HTTP ${response.status}${rateLimit}`)
+}
+
+async function readResponseBody(response: Response): Promise<string> {
+  if (!response.body) return ""
+
+  const chunks: Uint8Array[] = []
+  let size = 0
+  for await (const chunk of response.body) {
+    size += chunk.byteLength
+    if (size > RESPONSE_SIZE_MAX) {
+      throw new Error(`response exceeds maximum ${RESPONSE_SIZE_MAX} bytes`)
+    }
+    chunks.push(chunk)
+  }
+  return Buffer.concat(chunks, size).toString("utf8")
+}
+
+function githubApiUrl(repository: string): string {
+  return `https://api.github.com/repos/${repository}`
+}
+
+function requiredSafeInteger(value: unknown, key: string, label: string): number {
+  if (!value || typeof value !== "object") throw new Error(`${label} returned invalid metadata`)
+  const result = Reflect.get(value, key)
+  if (!Number.isSafeInteger(result) || (result as number) < 0) throw new Error(`${label} returned invalid ${key}`)
+  return result as number
+}
+
+function repositoryLicense(metadata: unknown): string | undefined {
+  const license = nestedString(metadata, ["license", "spdx_id"])
+  return license && license !== "NOASSERTION" ? license : undefined
+}
+
+function normalizeLicense(value: unknown): string | undefined {
+  if (typeof value === "string" && value.length > 0) return value
+  if (value && typeof value === "object") return objectString(value, "type")
+  return undefined
+}
+
+function nestedString(value: unknown, path: string[]): string | undefined {
+  const nested = nestedValue(value, path)
+  return typeof nested === "string" && nested.length > 0 ? nested : undefined
+}
+
+function nestedValue(value: unknown, path: string[]): unknown {
+  let current = value
+  for (const key of path) {
+    if (!current || typeof current !== "object") return undefined
+    current = Reflect.get(current, key)
+  }
+  return current
+}
+
+function objectString(value: unknown, key: string): string | undefined {
+  return nestedString(value, [key])
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}
+
+function redactToken(message: string, token?: string): string {
+  return token ? message.replaceAll(token, "[REDACTED]") : message
+}
+
+async function main() {
+  const arguments_ = process.argv.slice(2)
+  if (arguments_.length !== 1)
+    throw new Error("Usage: bun scripts/update-package-facts.ts <community-entries-directory>")
+  const facts = await updatePackageFacts(arguments_[0], { githubToken: process.env.GITHUB_TOKEN })
+  console.log(`Updated ${resolve(arguments_[0], "facts.json")} with ${facts.packages.length} package records.`)
+}
+
+if (import.meta.main) {
+  main().catch((error) => {
+    console.error(errorMessage(error))
+    process.exit(1)
+  })
+}

packages/web/scripts/validate-packages.ts

@@ -1,8 +1,10 @@
 #!/usr/bin/env bun
 
 import { promises as fs } from "node:fs"
-import { basename, resolve } from "node:path"
-import { githubRepository, packageEntrySchema, type PackageEntry } from "../src/lib/package-schema"
+import { resolve } from "node:path"
+import { githubRepository, githubRepositoryIdentifier, type PackageEntry } from "../src/lib/package-schema"
+import { loadRemotePackageFacts, PACKAGE_COUNT_MAX } from "../src/lib/remote-package-facts"
+import { parsePackageEntryFile } from "./package-entry-file"
 
 type EntryFile = { entry: PackageEntry; path: string }
 
@@ -22,15 +24,19 @@ async function main() {
       ),
     )
   ).flat()
+  if (files.length > PACKAGE_COUNT_MAX) {
+    fail(`Package entry count ${files.length} exceeds maximum ${PACKAGE_COUNT_MAX}`)
+  }
   const entries = (await Promise.all(files.map((path) => parseEntry(path, violations)))).flat()
-  const pathsById = new Map<string, string>()
+  const entriesById = new Map<string, EntryFile>()
 
-  for (const { entry, path } of entries) {
-    const previous = pathsById.get(entry.id)
-    if (previous) violations.push(`duplicate id \`${entry.id}\`: ${previous}, ${path}`)
-    else pathsById.set(entry.id, path)
+  for (const entryFile of entries) {
+    const previous = entriesById.get(entryFile.entry.id)
+    if (previous) violations.push(`duplicate id \`${entryFile.entry.id}\`: ${previous.path}, ${entryFile.path}`)
+    else entriesById.set(entryFile.entry.id, entryFile)
   }
 
+  await validateFacts(targetDirectory, targetDirectory !== localDirectory, entriesById, violations)
   if (network) await validateNetwork(entries, violations)
 
   if (violations.length) {
@@ -42,6 +48,47 @@ async function main() {
   console.log(`Package validation passed for ${entries.length} entries${network ? " with network checks" : ""}.`)
 }
 
+async function validateFacts(
+  directory: string,
+  required: boolean,
+  entriesById: Map<string, EntryFile>,
+  violations: string[],
+): Promise<void> {
+  const factsPath = resolve(directory, "facts.json")
+  try {
+    const facts = await loadRemotePackageFacts(directory, required)
+    const factsById = new Map(facts.map((fact) => [fact.id, fact]))
+    for (const fact of facts) {
+      const entryFile = entriesById.get(fact.id)
+      if (!entryFile) {
+        if (fact.version !== undefined || fact.license !== undefined) {
+          violations.push(`${factsPath}: unknown package id \`${fact.id}\``)
+        }
+      } else if (entryFile.entry.official && (fact.version !== undefined || fact.license !== undefined)) {
+        violations.push(`${factsPath}: first-party fact \`${fact.id}\` may contain only stars`)
+      }
+    }
+    if (required) {
+      for (const { entry } of entriesById.values()) {
+        if (entry.official) continue
+        const fact = factsById.get(entry.id)
+        if (githubRepository(entry.source.url) && fact?.stars === undefined) {
+          violations.push(`${factsPath}: missing stars for \`${entry.id}\``)
+        }
+        const hasVersionSource = entry.distributions.some(
+          (distribution) =>
+            (distribution.type === "npm" || distribution.type === "github-release") && distribution.identifier,
+        )
+        if (!entry.official && hasVersionSource && fact?.version === undefined) {
+          violations.push(`${factsPath}: missing version for \`${entry.id}\``)
+        }
+      }
+    }
+  } catch (error) {
+    violations.push(error instanceof Error ? error.message : String(error))
+  }
+}
+
 async function readDirectory(directory: string, allowMissing: boolean): Promise<string[]> {
   let entries
   try {
@@ -57,23 +104,10 @@ async function readDirectory(directory: string, allowMissing: boolean): Promise<
 }
 
 async function parseEntry(path: string, violations: string[]): Promise<EntryFile[]> {
-  const filename = basename(path, ".mdx")
   try {
-    const contents = await Bun.file(path).text()
-    const match = contents.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)
-    if (!match) throw new Error("missing YAML frontmatter")
-    const result = packageEntrySchema.safeParse(Bun.YAML.parse(match[1]))
-    if (!result.success) {
-      for (const issue of result.error.issues) {
-        const field = issue.path.length ? `${issue.path.join(".")}: ` : ""
-        violations.push(`${path}: ${field}${issue.message}`)
-      }
-      return []
-    }
-    if (result.data.id !== filename) violations.push(`${path}: frontmatter id must match filename \`${filename}\``)
-    return [{ entry: result.data, path }]
+    return [{ entry: await parsePackageEntryFile(path), path }]
   } catch (error) {
-    violations.push(`${path}: ${error instanceof Error ? error.message : String(error)}`)
+    violations.push(error instanceof Error ? error.message : String(error))
     return []
   }
 }
@@ -85,36 +119,51 @@ async function validateNetwork(entries: EntryFile[], violations: string[]) {
     const repositoryUrl = repository
       ? `https://api.github.com/repos/${repository}`
       : entry.source.url.replace(/\.git$/, "")
-    checks.set(`repo:${repositoryUrl}`, { label: `${path}: source repository`, url: repositoryUrl })
+    checks.set(repositoryUrl, { label: `${path}: source repository`, url: repositoryUrl })
 
     for (const distribution of entry.distributions) {
       if (distribution.type === "npm" && distribution.identifier) {
-        checks.set(`npm:${distribution.identifier}`, {
+        const url = `https://registry.npmjs.org/${encodeURIComponent(distribution.identifier)}`
+        checks.set(url, {
           label: `${path}: npm package \`${distribution.identifier}\``,
-          url: `https://registry.npmjs.org/${encodeURIComponent(distribution.identifier)}`,
+          url,
         })
       }
       if (distribution.type === "github-release" && distribution.identifier) {
-        checks.set(`release:${distribution.identifier}`, {
+        const repository = githubRepositoryIdentifier(distribution.identifier)
+        if (!repository) {
+          violations.push(`${path}: invalid GitHub repository identifier \`${distribution.identifier}\``)
+          continue
+        }
+        const url = `https://api.github.com/repos/${repository}`
+        checks.set(url, {
           label: `${path}: GitHub repository \`${distribution.identifier}\``,
-          url: `https://api.github.com/repos/${distribution.identifier.replace(/\.git$/, "")}`,
+          url,
         })
       }
     }
   }
 
-  await Promise.all(
-    [...checks.values()].map(async ({ label, url }) => {
-      try {
-        const response = await fetch(url, {
-          headers: { Accept: "application/vnd.github+json", "User-Agent": "opentui-package-validator" },
-        })
-        if (!response.ok) violations.push(`${label} does not exist (${response.status})`)
-      } catch (error) {
-        violations.push(`${label} check failed: ${error instanceof Error ? error.message : String(error)}`)
+  for (const { label, url } of checks.values()) {
+    try {
+      const github = new URL(url).hostname === "api.github.com"
+      const response = await fetch(url, {
+        headers: {
+          Accept: "application/vnd.github+json",
+          "User-Agent": "opentui-package-validator",
+          ...(github && process.env.GITHUB_TOKEN ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } : {}),
+          ...(github ? { "X-GitHub-Api-Version": "2022-11-28" } : {}),
+        },
+        signal: AbortSignal.timeout(10_000),
+      })
+      if (!response.ok) {
+        const reset = response.headers.get("x-ratelimit-reset")
+        violations.push(`${label} does not exist (${response.status}${reset ? `; rate limit resets at ${reset}` : ""})`)
       }
-    }),
-  )
+    } catch (error) {
+      violations.push(`${label} check failed: ${error instanceof Error ? error.message : String(error)}`)
+    }
+  }
 }
 
 function fail(message: string): never {

packages/web/src/lib/package-schema.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, test } from "bun:test"
+import { githubRepositoryIdentifier, packageEntrySchema } from "./package-schema"
+
+const communityEntry = {
+  id: "community-package",
+  name: "Community Package",
+  summary: "A community package.",
+  kind: "library" as const,
+  official: false,
+  maintainers: ["@maintainer"],
+  source: { url: "https://github.com/example/community-package" },
+  distributions: [{ type: "npm" as const, identifier: "community-package" }],
+}
+
+describe("package categories", () => {
+  test("accepts and trims arbitrary category text", () => {
+    const result = packageEntrySchema.parse({
+      ...communityEntry,
+      categories: ["  terminal UI  ", "C++ bindings", "tools/integrations"],
+    })
+
+    expect(result.categories).toEqual(["terminal UI", "C++ bindings", "tools/integrations"])
+  })
+
+  test("detects duplicates after trimming", () => {
+    const result = packageEntrySchema.safeParse({
+      ...communityEntry,
+      categories: ["testing", " testing "],
+    })
+
+    expect(result.success).toBe(false)
+  })
+
+  test.each([{ categories: [] }, { categories: ["   "] }])("rejects empty categories: %j", ({ categories }) => {
+    expect(packageEntrySchema.safeParse({ ...communityEntry, categories }).success).toBe(false)
+  })
+
+  test("rejects categories longer than 32 characters", () => {
+    expect(packageEntrySchema.safeParse({ ...communityEntry, categories: ["a".repeat(33)] }).success).toBe(false)
+    expect(packageEntrySchema.safeParse({ ...communityEntry, categories: ["😀".repeat(32)] }).success).toBe(true)
+    expect(packageEntrySchema.safeParse({ ...communityEntry, categories: ["😀".repeat(33)] }).success).toBe(false)
+  })
+
+  test("rejects more than three categories", () => {
+    expect(
+      packageEntrySchema.safeParse({ ...communityEntry, categories: ["one", "two", "three", "four"] }).success,
+    ).toBe(false)
+  })
+})
+
+test("normalizes GitHub repository identifiers", () => {
+  expect(githubRepositoryIdentifier("owner/repository")).toBe("owner/repository")
+  expect(githubRepositoryIdentifier("https://github.com/owner/repository.git")).toBe("owner/repository")
+  expect(githubRepositoryIdentifier("not-a-repository")).toBeUndefined()
+})
+
+test("rejects package IDs longer than 128 characters", () => {
+  expect(packageEntrySchema.safeParse({ ...communityEntry, id: "a".repeat(129) }).success).toBe(false)
+})

packages/web/src/lib/package-schema.ts

@@ -1,18 +1,9 @@
 import { z } from "astro/zod"
 
-export const PACKAGE_CATEGORIES = [
-  "components",
-  "developer-tools",
-  "documentation",
-  "frameworks",
-  "input",
-  "integrations",
-  "rendering",
-  "testing",
-  "utilities",
-] as const
-
-const packageId = z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/, "must be a lowercase, hyphenated package ID")
+export const packageId = z
+  .string()
+  .max(128)
+  .regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/, "must be a lowercase, hyphenated package ID")
 const nonemptyString = z.string().trim().min(1)
 const absoluteUrl = z.url().refine((value) => ["http:", "https:"].includes(new URL(value).protocol), {
   message: "must be an HTTP(S) URL",
@@ -33,6 +24,12 @@ const gitUrl = z.string().refine((value) => {
 }, "must be a git URL")
 const unique = <T extends z.ZodType>(schema: T) =>
   z.array(schema).refine((values) => new Set(values).size === values.length, "must not contain duplicates")
+const packageCategory = z
+  .string()
+  .trim()
+  .min(1)
+  .max(64)
+  .refine((value) => [...value].length <= 32, "must contain at most 32 characters")
 
 export const packageEntrySchema = z
   .strictObject({
@@ -53,6 +50,7 @@ export const packageEntrySchema = z
     official: z.boolean(),
     maintainers: unique(nonemptyString.regex(/^@?[A-Za-z0-9](?:[A-Za-z0-9-]{0,37})$/, "must be a GitHub handle"))
       .min(1)
+      .max(16)
       .optional(),
     source: z.strictObject({
       url: gitUrl,
@@ -69,6 +67,7 @@ export const packageEntrySchema = z
           docs: sitePathOrUrl.optional(),
         }),
       )
+      .max(32)
       .optional(),
     distributions: z
       .array(
@@ -78,7 +77,8 @@ export const packageEntrySchema = z
           install: nonemptyString.optional(),
         }),
       )
-      .min(1),
+      .min(1)
+      .max(8),
     links: z
       .strictObject({
         homepage: absoluteUrl.optional(),
@@ -87,7 +87,7 @@ export const packageEntrySchema = z
         changelog: sitePathOrUrl.optional(),
       })
       .optional(),
-    categories: unique(z.enum(PACKAGE_CATEGORIES)).optional(),
+    categories: unique(packageCategory).min(1).max(3).optional(),
     status: z.enum(["active", "archived", "deprecated"]).default("active"),
   })
   .superRefine((entry, context) => {
@@ -130,3 +130,8 @@ export function githubRepository(source: string): string | undefined {
     return undefined
   }
 }
+
+export function githubRepositoryIdentifier(value: string): string | undefined {
+  const repository = githubRepository(value) ?? value.replace(/\.git$/, "")
+  return /^[^/\s]+\/[^/\s]+$/.test(repository) ? repository : undefined
+}

packages/web/src/lib/remote-package-facts.test.ts

@@ -0,0 +1,178 @@
+import { afterEach, describe, expect, test } from "bun:test"
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"
+import { join } from "node:path"
+import { tmpdir } from "node:os"
+import { updatePackageFacts } from "../../scripts/update-package-facts"
+import { remotePackageFactsSchema } from "./remote-package-facts"
+
+const temporaryDirectories: string[] = []
+
+afterEach(async () => {
+  await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })))
+})
+
+describe("remote package facts schema", () => {
+  test("rejects malformed facts and bounds", () => {
+    expect(remotePackageFactsSchema.safeParse({ schemaVersion: 1, packages: [{ id: "empty" }] }).success).toBe(false)
+    expect(
+      remotePackageFactsSchema.safeParse({ schemaVersion: 1, packages: [{ id: "bad", version: "x".repeat(129) }] })
+        .success,
+    ).toBe(false)
+    expect(remotePackageFactsSchema.safeParse({ schemaVersion: 1, packages: [{ id: "bad", stars: -1 }] }).success).toBe(
+      false,
+    )
+    expect(
+      remotePackageFactsSchema.safeParse({
+        schemaVersion: 1,
+        packages: [
+          { id: "duplicate", stars: 1 },
+          { id: "duplicate", stars: 2 },
+        ],
+      }).success,
+    ).toBe(false)
+    expect(
+      remotePackageFactsSchema.safeParse({
+        schemaVersion: 1,
+        packages: Array.from({ length: 257 }, (_, index) => ({ id: `package-${index}`, stars: 0 })),
+      }).success,
+    ).toBe(false)
+  })
+})
+
+describe("package facts updater", () => {
+  test("deduplicates requests, authenticates GitHub, and applies source precedence", async () => {
+    const root = await temporaryRoot()
+    const firstParty = join(root, "official")
+    const community = join(root, "community")
+    await Promise.all([mkdir(firstParty), mkdir(community)])
+    await writeEntry(firstParty, "official-one", true, [{ type: "npm", identifier: "ignored" }])
+    await writeEntry(firstParty, "official-two", true, [{ type: "source" }])
+    await writeEntry(community, "npm-package", false, [
+      { type: "github-release", identifier: "example/shared" },
+      { type: "npm", identifier: "npm-package" },
+      { type: "npm", identifier: "beta-package" },
+    ])
+    await writeEntry(community, "release-package", false, [{ type: "github-release", identifier: "example/shared" }])
+    await writeEntry(
+      community,
+      "non-github-package",
+      false,
+      [{ type: "npm", identifier: "non-github-package" }],
+      "https://gitlab.com/example/non-github-package",
+    )
+
+    const calls: Array<{ url: string; init?: RequestInit }> = []
+    const responses: Record<string, unknown> = {
+      "https://api.github.com/repos/anomalyco/opentui": {
+        stargazers_count: 100,
+        license: { spdx_id: "MIT" },
+      },
+      "https://api.github.com/repos/example/shared": { stargazers_count: 7, license: { spdx_id: "Apache-2.0" } },
+      "https://api.github.com/repos/example/shared/releases/latest": { tag_name: "v2.0.0" },
+      "https://gitlab.com/example/non-github-package": {},
+      "https://registry.npmjs.org/beta-package": {},
+      "https://registry.npmjs.org/ignored": {},
+      "https://registry.npmjs.org/non-github-package/latest": { version: "3.0.0", license: "BSD-2-Clause" },
+      "https://registry.npmjs.org/npm-package/latest": { version: "1.4.0", license: "ISC" },
+    }
+    const fetchMock = (async (input: string | URL | Request, init?: RequestInit) => {
+      const url = String(input)
+      calls.push({ url, init })
+      const body = responses[url]
+      if (!body) return new Response("missing", { status: 404 })
+      return Response.json(body)
+    }) as typeof fetch
+
+    const facts = await updatePackageFacts(community, {
+      firstPartyDirectory: firstParty,
+      fetch: fetchMock,
+      githubToken: "secret-token",
+    })
+
+    expect(calls.filter(({ url }) => url === "https://api.github.com/repos/anomalyco/opentui")).toHaveLength(1)
+    expect(calls.filter(({ url }) => url.endsWith("/releases/latest"))).toHaveLength(1)
+    expect(calls.filter(({ url }) => url === "https://registry.npmjs.org/ignored")).toHaveLength(1)
+    expect(calls.find(({ url }) => url === "https://registry.npmjs.org/ignored")?.init?.method).toBe("HEAD")
+    expect(calls.filter(({ url }) => url === "https://registry.npmjs.org/beta-package/latest")).toHaveLength(0)
+    expect(calls.filter(({ url }) => url === "https://gitlab.com/example/non-github-package")).toHaveLength(1)
+    const githubHeaders = new Headers(calls.find(({ url }) => url.includes("api.github.com"))?.init?.headers)
+    expect(githubHeaders.get("Authorization")).toBe("Bearer secret-token")
+    expect(githubHeaders.get("Accept")).toBe("application/vnd.github+json")
+    expect(githubHeaders.get("X-GitHub-Api-Version")).toBe("2022-11-28")
+    expect(githubHeaders.get("User-Agent")).toBe("opentui-package-facts")
+    expect(facts.packages).toEqual([
+      { id: "non-github-package", version: "3.0.0", license: "BSD-2-Clause" },
+      { id: "npm-package", version: "1.4.0", license: "ISC", stars: 7 },
+      { id: "official-one", stars: 100 },
+      { id: "official-two", stars: 100 },
+      { id: "release-package", version: "v2.0.0", license: "Apache-2.0", stars: 7 },
+    ])
+    expect(JSON.parse(await readFile(join(community, "facts.json"), "utf8"))).toEqual(facts)
+    await expect(
+      updatePackageFacts(community, {
+        firstPartyDirectory: firstParty,
+        fetch: fetchMock,
+        githubToken: "invalid token",
+      }),
+    ).rejects.toThrow("GITHUB_TOKEN must be a nonempty token without whitespace")
+
+    await writeEntry(community, "ssh-package", false, [{ type: "source" }], "ssh://git@gitlab.com/example/package.git")
+    await expect(updatePackageFacts(community, { firstPartyDirectory: firstParty, fetch: fetchMock })).rejects.toThrow(
+      "Cannot verify non-HTTP source URL",
+    )
+  })
+
+  test("stops before exceeding the GitHub request budget", async () => {
+    const root = await temporaryRoot()
+    const firstParty = join(root, "official")
+    const community = join(root, "community")
+    await Promise.all([mkdir(firstParty), mkdir(community)])
+    await Promise.all(
+      Array.from({ length: 63 }, (_, packageIndex) =>
+        writeEntry(
+          community,
+          `package-${packageIndex}`,
+          false,
+          Array.from({ length: 8 }, (_, repositoryIndex) => ({
+            type: "github-release",
+            identifier: `owner-${packageIndex}/repository-${repositoryIndex}`,
+          })),
+        ),
+      ),
+    )
+
+    let requests = 0
+    const fetchMock = (async () => {
+      requests++
+      return Response.json({ stargazers_count: 1, tag_name: "v1.0.0", license: { spdx_id: "MIT" } })
+    }) as typeof fetch
+
+    await expect(updatePackageFacts(community, { firstPartyDirectory: firstParty, fetch: fetchMock })).rejects.toThrow(
+      "GitHub request count exceeds maximum 500",
+    )
+    expect(requests).toBe(500)
+  })
+})
+
+async function temporaryRoot(): Promise<string> {
+  const directory = await mkdtemp(join(tmpdir(), "opentui-package-facts-"))
+  temporaryDirectories.push(directory)
+  return directory
+}
+
+async function writeEntry(
+  directory: string,
+  id: string,
+  official: boolean,
+  distributions: Array<{ type: string; identifier?: string }>,
+  source = official ? "https://github.com/anomalyco/opentui" : "https://github.com/example/shared",
+) {
+  const maintainers = official ? "" : "maintainers:\n  - example\n"
+  const distributionYaml = distributions
+    .map(({ type, identifier }) => `  - type: ${type}${identifier ? `\n    identifier: ${identifier}` : ""}`)
+    .join("\n")
+  await writeFile(
+    join(directory, `${id}.mdx`),
+    `---\nid: ${id}\nname: ${id}\nsummary: Test package.\nkind: library\nofficial: ${official}\n${maintainers}source:\n  url: ${source}\ndistributions:\n${distributionYaml}\n---\n`,
+  )
+}

packages/web/src/lib/remote-package-facts.ts

@@ -0,0 +1,69 @@
+import { readFile, stat } from "node:fs/promises"
+import { resolve } from "node:path"
+import { z } from "astro/zod"
+import { packageId } from "./package-schema"
+
+const FACTS_FILE_SIZE_MAX = 1024 * 1024
+export const PACKAGE_COUNT_MAX = 256
+const boundedString = z.string().min(1).max(128)
+
+export const remotePackageFactSchema = z
+  .strictObject({
+    id: packageId,
+    version: boundedString.optional(),
+    license: boundedString.optional(),
+    stars: z.number().int().safe().nonnegative().optional(),
+  })
+  .refine((fact) => fact.version !== undefined || fact.license !== undefined || fact.stars !== undefined, {
+    message: "must contain at least one fact",
+  })
+
+export const remotePackageFactsSchema = z.strictObject({
+  schemaVersion: z.literal(1),
+  packages: z
+    .array(remotePackageFactSchema)
+    .max(PACKAGE_COUNT_MAX)
+    .refine((packages) => new Set(packages.map((fact) => fact.id)).size === packages.length, {
+      message: "package IDs must not contain duplicates",
+    }),
+})
+
+export type RemotePackageFact = z.infer<typeof remotePackageFactSchema>
+export type RemotePackageFacts = z.infer<typeof remotePackageFactsSchema>
+
+export async function loadRemotePackageFacts(directory?: string, required = false): Promise<RemotePackageFact[]> {
+  const indexDirectory = directory ?? process.env.OPENTUI_INDEX_DIR ?? import.meta.env.OPENTUI_INDEX_DIR
+  if (!indexDirectory) return []
+
+  const path = resolve(indexDirectory, "facts.json")
+  let fileSize: number
+  try {
+    fileSize = (await stat(path)).size
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      if (!required) return []
+      throw new Error(`${path} is required`)
+    }
+    throw error
+  }
+  if (fileSize > FACTS_FILE_SIZE_MAX) {
+    throw new Error(`${path} is ${fileSize} bytes; maximum is ${FACTS_FILE_SIZE_MAX}`)
+  }
+
+  let input: unknown
+  try {
+    input = JSON.parse(await readFile(path, "utf8"))
+  } catch (error) {
+    throw new Error(`${path}: ${error instanceof Error ? error.message : String(error)}`)
+  }
+
+  const result = remotePackageFactsSchema.safeParse(input)
+  if (!result.success) {
+    const issues = result.error.issues.map((issue) => {
+      const field = issue.path.length ? `${issue.path.join(".")}: ` : ""
+      return `${field}${issue.message}`
+    })
+    throw new Error(`${path}: ${issues.join("; ")}`)
+  }
+  return result.data.packages
+}

packages/web/src/pages/packages/[id].astro

@@ -5,11 +5,17 @@ import ProseH2 from "../../components/ProseH2.astro"
 import ProseH3 from "../../components/ProseH3.astro"
 import ProseTable from "../../components/ProseTable.astro"
 import SitePage from "../../layouts/SitePage.astro"
-import { loadFirstPartyPackageFacts, type PlatformFact } from "../../lib/package-facts"
+import { loadFirstPartyPackageFacts, type PackageFacts, type PlatformFact } from "../../lib/package-facts"
+import { loadRemotePackageFacts, type RemotePackageFact } from "../../lib/remote-package-facts"
+
+type PagePackageFacts = RemotePackageFact & Partial<Omit<PackageFacts, "id">>
 
 export async function getStaticPaths() {
   const entries = await getCollection("packages")
-  const facts = new Map((await loadFirstPartyPackageFacts()).map((fact) => [fact.id, fact]))
+  const facts = new Map<string, PagePackageFacts>((await loadRemotePackageFacts()).map((fact) => [fact.id, fact]))
+  for (const fact of await loadFirstPartyPackageFacts()) {
+    facts.set(fact.id, { ...facts.get(fact.id), ...fact })
+  }
 
   return entries.map((entry) => ({
     params: { id: entry.id },
@@ -19,7 +25,7 @@ export async function getStaticPaths() {
 
 interface Props {
   entry: CollectionEntry<"packages">
-  fact?: Awaited<ReturnType<typeof loadFirstPartyPackageFacts>>[number]
+  fact?: PagePackageFacts
 }
 
 const { entry, fact } = Astro.props
@@ -65,14 +71,9 @@ function groupPlatforms(platforms: PlatformFact[]) {
     <p class="facts">
       <span>{entry.id}</span>
       <span>[{entry.data.official ? "official" : "community"}]</span>
-      {
-        entry.data.official && (
-          <>
-            {fact?.license && <span>{fact.license}</span>}
-            {fact?.version && <span>{fact.version}</span>}
-          </>
-        )
-      }
+      {fact?.license && <span>{fact.license}</span>}
+      {fact?.version && <span>{fact.version}</span>}
+      {fact?.stars !== undefined && <span>{fact.stars.toLocaleString("en-US")} GitHub stars</span>}
       {entry.data.status !== "active" && <span>{entry.data.status}</span>}
     </p>
     <h1>{entry.data.name}</h1>