Add Markdown for Agents content negotiation

When clients send Accept: text/markdown, convert HTML responses to
markdown with Content-Type text/markdown and x-markdown-tokens while
leaving normal browser HTML responses unchanged.

Co-authored-by: Julien Thibeaut <ibelick@users.noreply.github.com>

package-lock.json

@@ -16,6 +16,7 @@
         "@fontsource/jetbrains-mono": "^5.2.8",
         "@tailwindcss/vite": "^4.3.3",
         "astro": "^5.16.7",
+        "markdown-for-agents": "^1.3.4",
         "marked": "^17.0.1",
         "motion": "^12.24.12",
         "react": "^19.2.3",
@@ -4289,6 +4290,37 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/htmlparser2": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
+      "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
+      "funding": [
+        "https://github.com/fb55/htmlparser2?sponsor=1",
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fb55"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "domelementtype": "^2.3.0",
+        "domhandler": "^5.0.3",
+        "domutils": "^3.2.2",
+        "entities": "^7.0.1"
+      }
+    },
+    "node_modules/htmlparser2/node_modules/entities": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
+      "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=0.12"
+      },
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
     "node_modules/http-cache-semantics": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz",
@@ -4755,6 +4787,19 @@
         "source-map-js": "^1.2.1"
       }
     },
+    "node_modules/markdown-for-agents": {
+      "version": "1.3.4",
+      "resolved": "https://registry.npmjs.org/markdown-for-agents/-/markdown-for-agents-1.3.4.tgz",
+      "integrity": "sha512-kyXaAd/A+6SrZdaEbzeKL1wyJck5lckIXeTa1w04SCqxpzV0vxAQDT7FlJfXsYdm426L/TB39p7HKMpM4uFVsA==",
+      "license": "MIT",
+      "dependencies": {
+        "domhandler": "^5.0.3",
+        "htmlparser2": "^10.1.0"
+      },
+      "engines": {
+        "node": ">=22"
+      }
+    },
     "node_modules/markdown-table": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz",

package.json

@@ -26,6 +26,7 @@
     "@fontsource/jetbrains-mono": "^5.2.8",
     "@tailwindcss/vite": "^4.3.3",
     "astro": "^5.16.7",
+    "markdown-for-agents": "^1.3.4",
     "marked": "^17.0.1",
     "motion": "^12.24.12",
     "react": "^19.2.3",
@@ -36,10 +37,10 @@
   "devDependencies": {
     "@astrojs/check": "^0.9.9",
     "@tailwindcss/typography": "^0.5.19",
-    "@types/react": "^19.2.7",
-    "@types/react-dom": "^19.2.3",
     "@types/marked": "^5.0.2",
     "@types/node": "^25.9.3",
+    "@types/react": "^19.2.7",
+    "@types/react-dom": "^19.2.3",
     "prettier": "^3.3.3",
     "prettier-plugin-astro": "^0.14.1",
     "prettier-plugin-tailwindcss": "^0.6.8",

src/lib/markdown-negotiation.ts

@@ -0,0 +1,130 @@
+import { convert, estimateTokens } from "markdown-for-agents";
+
+const HTML_CONTENT_TYPE = /^text\/html\b/i;
+const MARKDOWN_CONTENT_TYPE = "text/markdown; charset=utf-8";
+
+type MediaRange = {
+  type: string;
+  quality: number;
+  index: number;
+};
+
+function parseAccept(accept: string): MediaRange[] {
+  return accept
+    .split(",")
+    .map((part, index) => {
+      const [rawType, ...params] = part.trim().split(";").map((s) => s.trim());
+      const type = (rawType ?? "").toLowerCase();
+      if (!type) return null;
+
+      let quality = 1;
+      for (const param of params) {
+        const match = /^q\s*=\s*([0-9]*\.?[0-9]+)$/i.exec(param);
+        if (match) {
+          quality = Number(match[1]);
+          break;
+        }
+      }
+
+      return { type, quality: Number.isFinite(quality) ? quality : 0, index };
+    })
+    .filter((range): range is MediaRange => range !== null);
+}
+
+/** True when Accept explicitly prefers text/markdown over text/html. */
+export function prefersMarkdown(acceptHeader: string | null): boolean {
+  if (!acceptHeader) return false;
+
+  const ranges = parseAccept(acceptHeader);
+  // Browsers often send */*; only an explicit text/markdown means an agent ask.
+  const markdown = ranges.find((range) => range.type === "text/markdown");
+  if (!markdown || markdown.quality <= 0) return false;
+
+  const html = ranges.find((range) => range.type === "text/html");
+  if (!html) return true;
+
+  if (markdown.quality !== html.quality) {
+    return markdown.quality > html.quality;
+  }
+
+  // Equal quality: earlier Accept entry wins (Cloudflare-style negotiation).
+  return markdown.index <= html.index;
+}
+
+export function isHtmlResponse(response: Response): boolean {
+  const contentType = response.headers.get("content-type");
+  return contentType !== null && HTML_CONTENT_TYPE.test(contentType);
+}
+
+export function estimateTokenCount(text: string): number {
+  return estimateTokens(text).tokens;
+}
+
+export function htmlToAgentMarkdown(html: string): {
+  markdown: string;
+  markdownTokens: number;
+  originalTokens: number;
+} {
+  const originalTokens = estimateTokenCount(html);
+  const { markdown, tokenEstimate } = convert(html, { extract: true });
+  return {
+    markdown,
+    markdownTokens: tokenEstimate.tokens,
+    originalTokens,
+  };
+}
+
+function withVaryAccept(headers: Headers): void {
+  const existing = headers.get("Vary");
+  if (!existing) {
+    headers.set("Vary", "Accept");
+    return;
+  }
+  const parts = existing.split(",").map((part) => part.trim().toLowerCase());
+  if (!parts.includes("accept")) {
+    headers.set("Vary", `${existing}, Accept`);
+  }
+}
+
+/** Convert an HTML response to Markdown for Agents when negotiated. */
+export async function maybeNegotiateMarkdown(
+  request: Request,
+  response: Response,
+): Promise<Response> {
+  if (request.method !== "GET" && request.method !== "HEAD") {
+    return response;
+  }
+  if (!prefersMarkdown(request.headers.get("accept"))) {
+    return response;
+  }
+  if (!isHtmlResponse(response)) {
+    return response;
+  }
+
+  const html = await response.text();
+  const { markdown, markdownTokens, originalTokens } =
+    htmlToAgentMarkdown(html);
+
+  const headers = new Headers(response.headers);
+  headers.set("Content-Type", MARKDOWN_CONTENT_TYPE);
+  headers.set("x-markdown-tokens", String(markdownTokens));
+  headers.set("x-original-tokens", String(originalTokens));
+  withVaryAccept(headers);
+
+  // Body descriptors from the HTML response no longer apply.
+  for (const header of [
+    "content-length",
+    "content-encoding",
+    "content-range",
+    "etag",
+    "last-modified",
+  ]) {
+    headers.delete(header);
+  }
+
+  return new Response(request.method === "HEAD" ? null : markdown, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+}

src/middleware.ts

@@ -1,6 +1,7 @@
 import { defineMiddleware } from "astro:middleware";
 
 import { buildDiscoveryLinkHeader, getSiteOrigin } from "./lib/agent-discovery";
+import { maybeNegotiateMarkdown } from "./lib/markdown-negotiation";
 
 const securityHeaders = {
   "Content-Security-Policy":
@@ -39,6 +40,6 @@ export const onRequest = defineMiddleware(async (context, next) => {
     return applyResponseHeaders(response, origin);
   }
 
-  const response = await next();
-  return applyResponseHeaders(response, origin);
+  const response = applyResponseHeaders(await next(), origin);
+  return maybeNegotiateMarkdown(context.request, response);
 });

tests/markdown-negotiation.test.ts

@@ -0,0 +1,124 @@
+import { describe, test } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  estimateTokenCount,
+  htmlToAgentMarkdown,
+  isHtmlResponse,
+  maybeNegotiateMarkdown,
+  prefersMarkdown,
+} from "../src/lib/markdown-negotiation.ts";
+
+describe("markdown negotiation", () => {
+  test("prefers markdown when Accept asks for it", () => {
+    assert.equal(prefersMarkdown("text/markdown"), true);
+    assert.equal(prefersMarkdown("text/markdown, text/html"), true);
+    assert.equal(prefersMarkdown("text/html;q=0.8, text/markdown"), true);
+    assert.equal(prefersMarkdown("text/html, text/markdown;q=0.9"), false);
+    assert.equal(prefersMarkdown("text/html, text/markdown"), false);
+    assert.equal(prefersMarkdown("text/html"), false);
+    assert.equal(prefersMarkdown("*/*"), false);
+    assert.equal(prefersMarkdown(null), false);
+  });
+
+  test("detects HTML responses", () => {
+    assert.equal(
+      isHtmlResponse(
+        new Response("<html></html>", {
+          headers: { "content-type": "text/html; charset=utf-8" },
+        }),
+      ),
+      true,
+    );
+    assert.equal(
+      isHtmlResponse(
+        new Response("{}", {
+          headers: { "content-type": "application/json" },
+        }),
+      ),
+      false,
+    );
+  });
+
+  test("converts HTML pages into markdown with token counts", () => {
+    const html = `<!doctype html>
+<html>
+  <head>
+    <title>UI Skills</title>
+    <meta name="description" content="Skills for Design Engineers." />
+  </head>
+  <body>
+    <nav>Navigation</nav>
+    <main>
+      <h1>Hello</h1>
+      <p>World</p>
+    </main>
+    <footer>Footer</footer>
+  </body>
+</html>`;
+
+    const result = htmlToAgentMarkdown(html);
+    assert.match(result.markdown, /^---\n/);
+    assert.match(result.markdown, /title: UI Skills/);
+    assert.match(result.markdown, /# Hello/);
+    assert.match(result.markdown, /World/);
+    assert.doesNotMatch(result.markdown, /Navigation|Footer/);
+    assert.ok(result.markdownTokens > 0);
+    assert.ok(result.originalTokens >= result.markdownTokens);
+    assert.equal(estimateTokenCount(result.markdown), result.markdownTokens);
+  });
+
+  test("returns markdown when Accept prefers it", async () => {
+    const request = new Request("https://www.ui-skills.com/", {
+      headers: { Accept: "text/markdown" },
+    });
+    const htmlResponse = new Response(
+      `<!doctype html><html><head><title>Home</title></head><body><h1>UI Skills</h1></body></html>`,
+      {
+        headers: {
+          "content-type": "text/html; charset=utf-8",
+          etag: '"abc"',
+        },
+      },
+    );
+
+    const response = await maybeNegotiateMarkdown(request, htmlResponse);
+
+    assert.equal(response.headers.get("content-type"), "text/markdown; charset=utf-8");
+    assert.ok(Number(response.headers.get("x-markdown-tokens")) > 0);
+    assert.ok(Number(response.headers.get("x-original-tokens")) > 0);
+    assert.match(response.headers.get("vary") ?? "", /Accept/i);
+    assert.equal(response.headers.get("etag"), null);
+    assert.match(await response.text(), /UI Skills/);
+  });
+
+  test("leaves HTML alone for normal browser requests", async () => {
+    const request = new Request("https://www.ui-skills.com/", {
+      headers: { Accept: "text/html" },
+    });
+    const html = `<!doctype html><html><body><h1>UI Skills</h1></body></html>`;
+    const htmlResponse = new Response(html, {
+      headers: { "content-type": "text/html; charset=utf-8" },
+    });
+
+    const response = await maybeNegotiateMarkdown(request, htmlResponse);
+    assert.equal(
+      response.headers.get("content-type"),
+      "text/html; charset=utf-8",
+    );
+    assert.equal(await response.text(), html);
+  });
+
+  test("does not convert non-HTML responses", async () => {
+    const request = new Request("https://www.ui-skills.com/skills/registry.json", {
+      headers: { Accept: "text/markdown" },
+    });
+    const json = new Response('{"ok":true}', {
+      headers: { "content-type": "application/json" },
+    });
+
+    const response = await maybeNegotiateMarkdown(request, json);
+    assert.equal(response.headers.get("content-type"), "application/json");
+    assert.equal(await response.text(), '{"ok":true}');
+  });
+});

tests/smoke.mjs

@@ -17,8 +17,9 @@ server.stderr.on("data", (chunk) => {
   output += chunk.toString();
 });
 
-const fetchLocal = (path, timeoutMs = 5000) =>
+const fetchLocal = (path, init = {}, timeoutMs = 5000) =>
   fetch(`http://127.0.0.1:${port}${path}`, {
+    ...init,
     signal: AbortSignal.timeout(timeoutMs),
   });
 
@@ -72,6 +73,24 @@ try {
     throw new Error(`API catalog returned unexpected type: ${apiCatalogType}`);
   }
 
+  const markdownHomepage = await fetchLocal("/", {
+    headers: { Accept: "text/markdown" },
+  });
+  if (markdownHomepage.status !== 200) {
+    throw new Error(`Markdown homepage returned ${markdownHomepage.status}`);
+  }
+  const markdownType = markdownHomepage.headers.get("content-type") ?? "";
+  if (!markdownType.startsWith("text/markdown")) {
+    throw new Error(`Markdown homepage content-type was ${markdownType}`);
+  }
+  if (markdownHomepage.headers.get("x-markdown-tokens") === null) {
+    throw new Error("Markdown homepage is missing x-markdown-tokens");
+  }
+  const markdownBody = await markdownHomepage.text();
+  if (!markdownBody.trim()) {
+    throw new Error("Markdown homepage body was empty");
+  }
+
   const registry = await fetchLocal("/skills/registry.json");
   if (registry.status !== 200) {
     throw new Error(`Registry returned ${registry.status}`);