Site audits slow down and retry when a site rate limits them, instead of calling every page blocked (#579)

2026-09-14 19:59:50 +08:00 · 2026-09-08 14:39:16 -04:00
parent 6274e32b67
commit abcf4144f1
22 changed files with 856 additions and 87 deletions
@@ -14,6 +14,7 @@
 *   pnpm --dir badseo run audit http://host:port
 */
 import { crawlPage } from "../../src/server/workflows/site-audit-workflow-helpers";
+import { createCrawlThrottle } from "../../src/server/lib/audit/crawl-throttle";
 import {
  discoverUrls,
  parseRobotsTxt,
@@ -95,6 +96,8 @@ async function crawl(origin: string): Promise<{

  const pages: CrawledPageResult[] = [];
  const links: CrawlLink[] = [];
+  // One per crawl, as the production crawl chunk does.
+  const throttle = createCrawlThrottle(Date.now() + 90_000);

  const enqueue = (url: string, depth: number | null) => {
    const n = normalizeUrl(url);
@@ -108,7 +111,8 @@ async function crawl(origin: string): Promise<{

  while (
    (linkQueue.length > 0 || sitemapQueue.length > 0) &&
-    pages.length < MAX_PAGES
+    pages.length < MAX_PAGES &&
+    !throttle.stopped
  ) {
    const batch: CrawlEntry[] = [];
    while (
@@ -124,11 +128,14 @@ async function crawl(origin: string): Promise<{
    }

    const crawled = await Promise.all(
-      batch.map((e) => crawlPage(e.url, e.depth, sitemapSet.has(e.url))),
+      batch.map((e) =>
+        crawlPage(e.url, e.depth, sitemapSet.has(e.url), throttle),
+      ),
    );

    for (let i = 0; i < crawled.length; i++) {
      const page = crawled[i];
+      if (!page) continue;
      const depth = batch[i].depth;
      pages.push(page);

@@ -146,7 +153,8 @@ async function crawl(origin: string): Promise<{
    }
  }

-  const completed = linkQueue.length === 0 && sitemapQueue.length === 0;
+  const completed =
+    !throttle.stopped && linkQueue.length === 0 && sitemapQueue.length === 0;
  return { pages, links, completed };
 }

@@ -4,6 +4,17 @@ import { article } from "./helpers";

 const CAT = "HTTP status & links";

+/** 429s answered before a request is let through; the served request resets it. */
+const RATE_LIMIT_REFUSALS = 2;
+let refusals = 0;
+
+function stillRateLimited(): boolean {
+  refusals += 1;
+  if (refusals <= RATE_LIMIT_REFUSALS) return true;
+  refusals = 0;
+  return false;
+}
+
 // 18 — a URL that returns 404 (discovered via sitemap) --------------------
 const notFound: Fixture = {
  path: "/status/not-found",
@@ -85,7 +96,7 @@ const blocked: Fixture = {
  name: "Crawler blocked (403)",
  summary: 'Returns 403 Forbidden. The honest "we could not read this" case.',
  lesson:
-    "A 403, a 429, or a bot challenge means the crawler was blocked. A good audit says so, instead of reporting the page as broken. Real search bots may hit the same wall.",
+    "A 403 or a bot challenge means the crawler was blocked. A good audit says so, instead of reporting the page as broken. Real search bots may hit the same wall.",
  expectedIssues: ["blocked-page"],
  linkedFromCatalog: false,
  inSitemap: true,
@@ -101,7 +112,7 @@ const blocked: Fixture = {
          sections: [
            {
              h2: "Blocked is not the same as broken",
-              body: "When a page answers a crawler with 403, 429, or a challenge screen, the honest conclusion is not that the page is broken. It is that the crawler was not allowed to see it. A good audit says exactly that. Reporting a blocked page as a content problem would send you looking for a bug that is not there, when the real issue is access.",
+              body: "When a page answers a crawler with 403 or a challenge screen, the honest conclusion is not that the page is broken. It is that the crawler was not allowed to see it. A good audit says exactly that. Reporting a blocked page as a content problem would send you looking for a bug that is not there, when the real issue is access.",
            },
            {
              h2: "When your own protection backfires",
@@ -140,9 +151,93 @@ const brokenInternalLink: Fixture = {
    ),
 };

+// 20b — rate limited for the first requests, then served (429 → 200) --------
+const rateLimitedThenOk: Fixture = {
+  path: "/status/rate-limited",
+  category: CAT,
+  name: "Rate limited, then served (429)",
+  summary: `Answers 429 with Retry-After for the first ${RATE_LIMIT_REFUSALS} requests, then serves the page.`,
+  lesson:
+    "A 429 means the crawler is going too fast, not that it is unwelcome. A crawler that slows down and comes back gets the page; one that gives up records a page that was never actually broken.",
+  // Nothing to report: the audit backs off, retries, and reads the page.
+  expectedIssues: [],
+  handler: () => {
+    const html = renderPage({
+      fixture: rateLimitedThenOk,
+      title: "Rate limited, then served",
+      metaDescription:
+        "This URL refuses the first couple of requests with a 429 and a Retry-After header, then serves the page normally.",
+      bodyHtml: article({
+        h1: "429 first, then the actual page",
+        lede: "The first requests to this URL are refused with 429 Too Many Requests. Wait a moment and it answers normally.",
+        sections: [
+          {
+            h2: "429 is a speed limit, not a wall",
+            body: "Too Many Requests is the server asking a client to slow down. It is not an access denial and it is not a broken page. Plenty of sites put a rate limit in front of everything, so a crawler that fires twenty parallel requests trips it immediately even though every one of those pages is perfectly healthy and public.",
+          },
+          {
+            h2: "What a well-behaved crawler does",
+            body: "It waits. If the response carries a Retry-After header it honours it, otherwise it backs off for a growing delay, and it slows the rest of the crawl down at the same time rather than retrying one URL while hammering the next. Then it asks again. Search engines behave this way too, which is why a site that answers 429 constantly gets crawled less often and sees new pages indexed more slowly.",
+          },
+          {
+            h2: "Why reporting it as blocked is wrong",
+            body: "Recording a rate-limited URL as blocked tells the owner to go change their bot protection, which is not the problem. The page is public, the crawler was simply going too fast for it. The honest report is either the page itself, after a retry, or a note that the limit held even after the crawler slowed down.",
+          },
+        ],
+      }),
+    });
+    return stillRateLimited()
+      ? htmlResponse(html, { status: 429, headers: { "retry-after": "1" } })
+      : htmlResponse(html);
+  },
+};
+
+// 20c — rate limited on every request, no Retry-After ----------------------
+const rateLimitedAlways: Fixture = {
+  path: "/status/rate-limited-always",
+  category: CAT,
+  name: "Rate limited on every request (429)",
+  summary: "Answers 429 to every request, with no Retry-After header.",
+  lesson:
+    "If a URL still returns 429 after the crawler has slowed down and retried, the page genuinely cannot be audited — but it is rate limited, not blocked, and the fix is the rate limit rather than the bot rules.",
+  expectedIssues: ["rate-limited-page"],
+  linkedFromCatalog: false,
+  inSitemap: true,
+  handler: () =>
+    htmlResponse(
+      renderPage({
+        fixture: rateLimitedAlways,
+        title: "429 on every single request",
+        metaDescription:
+          "This URL answers 429 Too Many Requests to every request, no matter how long the crawler waits between them.",
+        bodyHtml: article({
+          h1: "429, every time",
+          lede: "No amount of backing off gets a different answer out of this URL.",
+          sections: [
+            {
+              h2: "When backing off is not enough",
+              body: "A crawler should slow down and retry a 429, but it cannot wait forever: a thousand-page audit that pauses a minute per refusal never finishes. After a few spaced-out attempts the honest thing is to record what happened and move on to the rest of the site.",
+            },
+            {
+              h2: "This is a rate limit, not a block",
+              body: "The distinction matters because the fixes are different. A blocked page means bot protection decided the crawler was not welcome, and the fix is an allowlist rule. A rate-limited page means the server capped how many requests it will answer, and the fix is a higher limit, an exception for known crawlers, or a smaller crawl. Reporting one as the other sends the site owner into the wrong settings screen.",
+            },
+            {
+              h2: "Search engines see this too",
+              body: "Googlebot treats sustained 429s as a signal to crawl less. If a rate limit is strict enough to stop an audit crawler, it is strict enough to slow down how quickly your new and updated pages get discovered and indexed. That makes it worth fixing, not just working around.",
+            },
+          ],
+        }),
+      }),
+      { status: 429 },
+    ),
+};
+
 export const httpStatusFixtures: Fixture[] = [
  notFound,
  serverError,
  blocked,
+  rateLimitedThenOk,
+  rateLimitedAlways,
  brokenInternalLink,
 ];
@@ -31,6 +31,9 @@ export function ResultsView({
  onTabChange: (tab: ResultsTab) => void;
 }) {
  const { audit, pages, lighthouse, issues } = data;
+  const crawlStopped = issues.some(
+    (issue) => issue.issueType === "crawl-rate-limited",
+  );
  const hasPerformanceTab = lighthouse.length > 0;
  const activeTab =
    tab === "performance" && !hasPerformanceTab ? "issues" : tab;
@@ -39,43 +42,55 @@ export function ResultsView({
    () => pages.filter((page) => page.fetchClass === "blocked").length,
    [pages],
  );
+  const rateLimitedCount = useMemo(
+    () => pages.filter((page) => page.fetchClass === "rate_limited").length,
+    [pages],
+  );

  return (
    <>
      {blockedCount > 0 && (
-        <div className="flex items-start gap-3 rounded-lg border border-warning/30 bg-warning/5 px-4 py-3 text-sm">
-          <ShieldAlert className="mt-0.5 size-4 shrink-0 text-warning" />
-          <p>
-            <span className="font-medium">
-              We were blocked on {blockedCount}{" "}
-              {blockedCount === 1 ? "page" : "pages"}.
-            </span>{" "}
-            <span className="text-base-content/70">
-              The site's bot protection challenged our crawler, so those pages
-              couldn't be audited. We don't have a workaround for this yet.
-              Desktop crawlers run from your own machine and usually get past
-              it: try{" "}
-              <a
-                className="link link-primary"
-                href="https://github.com/PhialsBasement/LibreCrawl"
-                target="_blank"
-                rel="noreferrer"
-              >
-                LibreCrawl
-              </a>{" "}
-              (free, open source) or{" "}
-              <a
-                className="link link-primary"
-                href="https://www.screamingfrog.co.uk/seo-spider/"
-                target="_blank"
-                rel="noreferrer"
-              >
-                Screaming Frog
-              </a>{" "}
-              (free up to 500 URLs).
-            </span>
-          </p>
-        </div>
+        <CrawlWarning
+          headline={`We were blocked on ${blockedCount} ${blockedCount === 1 ? "page" : "pages"}.`}
+        >
+          The site's bot protection challenged our crawler, so those pages
+          couldn't be audited. We don't have a workaround for this yet. Desktop
+          crawlers run from your own machine and usually get past it: try{" "}
+          <a
+            className="link link-primary"
+            href="https://github.com/PhialsBasement/LibreCrawl"
+            target="_blank"
+            rel="noreferrer"
+          >
+            LibreCrawl
+          </a>{" "}
+          (free, open source) or{" "}
+          <a
+            className="link link-primary"
+            href="https://www.screamingfrog.co.uk/seo-spider/"
+            target="_blank"
+            rel="noreferrer"
+          >
+            Screaming Frog
+          </a>{" "}
+          (free up to 500 URLs).
+        </CrawlWarning>
+      )}
+
+      {(rateLimitedCount > 0 || crawlStopped) && (
+        <CrawlWarning
+          headline={
+            crawlStopped
+              ? "The crawl stopped early because of the site’s rate limit."
+              : `The site rate limited us on ${rateLimitedCount} ${rateLimitedCount === 1 ? "page" : "pages"}.`
+          }
+        >
+          {crawlStopped
+            ? "The requested cooldown exceeded the audit time limit, so some URLs were left unvisited. This report is incomplete. "
+            : "Pages that returned 429 Too Many Requests could not be audited. "}
+          Re-run the audit after the rate limit resets, or ask the site owner to
+          allow the "OpenSEO-Audit" crawler.
+        </CrawlWarning>
      )}

      <StatsStrip
@@ -130,6 +145,25 @@ export function ResultsView({
  );
 }

+/** Banner for pages the crawler could not read (bot protection, rate limits). */
+function CrawlWarning({
+  headline,
+  children,
+}: {
+  headline: string;
+  children: ReactNode;
+}) {
+  return (
+    <div className="flex items-start gap-3 rounded-lg border border-warning/30 bg-warning/5 px-4 py-3 text-sm">
+      <ShieldAlert className="mt-0.5 size-4 shrink-0 text-warning" />
+      <p>
+        <span className="font-medium">{headline}</span>{" "}
+        <span className="text-base-content/70">{children}</span>
+      </p>
+    </div>
+  );
+}
+
 function useResultStats(
  pages: AuditResultsData["pages"],
  lighthouse: AuditResultsData["lighthouse"],
@@ -6,6 +6,7 @@ import {
  index,
 } from "drizzle-orm/sqlite-core";
 import { sql } from "drizzle-orm";
+import { PAGE_FETCH_CLASSES } from "@/shared/audit-fetch-class";
 import { projects } from "./app.schema";

 // ============================================================================
@@ -113,8 +114,7 @@ export const auditPages = sqliteTable(
      .default(false),
    // SHA-256 of the visible body text, for duplicate-content grouping
    contentHash: text("content_hash"),
-    // How the fetch resolved: ok | blocked (WAF/bot challenge) | error
-    fetchClass: text("fetch_class", { enum: ["ok", "blocked", "error"] })
+    fetchClass: text("fetch_class", { enum: PAGE_FETCH_CLASSES })
      .notNull()
      .default("ok"),
    // Performance
@@ -7,6 +7,7 @@ import {
  real,
  text,
 } from "drizzle-orm/pg-core";
+import { PAGE_FETCH_CLASSES } from "@/shared/audit-fetch-class";
 import { projects } from "./app.schema";

 // Timestamps are stored as *text* (same column shape as the SQLite schema); see
@@ -112,8 +113,7 @@ export const auditPages = pgTable(
    inSitemap: boolean("in_sitemap").notNull().default(false),
    // SHA-256 of the visible body text, for duplicate-content grouping
    contentHash: text("content_hash"),
-    // How the fetch resolved: ok | blocked (WAF/bot challenge) | error
-    fetchClass: text("fetch_class", { enum: ["ok", "blocked", "error"] })
+    fetchClass: text("fetch_class", { enum: PAGE_FETCH_CLASSES })
      .notNull()
      .default("ok"),
    // Performance
@@ -22,6 +22,7 @@ import type {
  CrawledPageResult,
  LighthouseResult,
 } from "@/server/lib/audit/types";
+import type { PageFetchClass } from "@/shared/audit-fetch-class";

 async function createAudit(data: {
  id: string;
@@ -309,17 +310,20 @@ async function getPagesForAudit(auditId: string) {
    .where(eq(auditPages.auditId, auditId));
 }

-async function countBlockedPages(auditId: string): Promise<number> {
+async function countPagesByFetchClass(
+  auditId: string,
+  fetchClass: PageFetchClass,
+): Promise<number> {
  const rows = await db
-    .select({ blocked: count() })
+    .select({ pages: count() })
    .from(auditPages)
    .where(
      and(
        eq(auditPages.auditId, auditId),
-        eq(auditPages.fetchClass, "blocked"),
+        eq(auditPages.fetchClass, fetchClass),
      ),
    );
-  return rows[0]?.blocked ?? 0;
+  return rows[0]?.pages ?? 0;
 }

 async function hasPagesForAudit(auditId: string): Promise<boolean> {
@@ -439,7 +443,7 @@ export const AuditRepository = {
  getLatestAuditForProject,
  getIssuesForAudit,
  getPagesForAudit,
-  countBlockedPages,
+  countPagesByFetchClass,
  hasPagesForAudit,
  getAuditsByProject,
  getAuditUsageForOrganization,
@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";
+
+afterEach(() => {
+  vi.useRealTimers();
+});
+
+describe("createCrawlThrottle", () => {
+  it("backs off exponentially and still pauses after a URL exhausts its retries", async () => {
+    vi.useFakeTimers();
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+    for (const [attempt, delay] of [
+      [1, 1_000],
+      [2, 2_000],
+      [3, 4_000],
+      [4, 8_000],
+    ]) {
+      expect(throttle.backoff(attempt, null)).toBe(attempt <= 3);
+      let ready = false;
+      const waiting = throttle.ready().then((result) => {
+        ready = result;
+      });
+      await vi.advanceTimersByTimeAsync(delay - 1);
+      expect(ready).toBe(false);
+      await vi.advanceTimersByTimeAsync(1);
+      await waiting;
+      expect(ready).toBe(true);
+    }
+  });
+
+  it.each(["60", "Thu, 01 Jan 2026 00:01:00 GMT"])(
+    "honors a full sixty-second Retry-After: %s",
+    async (retryAfter) => {
+      vi.useFakeTimers();
+      vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));
+      const throttle = createCrawlThrottle(Date.now() + 90_000);
+      expect(throttle.backoff(1, retryAfter)).toBe(true);
+      let ready = false;
+      const waiting = throttle.ready().then((result) => {
+        ready = result;
+      });
+      await vi.advanceTimersByTimeAsync(59_999);
+      expect(ready).toBe(false);
+      await vi.advanceTimersByTimeAsync(1);
+      await waiting;
+      expect(ready).toBe(true);
+    },
+  );
+
+  it("does not label an ordinary chunk deadline as a rate-limit stop", async () => {
+    vi.useFakeTimers();
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+    await vi.advanceTimersByTimeAsync(90_000);
+    expect(await throttle.ready()).toBe(false);
+    expect(throttle.stopped).toBe(false);
+  });
+
+  it("stops without scheduling a timer when the cooldown exceeds the crawl budget", async () => {
+    vi.useFakeTimers();
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+    expect(throttle.backoff(1, "600")).toBe(false);
+    expect(throttle.stopped).toBe(true);
+    expect(await throttle.ready()).toBe(false);
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it("holds a waiter through a pause that another 429 extends", async () => {
+    vi.useFakeTimers();
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+    throttle.backoff(1, "1");
+    let released = false;
+    const waiting = throttle.ready().then((result) => {
+      released = result;
+    });
+    await vi.advanceTimersByTimeAsync(900);
+    throttle.backoff(1, "1");
+    await vi.advanceTimersByTimeAsync(200);
+    expect(released).toBe(false);
+    await vi.advanceTimersByTimeAsync(800);
+    await waiting;
+    expect(released).toBe(true);
+  });
+});
@@ -0,0 +1,52 @@
+/** Retries per URL after its first 429. Every 429 still pauses new URLs. */
+const MAX_RETRIES = 3;
+const FIRST_DELAY_MS = 1_000;
+
+/** `Retry-After` is either delay-seconds or an HTTP-date. */
+function parseRetryAfterMs(header: string | null): number | null {
+  if (!header) return null;
+  const value = header.trim();
+  if (/^\d+$/.test(value)) return Number(value) * 1_000;
+  const at = Date.parse(value);
+  return Number.isNaN(at) ? null : Math.max(0, at - Date.now());
+}
+
+export interface CrawlThrottle {
+  /** False when this chunk can no longer start a request. */
+  ready(): Promise<boolean>;
+  /** Pause the origin on every 429; return whether this URL may retry. */
+  backoff(attempt: number, retryAfter: string | null): boolean;
+  readonly stopped: boolean;
+}
+
+/** One origin cooldown, bounded by the scheduler's existing chunk deadline. */
+export function createCrawlThrottle(deadlineAt: number): CrawlThrottle {
+  let pausedUntil = 0;
+  let stopped = false;
+  return {
+    async ready() {
+      // Another request can extend the shared pause while this one waits.
+      while (pausedUntil > Date.now()) {
+        if (stopped) return false;
+        await new Promise((resolve) =>
+          setTimeout(resolve, pausedUntil - Date.now()),
+        );
+      }
+      return !stopped && Date.now() < deadlineAt;
+    },
+    backoff(attempt, retryAfter) {
+      const delayMs = Math.max(
+        FIRST_DELAY_MS,
+        parseRetryAfterMs(retryAfter) ?? FIRST_DELAY_MS * 2 ** (attempt - 1),
+      );
+      pausedUntil = Math.max(pausedUntil, Date.now() + delayMs);
+      // Never shorten the site's requested wait to fit our budget. Stop the
+      // crawl instead; the scheduler preserves unvisited URLs as incomplete.
+      if (pausedUntil >= deadlineAt) stopped = true;
+      return !stopped && attempt <= MAX_RETRIES;
+    },
+    get stopped() {
+      return stopped;
+    },
+  };
+}
@@ -3,10 +3,8 @@ import {
  adjustCrawlWindow,
  RETRY_CRAWL_WINDOW,
 } from "@/server/lib/audit/crawl-window";
-import type {
-  CrawledPageResult,
-  PageFetchClass,
-} from "@/server/lib/audit/types";
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+import type { PageFetchClass } from "@/shared/audit-fetch-class";

 function page(
  fetchClass: PageFetchClass,
@@ -39,6 +37,7 @@ function page(
    contentHash: null,
    isHtml: true,
    htmlBytes,
+    rateLimited: false,
    imagesTotal: 0,
    imagesMissingAlt: 0,
    images: [],
@@ -70,6 +69,14 @@ describe("adjustCrawlWindow", () => {
    expect(adjustCrawlWindow(20, recent)).toBe(10);
  });

+  it("treats a 429 the retries recovered from as trouble", () => {
+    const recent = Array.from({ length: 10 }, () => ({
+      ...page("ok", 300),
+      rateLimited: true,
+    }));
+    expect(adjustCrawlWindow(20, recent)).toBe(10);
+  });
+
  it("never shrinks below the minimum", () => {
    const recent = Array.from({ length: 10 }, () => page("error", 15_000));
    expect(adjustCrawlWindow(6, recent)).toBe(5);
@@ -75,6 +75,9 @@ export function adjustCrawlWindow(
  const troubled = recent.filter(
    (page) =>
      page.fetchClass !== "ok" ||
+      // A 429 the retries recovered from still says we are crawling faster
+      // than the site allows.
+      page.rateLimited ||
      (page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS,
  ).length;
  let next = windowSize;
@@ -4,6 +4,7 @@
 * orphans) live in multipage.ts.
 */
 import type { DetectedIssue } from "@/server/lib/audit/issues/page-reporters";
+import type { PageFetchClass } from "@/shared/audit-fetch-class";

 const DUPLICATE_GROUP_SAMPLE = 3;

@@ -11,7 +12,7 @@ export interface SlimPage {
  id: string;
  url: string;
  statusCode: number | null;
-  fetchClass: "ok" | "blocked" | "error";
+  fetchClass: PageFetchClass;
  title: string | null;
  metaDescription: string | null;
  contentHash: string | null;
@@ -42,6 +42,7 @@ function makePage(overrides: Partial<CrawledPageResult>): CrawledPageResult {
    contentHash: "abc123",
    isHtml: true,
    htmlBytes: 10_000,
+    rateLimited: false,
    imagesTotal: 0,
    imagesMissingAlt: 0,
    images: [],
@@ -71,6 +72,12 @@ describe("runPageReporters", () => {
    ).toEqual(["blocked-page"]);
  });

+  it("reports only rate-limited-page for a fetch the site kept 429ing", () => {
+    expect(
+      issueTypes(makePage({ fetchClass: "rate_limited", statusCode: 429 })),
+    ).toEqual(["rate-limited-page"]);
+  });
+
  it("reports nothing for a fetch error", () => {
    expect(
      issueTypes(makePage({ fetchClass: "error", statusCode: 0 })),
@@ -50,6 +50,10 @@ export function runPageReporters(page: CrawledPageResult): DetectedIssue[] {
    report("blocked-page", { statusCode: page.statusCode });
    return issues;
  }
+  if (page.fetchClass === "rate_limited") {
+    report("rate-limited-page", { statusCode: page.statusCode });
+    return issues;
+  }
  if (page.fetchClass === "error") {
    return issues;
  }
@@ -3,6 +3,7 @@
 */

 import { z } from "zod";
+import type { PageFetchClass } from "@/shared/audit-fetch-class";
 import { MIN_AUDIT_PAGES, PAID_MAX_AUDIT_PAGES } from "@/shared/audit-limits";
 import { jsonCodec } from "@/shared/json";

@@ -39,9 +40,6 @@ export function parseAuditConfig(configRaw: string | null): AuditConfig | null {
  return result.success ? result.data : null;
 }

-/** How a page fetch resolved. "blocked" = WAF/bot challenge stood in the way. */
-export type PageFetchClass = "ok" | "blocked" | "error";
-
 /** One outgoing link edge, deduped by target URL within a page. */
 export interface PageLink {
  targetUrl: string;
@@ -146,6 +144,12 @@ export interface CrawledPageResult {
   * response time is measured at headers and says nothing about body size.
   */
  htmlBytes: number;
+  /**
+   * True when a 429 was retried for this URL (whatever the retry returned).
+   * Not persisted — narrows the crawl window so the pages after it are
+   * fetched more slowly.
+   */
+  rateLimited: boolean;
  imagesTotal: number;
  imagesMissingAlt: number;
  images: Array<{ src: string | null; alt: string | null }>;
@@ -9,6 +9,7 @@ import {
  getIssueDescriptor,
  ISSUE_SEVERITY_ORDER,
 } from "@/shared/audit-issues";
+import { PAGE_FETCH_CLASSES } from "@/shared/audit-fetch-class";
 import { mcpResponse } from "@/server/mcp/formatters";
 import { buildProjectMeta } from "@/server/mcp/context";
 import {
@@ -69,7 +70,7 @@ export const runSiteAuditTool = {
  config: {
    title: "Run site audit",
    description:
-      "Start a site audit: crawls the site (robots.txt-aware, same-origin), checks every page for SEO issues (broken links, duplicate/missing titles and descriptions, redirect chains, orphan pages, canonical problems, thin content, and more), and optionally runs Lighthouse on a sample of pages. Runs in the background — poll get_audit_status, then read get_audit_issues. If the site blocks our crawler, pages are honestly flagged as blocked rather than misreported.",
+      "Start a site audit: crawls the site (robots.txt-aware, same-origin), checks every page for SEO issues (broken links, duplicate/missing titles and descriptions, redirect chains, orphan pages, canonical problems, thin content, and more), and optionally runs Lighthouse on a sample of pages. Runs in the background — poll get_audit_status, then read get_audit_issues. If the site rate limits the crawler it slows down and retries; pages it still cannot read are honestly flagged as blocked or rate-limited rather than misreported.",
    inputSchema: runInputSchema,
    outputSchema: z
      .object({
@@ -333,10 +334,10 @@ const pagesInputSchema = {
  projectId: projectIdSchema,
  auditId: auditIdSchema,
  fetchClass: z
-    .enum(["ok", "blocked", "error"])
+    .enum(PAGE_FETCH_CLASSES)
    .optional()
    .describe(
-      'Filter by fetch outcome ("blocked" = the site\'s bot protection challenged the crawler).',
+      'Filter by fetch outcome ("blocked" = the site\'s bot protection challenged the crawler; "rate_limited" = a 429 prevented the crawler from reading the page).',
    ),
  statusCode: z
    .number()
@@ -1,10 +1,176 @@
 import { spawnSync } from "node:child_process";
 import { afterEach, describe, expect, it, vi } from "vitest";
-import { crawlPage } from "./site-audit-workflow-helpers";
+import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";
+import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";

-afterEach(() => vi.unstubAllGlobals());
+const PAGE_URL = "https://example.com/page";
+const PAGE_HTML =
+  "<html><head><title>A page</title></head><body><h1>A page</h1></body></html>";
+
+/**
+ * Answer each fetch with the next reply, repeating the last one. Every call
+ * builds a fresh Response: a body can only be read (or cancelled) once.
+ */
+function stubFetch(...replies: Array<{ status: number; retryAfter?: string }>) {
+  let index = 0;
+  return vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+    const reply = replies[Math.min(index++, replies.length - 1)];
+    return new Response(PAGE_HTML, {
+      status: reply.status,
+      headers: {
+        "content-type": "text/html",
+        ...(reply.retryAfter ? { "retry-after": reply.retryAfter } : {}),
+      },
+    });
+  });
+}
+
+function crawl() {
+  return crawlPage(
+    PAGE_URL,
+    0,
+    false,
+    createCrawlThrottle(Date.now() + 90_000),
+  );
+}
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllGlobals();
+});

 describe("crawlPage", () => {
+  it("waits out the server's Retry-After and keeps the retried page", async () => {
+    vi.useFakeTimers();
+    const fetchMock = stubFetch(
+      { status: 429, retryAfter: "5" },
+      { status: 200 },
+    );
+
+    const crawled = crawl();
+    await vi.advanceTimersByTimeAsync(4_000);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+
+    await vi.advanceTimersByTimeAsync(2_000);
+    const page = await crawled;
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    expect(page?.fetchClass).toBe("ok");
+    expect(page?.title).toBe("A page");
+    // Recovered, but the crawl window should still slow down after it.
+    expect(page?.rateLimited).toBe(true);
+  });
+
+  it("holds every other fetch in the chunk while one URL's 429 pause runs", async () => {
+    vi.useFakeTimers();
+    const fetched: string[] = [];
+    vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
+      fetched.push(new Request(input).url);
+      // Only the first request to the first URL is refused.
+      const refused = fetched.length === 1;
+      return new Response(PAGE_HTML, {
+        status: refused ? 429 : 200,
+        headers: {
+          "content-type": "text/html",
+          ...(refused ? { "retry-after": "5" } : {}),
+        },
+      });
+    });
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+
+    const first = crawlPage(`${PAGE_URL}/first`, 0, false, throttle);
+    await vi.advanceTimersByTimeAsync(0);
+    const second = crawlPage(`${PAGE_URL}/second`, 0, false, throttle);
+    await vi.advanceTimersByTimeAsync(4_000);
+    expect(fetched).toEqual([`${PAGE_URL}/first`]);
+
+    await vi.advanceTimersByTimeAsync(2_000);
+    const pages = await Promise.all([first, second]);
+    expect(fetched).toHaveLength(3);
+    expect(pages.map((page) => page?.fetchClass)).toEqual(["ok", "ok"]);
+  });
+
+  it("records a page the site keeps rate limiting, without calling it blocked", async () => {
+    vi.useFakeTimers();
+    const fetchMock = stubFetch({ status: 429 });
+
+    const crawled = crawl();
+    await vi.advanceTimersByTimeAsync(30_000);
+    const page = await crawled;
+
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    expect(page?.fetchClass).toBe("rate_limited");
+  });
+
+  it("keeps the origin paused after a URL exhausts its retries", async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(0);
+    const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(
+      async () =>
+        new Response(PAGE_HTML, {
+          status: Date.now() < 15_000 ? 429 : 200,
+          headers: { "content-type": "text/html" },
+        }),
+    );
+    const throttle = createCrawlThrottle(90_000);
+    const first = crawlPage(`${PAGE_URL}/first`, 0, false, throttle);
+    await vi.advanceTimersByTimeAsync(7_000);
+    expect((await first)?.fetchClass).toBe("rate_limited");
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+
+    const second = crawlPage(`${PAGE_URL}/second`, 0, false, throttle);
+    await vi.advanceTimersByTimeAsync(7_999);
+    expect(fetchMock).toHaveBeenCalledTimes(4);
+    await vi.advanceTimersByTimeAsync(1);
+    expect((await second)?.fetchClass).toBe("ok");
+  });
+
+  it("recovers concurrent pages when the origin asks for sixty seconds", async () => {
+    vi.useFakeTimers();
+    vi.setSystemTime(0);
+    const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(
+      async () =>
+        new Response(PAGE_HTML, {
+          status: Date.now() < 60_000 ? 429 : 200,
+          headers: { "content-type": "text/html", "retry-after": "60" },
+        }),
+    );
+    const throttle = createCrawlThrottle(90_000);
+    const pages = Promise.all(
+      Array.from({ length: 5 }, (_, i) =>
+        crawlPage(`${PAGE_URL}/${i}`, 0, false, throttle),
+      ),
+    );
+    await vi.advanceTimersByTimeAsync(59_999);
+    expect(fetchMock).toHaveBeenCalledTimes(5);
+    await vi.advanceTimersByTimeAsync(1);
+    expect((await pages).map((page) => page?.fetchClass)).toEqual(
+      Array(5).fill("ok"),
+    );
+    expect(fetchMock).toHaveBeenCalledTimes(10);
+  });
+
+  it("records the observed 429 but does not fetch another URL when the wait cannot fit", async () => {
+    vi.useFakeTimers();
+    const fetchMock = stubFetch({ status: 429, retryAfter: "600" });
+    const throttle = createCrawlThrottle(Date.now() + 90_000);
+    expect((await crawlPage(PAGE_URL, 0, false, throttle))?.fetchClass).toBe(
+      "rate_limited",
+    );
+    expect(
+      await crawlPage(`${PAGE_URL}/unvisited`, 0, false, throttle),
+    ).toBeNull();
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry a 403 — bot protection is not a speed limit", async () => {
+    const fetchMock = stubFetch({ status: 403 });
+
+    const page = await crawl();
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(page?.fetchClass).toBe("blocked");
+  });
+
  it("preserves extracted metadata and nested values when releasing the HTML", async () => {
    vi.stubGlobal(
      "fetch",
@@ -37,7 +203,12 @@ describe("crawlPage", () => {
      ),
    );

-    const page = await crawlPage("https://example.com/", 2, true);
+    const page = await crawlPage(
+      "https://example.com/",
+      2,
+      true,
+      createCrawlThrottle(Date.now() + 90_000),
+    );

    expect(page).toMatchObject({
      url: "https://example.com/",
@@ -85,8 +256,8 @@ describe("crawlPage", () => {
      crawlDepth: 2,
      inSitemap: true,
    });
-    expect(page.contentHash).toMatch(/^[a-f0-9]{64}$/);
-    expect(page.htmlBytes).toBeGreaterThan(0);
+    expect(page?.contentHash).toMatch(/^[a-f0-9]{64}$/);
+    expect(page?.htmlBytes).toBeGreaterThan(0);
  });

  it("does not retain large source HTML in queued crawl results", () => {
@@ -104,6 +275,8 @@ describe("crawlPage", () => {
        `
          import assert from "node:assert/strict";
          import { crawlPage } from ${JSON.stringify(new URL("./site-audit-workflow-helpers.ts", import.meta.url).href)};
+          import { createCrawlThrottle } from ${JSON.stringify(new URL("../lib/audit/crawl-throttle.ts", import.meta.url).href)};
+          const throttle = createCrawlThrottle(Date.now() + 90_000);
          globalThis.fetch = async () => {
            const html = '<title>Example memory regression 🌱</title>' +
              '<meta name="description" content="A small description">' +
@@ -125,11 +298,11 @@ describe("crawlPage", () => {
            const { heapUsed, external } = process.memoryUsage();
            return heapUsed + external;
          };
-          await crawlPage("https://example.com/warmup", 0, true);
+          await crawlPage("https://example.com/warmup", 0, true, throttle);
          const before = await collect();
          const pages = [];
          for (let i = 0; i < 50; i++) {
-            pages.push(await crawlPage("https://example.com/" + i, 0, true));
+            pages.push(await crawlPage("https://example.com/" + i, 0, true, throttle));
          }
          const retained = (await collect()) - before;
          assert.equal(pages.length, 50);
@@ -1,9 +1,8 @@
-import type {
-  CrawledPageResult,
-  PageFetchClass,
-} from "@/server/lib/audit/types";
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+import type { PageFetchClass } from "@/shared/audit-fetch-class";
 import { sha256Hex } from "@/server/lib/audit/ids";
 import { normalizeUrl } from "@/server/lib/audit/url-utils";
+import type { CrawlThrottle } from "@/server/lib/audit/crawl-throttle";

 const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";
 const MAX_HTML_BYTES = 1024 * 1024;
@@ -26,10 +25,12 @@ function classifyFetch(
  bodySnippet: string,
 ): PageFetchClass {
  if (statusCode === 0) return "error";
+  // A final 429 means rate limiting, whether retries were exhausted or the
+  // requested cooldown exceeded the crawl budget. Checked before
+  // cf-mitigated: a Cloudflare rate-limiting rule sets that header too.
+  if (statusCode === 429) return "rate_limited";
  if (headers.get("cf-mitigated")) return "blocked";
-  if (statusCode === 401 || statusCode === 403 || statusCode === 429) {
-    return "blocked";
-  }
+  if (statusCode === 401 || statusCode === 403) return "blocked";
  if (statusCode === 503) {
    const snippet = bodySnippet.toLowerCase();
    if (CHALLENGE_BODY_MARKERS.some((marker) => snippet.includes(marker))) {
@@ -55,14 +56,15 @@ function parseLinkHeaderCanonical(
  return null;
 }

-export async function crawlPage(
-  url: string,
-  crawlDepth: number | null,
-  inSitemap: boolean,
-): Promise<CrawledPageResult> {
-  const startTime = Date.now();
-
-  try {
+/**
+ * Fetch one URL, pausing the whole chunk and retrying while the site 429s
+ * (see crawl-throttle.ts). `responseTimeMs` is measured from the last attempt
+ * so backoff waiting never looks like a slow server.
+ */
+async function fetchPage(url: string, throttle: CrawlThrottle) {
+  for (let attempt = 1; ; attempt++) {
+    if (!(await throttle.ready())) return null;
+    const startedAt = Date.now();
    // Manual redirect handling: each hop is recorded as its own page row and
    // its target is enqueued by the frontier, so redirect chains and loops are
    // detectable from the recorded rows. Trailing-slash redirects (/docs ->
@@ -77,8 +79,38 @@ export async function crawlPage(
      redirect: "manual",
      signal: AbortSignal.timeout(15_000),
    });
+    const result = {
+      response,
+      responseTimeMs: Date.now() - startedAt,
+      // A retry means an earlier attempt was 429'd; a 429 handed back after
+      // the last retry is already classified rate_limited and needs no flag.
+      rateLimited: attempt > 1,
+    };
+    if (response.status !== 429) return result;

-    const responseTimeMs = Date.now() - startTime;
+    const retry = throttle.backoff(
+      attempt,
+      response.headers.get("retry-after"),
+    );
+    // The shared cooldown applies even when this URL has no retries left.
+    if (!retry) return result;
+    await response.body?.cancel();
+  }
+}
+
+/** Null leaves this URL deferred when the shared cooldown stops its fetch. */
+export async function crawlPage(
+  url: string,
+  crawlDepth: number | null,
+  inSitemap: boolean,
+  throttle: CrawlThrottle,
+): Promise<CrawledPageResult | null> {
+  const startTime = Date.now();
+
+  try {
+    const fetched = await fetchPage(url, throttle);
+    if (!fetched) return null;
+    const { response, responseTimeMs, rateLimited } = fetched;
    const statusCode = response.status;
    const xRobotsTag = response.headers.get("x-robots-tag");
    const headerCanonicalUrl = parseLinkHeaderCanonical(
@@ -99,6 +131,7 @@ export async function crawlPage(
        headerCanonicalUrl,
        crawlDepth,
        inSitemap,
+        rateLimited,
      });
    }

@@ -127,6 +160,7 @@ export async function crawlPage(
        // The body was still fetched and buffered; report its size so the
        // crawl window's byte budget sees blocked/error pages too.
        htmlBytes: body.length,
+        rateLimited,
      });
    }

@@ -177,6 +211,7 @@ export async function crawlPage(
        : null,
      isHtml: true,
      htmlBytes: body.length,
+      rateLimited,
      imagesTotal: analysis.images.length,
      // Only a truly absent alt attribute counts: alt="" is the correct
      // markup for decorative images.
@@ -251,6 +286,7 @@ function emptyPageResult(input: {
  crawlDepth: number | null;
  inSitemap: boolean;
  htmlBytes?: number;
+  rateLimited?: boolean;
 }): CrawledPageResult {
  return {
    id: crypto.randomUUID(),
@@ -278,6 +314,7 @@ function emptyPageResult(input: {
    contentHash: null,
    isHtml: false,
    htmlBytes: input.htmlBytes ?? 0,
+    rateLimited: input.rateLimited ?? false,
    imagesTotal: 0,
    imagesMissingAlt: 0,
    images: [],
@@ -0,0 +1,185 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { CrawledPageResult } from "@/server/lib/audit/types";
+
+const mocks = vi.hoisted(() => ({
+  claimChunk: vi.fn(),
+  getStats:
+    vi.fn<
+      () => Promise<{ attempted: number; pending: number; seen: number }>
+    >(),
+  recordBatch: vi.fn(),
+  releaseUrls: vi.fn<(urls: string[]) => Promise<void>>(),
+  insertCrawledBatch: vi.fn(),
+  pgStep: vi.fn(),
+}));
+vi.mock("@/server/features/audit/AuditScratchpad", () => ({
+  getAuditScratchpad: () => mocks,
+}));
+vi.mock("@/server/features/audit/repositories/AuditRepository", () => ({
+  AuditRepository: {
+    insertCrawledBatch: mocks.insertCrawledBatch,
+    updateAuditProgress: vi.fn(),
+  },
+}));
+vi.mock("@/server/lib/audit/progress-kv", () => ({
+  AuditProgressKV: { pushCrawledUrls: vi.fn() },
+}));
+vi.mock("@/server/workflows/pgStep", () => ({ pgStep: mocks.pgStep }));
+vi.mock("@/server/lib/audit/ids", () => ({
+  deterministicAuditRowId: async (_auditId: string, url: string) => url,
+  sha256Hex: async () => "content-hash",
+}));
+
+import { runCrawlPhase } from "@/server/workflows/siteAuditWorkflowCrawl";
+import { parseRobotsTxt } from "@/server/lib/audit/discovery";
+
+const ORIGIN = "https://example.com";
+const HTML = "<html><title>A page</title><body><h1>A page</h1></body></html>";
+let saved: CrawledPageResult[];
+
+beforeEach(async () => {
+  // Load the lazy HTML parser before advancing the fake network clock.
+  await import("@/server/lib/audit/page-analyzer");
+  vi.useFakeTimers();
+  vi.setSystemTime(0);
+  saved = [];
+  mocks.pgStep.mockImplementation((_step, _name, _config, fn: () => unknown) =>
+    fn(),
+  );
+  mocks.claimChunk.mockImplementation(
+    async (_chunk: number, limit: number) => ({
+      urls: Array.from({ length: limit }, (_, i) => ({
+        url: `${ORIGIN}/${i}`,
+        depth: 0,
+        inSitemap: true,
+      })),
+      isRetry: false,
+    }),
+  );
+  mocks.getStats.mockImplementation(async () => ({
+    attempted: saved.length,
+    pending: 100 - saved.length,
+    seen: 100,
+  }));
+  mocks.recordBatch.mockImplementation(() => mocks.getStats());
+  mocks.insertCrawledBatch.mockImplementation(
+    async (_audit, pages: CrawledPageResult[]) => {
+      saved.push(...pages);
+    },
+  );
+});
+afterEach(() => {
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+});
+
+function crawl(maxPages = 100) {
+  return runCrawlPhase(
+    { do: vi.fn(), sleep: vi.fn(), sleepUntil: vi.fn(), waitForEvent: vi.fn() },
+    {
+      auditId: "audit",
+      workflowInstanceId: "workflow",
+      origin: ORIGIN,
+      maxPages,
+      seededCount: maxPages,
+      robots: parseRobotsTxt("", ORIGIN),
+    },
+  );
+}
+
+function serve(status: (now: number) => number, retryAfter?: string) {
+  return vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+    // Network responses settle after all requests in the window are launched.
+    await new Promise((resolve) => setTimeout(resolve, 1));
+    return new Response(HTML, {
+      status: status(Date.now()),
+      headers: {
+        "content-type": "text/html",
+        ...(retryAfter ? { "retry-after": retryAfter } : {}),
+      },
+    });
+  });
+}
+
+describe("crawl rate-limit budget", () => {
+  it("honors the final URL's cooldown before starting the next chunk", async () => {
+    const total = 210;
+    mocks.claimChunk.mockImplementation(
+      async (chunk: number, limit: number) => ({
+        urls: Array.from({ length: limit }, (_, i) => ({
+          url: `${ORIGIN}/${(chunk - 1) * 200 + i}`,
+          depth: 0,
+          inSitemap: true,
+        })),
+        isRetry: false,
+      }),
+    );
+    mocks.getStats.mockImplementation(async () => ({
+      attempted: saved.length,
+      pending: total - saved.length,
+      seen: total,
+    }));
+    let lastRefusalAt = 0;
+    let nextChunkStartedAt = 0;
+    vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
+      const url = new Request(input).url;
+      if (url === `${ORIGIN}/200`) nextChunkStartedAt = Date.now();
+      await new Promise((resolve) => setTimeout(resolve, 1));
+      const refused = url === `${ORIGIN}/199`;
+      if (refused) lastRefusalAt = Date.now();
+      return new Response(HTML, {
+        status: refused ? 429 : 200,
+        headers: { "content-type": "text/html", "retry-after": "5" },
+      });
+    });
+
+    const result = crawl(total);
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(await result).toEqual({ pagesCrawled: total, completed: true });
+    expect(mocks.claimChunk).toHaveBeenCalledTimes(2);
+    expect(lastRefusalAt).toBeGreaterThan(0);
+    expect(nextChunkStartedAt - lastRefusalAt).toBeGreaterThanOrEqual(5_000);
+  });
+
+  it("finishes all pages after a sixty-second origin cooldown", async () => {
+    const fetchMock = serve((now) => (now < 60_000 ? 429 : 200), "60");
+    const result = crawl();
+    await vi.advanceTimersByTimeAsync(59_999);
+    expect(fetchMock).toHaveBeenCalledTimes(10);
+    await vi.advanceTimersByTimeAsync(1);
+    await vi.advanceTimersByTimeAsync(10_000);
+    expect(await result).toEqual({ pagesCrawled: 100, completed: true });
+    expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);
+    expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
+  });
+
+  it("stops a permanently limited origin without restarting in another chunk", async () => {
+    const fetchMock = serve(() => 429);
+    const result = crawl();
+    await vi.advanceTimersByTimeAsync(100_000);
+    const outcome = await result;
+    expect(outcome).toMatchObject({ completed: false, rateLimited: true });
+    expect(outcome.pagesCrawled).toBeLessThan(100);
+    expect(fetchMock.mock.calls.length).toBeLessThan(400);
+    expect(saved.every((page) => page.fetchClass === "rate_limited")).toBe(
+      true,
+    );
+    expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
+    expect(mocks.releaseUrls).toHaveBeenCalledTimes(1);
+    expect(mocks.releaseUrls.mock.calls[0][0].length).toBe(100 - saved.length);
+  });
+
+  it("leaves URLs unvisited when Retry-After is longer than the entire budget", async () => {
+    const fetchMock = serve(() => 429, "600");
+    const result = crawl();
+    await vi.advanceTimersByTimeAsync(100);
+    expect(await result).toEqual({
+      pagesCrawled: 10,
+      completed: false,
+      rateLimited: true,
+    });
+    expect(fetchMock).toHaveBeenCalledTimes(10);
+    expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(90);
+    expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
+  });
+});
@@ -19,6 +19,7 @@ import {
  CRAWL_WINDOW,
  RETRY_CRAWL_WINDOW,
 } from "@/server/lib/audit/crawl-window";
+import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";
 import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
 import { pgStep } from "@/server/workflows/pgStep";
 import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
@@ -92,6 +93,7 @@ export type CrawlPhaseResult = {
  pagesCrawled: number;
  /** True when the frontier was exhausted before hitting maxPages. */
  completed: boolean;
+  rateLimited?: boolean;
 };

 export async function runCrawlPhase(
@@ -127,6 +129,13 @@ export async function runCrawlPhase(
    // with up-to-date scratchpad totals) — finalize must not see stale ones.
    attemptedTotal = result.attempted;
    pending = result.pending;
+    if (result.rateLimited) {
+      return {
+        pagesCrawled: attemptedTotal,
+        completed: false,
+        rateLimited: true,
+      };
+    }
    // `?? initial`: an instance in flight across a deploy replays cached
    // step results from before endWindow existed.
    windowHint = result.endWindow ?? CRAWL_WINDOW.initial;
@@ -152,6 +161,7 @@ async function runCrawlChunk(
  attempted: number;
  pending: number;
  endWindow: number;
+  rateLimited?: boolean;
 }> {
  const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
    input;
@@ -188,6 +198,9 @@ async function runCrawlChunk(
  let nextIndex = 0;
  let attemptedInChunk = 0;
  const inFlight = new Set<Promise<void>>();
+  // Shared by every fetch in the chunk: one page's 429 pauses them all.
+  const throttle = createCrawlThrottle(deadlineAt);
+  const deferred: string[] = [];
  let persistThreshold = FIRST_PERSIST_BATCH_SIZE;
  let batch: CrawledPageResult[] = [];
  // Persistence runs concurrently with fetching (pipelined) but sequentially
@@ -221,8 +234,12 @@ async function runCrawlChunk(
  };

  const launch = (entry: ClaimedUrl) => {
-    const promise = crawlPage(entry.url, entry.depth, entry.inSitemap)
+    const promise = crawlPage(entry.url, entry.depth, entry.inSitemap, throttle)
      .then((page) => {
+        if (!page) {
+          deferred.push(entry.url);
+          return;
+        }
        attemptedInChunk += 1;
        batch.push(page);
        if (batch.length >= persistThreshold) flush();
@@ -242,7 +259,8 @@ async function runCrawlChunk(
      // queuedPersists changes when persistChain settles. Keep it out of the
      // loop condition because the type-aware linter cannot see that async
      // mutation and flags the otherwise valid backpressure check.
-      if (queuedPersists > MAX_QUEUED_PERSIST_BATCHES) break;
+      if (throttle.stopped || queuedPersists > MAX_QUEUED_PERSIST_BATCHES)
+        break;
      launch(claimed[nextIndex]);
      nextIndex += 1;
    }
@@ -254,6 +272,7 @@ async function runCrawlChunk(
    // backpressure, wait for the queue to drain and resume; otherwise the
    // chunk is done (leases exhausted or soft deadline hit).
    if (
+      !throttle.stopped &&
      queuedPersists > MAX_QUEUED_PERSIST_BATCHES &&
      nextIndex < claimed.length &&
      Date.now() < deadlineAt
@@ -266,8 +285,11 @@ async function runCrawlChunk(
  flush();
  await persistChain;

-  // Leases we never launched (soft deadline) go back to the queue.
-  const unattempted = claimed.slice(nextIndex).map((entry) => entry.url);
+  // Preserve URLs without a page result, including slots stopped by a cooldown.
+  const unattempted = [
+    ...deferred,
+    ...claimed.slice(nextIndex).map((entry) => entry.url),
+  ];
  if (unattempted.length > 0) {
    await scratchpad.releaseUrls(unattempted);
  }
@@ -276,11 +298,17 @@ async function runCrawlChunk(
  // persistCrawledPages), so a chunk that dies mid-way underreports by at
  // most one sub-batch, not a whole chunk.
  const stats = await scratchpad.getStats();
+  // The next chunk creates a fresh throttle. Honor the last response's
+  // cooldown before allowing it to fetch more URLs.
+  if (stats.pending > 0 && stats.attempted < maxPages) {
+    await throttle.ready();
+  }
  return {
    attemptedInChunk,
    attempted: stats.attempted,
    pending: stats.pending,
    endWindow: windowSize,
+    rateLimited: throttle.stopped,
  };
 }

@@ -350,12 +350,26 @@ async function finalizeAudit(args: {

    const issues = await runMultipageChecks({ auditId });
    issues.push(...(await runScratchpadLinkChecks(auditId, startUrl, crawl)));
+    if (crawl.rateLimited) {
+      issues.push({
+        issueType: "crawl-rate-limited",
+        pageId: null,
+        pageUrl: startUrl,
+      });
+    }
    await AuditRepository.insertIssues(auditId, issues);
    return { issueCount: issues.length };
  });

  await pgStep(step, "finalize", DB_STEP, async () => {
-    const blockedPages = await AuditRepository.countBlockedPages(auditId);
+    const blockedPages = await AuditRepository.countPagesByFetchClass(
+      auditId,
+      "blocked",
+    );
+    const rateLimitedPages = await AuditRepository.countPagesByFetchClass(
+      auditId,
+      "rate_limited",
+    );
    await AuditRepository.completeAudit(auditId, workflowInstanceId, {
      pagesCrawled: crawl.pagesCrawled,
      pagesTotal: crawl.pagesCrawled,
@@ -371,6 +385,7 @@ async function finalizeAudit(args: {
        pages_total: crawl.pagesCrawled,
        crawl_completed: crawl.completed,
        pages_blocked: blockedPages,
+        pages_rate_limited: rateLimitedPages,
        run_lighthouse: config.lighthouseStrategy !== "none",
      },
    });
@@ -0,0 +1,12 @@
+// How a page fetch resolved. "blocked" = WAF/bot challenge stood in the way;
+// "rate_limited" = a 429 prevented the crawler from reading the page.
+// Declared once so the SQLite and Postgres columns, the MCP filter, and the
+// PageFetchClass type can't drift apart.
+export const PAGE_FETCH_CLASSES = [
+  "ok",
+  "blocked",
+  "rate_limited",
+  "error",
+] as const;
+
+export type PageFetchClass = (typeof PAGE_FETCH_CLASSES)[number];
@@ -20,10 +20,26 @@ export const AUDIT_ISSUE_TYPES = {
    severity: "critical",
    title: "Crawler was blocked",
    explanation:
-      "The site returned a bot challenge or access denial (e.g. a Cloudflare challenge, 403, or 429) instead of the page. We report this honestly rather than pretending the page is broken — but it means this page could not be audited, and other crawlers like search engines may face similar friction.",
+      "The site returned a bot challenge or access denial (e.g. a Cloudflare challenge or a 403) instead of the page. We report this honestly rather than pretending the page is broken — but it means this page could not be audited, and other crawlers like search engines may face similar friction.",
    howToFix:
      'If you own this site, allowlist the "OpenSEO-Audit" user agent in your WAF/bot-protection settings (on Cloudflare: a WAF custom rule that skips bot protection when the user agent contains "OpenSEO-Audit"; on some free tiers you may need to relax bot protection). Then re-run the audit.',
  },
+  "rate-limited-page": {
+    severity: "warning",
+    title: "Rate limited (429)",
+    explanation:
+      "The server answered 429 Too Many Requests, so this page could not be audited. The crawler waits before retrying when the site's cooldown fits within the audit time limit.",
+    howToFix:
+      'Raise the rate limit for crawlers, or allowlist the "OpenSEO-Audit" user agent in your rate-limiting rules (on Cloudflare: a rate-limiting rule exception matching that user agent). Then re-run the audit. Re-running with fewer pages also helps if the limit is strict.',
+  },
+  "crawl-rate-limited": {
+    severity: "warning",
+    title: "Crawl stopped early: rate limit",
+    explanation:
+      "The site asked the crawler to wait longer than the audit time limit allowed. We stopped requesting pages. This report is incomplete; URLs we did not fetch are not recorded as broken or rate limited.",
+    howToFix:
+      "Re-run the audit after the site's rate limit resets, or ask the site owner to allow the OpenSEO-Audit crawler.",
+  },
  "server-error": {
    severity: "critical",
    title: "Server error (5xx)",