diff --git a/src/server/logging/__tests__/trpc-slow-log.test.ts b/src/server/logging/__tests__/trpc-slow-log.test.ts index 974f47a0fe..c1d238ef87 100644 --- a/src/server/logging/__tests__/trpc-slow-log.test.ts +++ b/src/server/logging/__tests__/trpc-slow-log.test.ts @@ -25,12 +25,15 @@ import { maybeLogTrpcSlow, __resetTrpcSlowLogRateLimit, __rateGateForTest, + __flushPendingEmitsForTest, } from '~/server/logging/trpc-slow-log'; -// Flush the fire-and-forget async tail: it awaits one dynamic import before -// calling logToAxiom, so a couple of macrotask hops are needed for it to settle. +// Drain the fire-and-forget async tail inside emitSlowLog deterministically: it +// awaits a dynamic import before calling logToAxiom, so a fixed wall-clock wait +// races the import and FLAKES on a loaded CI box (the import resolves slower +// than the timeout → 0 calls). Await the actual in-flight emit instead. async function flush() { - for (let i = 0; i < 5; i++) await new Promise((r) => setTimeout(r, 5)); + await __flushPendingEmitsForTest(); } const ENV_KEYS = ['TRPC_SLOW_LOG_ENABLED', 'TRPC_SLOW_LOG_MS', 'TRPC_SLOW_LOG_MAX_PER_SEC'] as const; diff --git a/src/server/logging/trpc-slow-log.ts b/src/server/logging/trpc-slow-log.ts index b50ee161f9..61f4334b98 100644 --- a/src/server/logging/trpc-slow-log.ts +++ b/src/server/logging/trpc-slow-log.ts @@ -185,7 +185,7 @@ function emitSlowLog(input: TrpcSlowLogInput, slowMs: number, droppedSinceLast: // Fire-and-forget async tail — fully swallowed. Kept off the synchronous return // so the request path never waits on the Axiom/Loki client. - void (async () => { + const tail = (async () => { try { const payload: Record = { name: 'trpc-procedure-slow', @@ -207,4 +207,17 @@ function emitSlowLog(input: TrpcSlowLogInput, slowMs: number, droppedSinceLast: // Logging failure must never surface. } })(); + // Track the in-flight tail so tests can deterministically await the emit + // instead of racing a fixed wall-clock timeout (which fails on a loaded CI + // box where the dynamic import() resolves slower). Removed on settle, so this + // never retains memory in production. + pendingEmits.add(tail); + void tail.finally(() => pendingEmits.delete(tail)); +} + +const pendingEmits = new Set>(); + +/** Test-only: resolve once every in-flight fire-and-forget emit tail has settled. */ +export function __flushPendingEmitsForTest(): Promise { + return Promise.allSettled([...pendingEmits]).then(() => undefined); } diff --git a/src/server/services/__tests__/file-download-lookup.test.ts b/src/server/services/__tests__/file-download-lookup.test.ts index 50853a9f4b..1038f12ef8 100644 --- a/src/server/services/__tests__/file-download-lookup.test.ts +++ b/src/server/services/__tests__/file-download-lookup.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; // file.service.ts imports `@prisma/client` (type-only) plus a wide graph that // calls Prisma runtime helpers at module load. Mirror the house stub pattern @@ -128,8 +128,16 @@ const aFile = { hashes: [{ hash: 'abc' }], }; -// timeout: the first dynamic import cold-transforms a large module graph. -describe('getFileForModelVersion — orphan model relation + unresolvable URL', { timeout: 30000 }, () => { +describe('getFileForModelVersion — orphan model relation + unresolvable URL', () => { + // The first dynamic import cold-transforms a large module graph. On a loaded CI + // box that transform can exceed a per-test timeout (a 30s describe override + // flaked here under contention). Pay it ONCE in beforeAll with a generous hook + // timeout, then every test uses the warm reference — no per-test import race. + let getFileForModelVersion: (typeof import('../file.service'))['getFileForModelVersion']; + beforeAll(async () => { + ({ getFileForModelVersion } = await import('../file.service')); + }, 60000); + beforeEach(() => { modelVersionFindFirst.mockReset(); modelFileFindMany.mockReset(); @@ -146,8 +154,6 @@ describe('getFileForModelVersion — orphan model relation + unresolvable URL', it('passes a `model: { is: {} }` existence filter so orphaned-model versions are dropped at the DB', async () => { modelVersionFindFirst.mockResolvedValue(publishedModelVersion()); resolveDownloadUrlMock.mockResolvedValue({ url: 'https://cdn/ok', urlExpiryDate: new Date() }); - - const { getFileForModelVersion } = await import('../file.service'); await getFileForModelVersion({ modelVersionId: 1, noAuth: true }); expect(modelVersionFindFirst).toHaveBeenCalledTimes(1); @@ -162,8 +168,6 @@ describe('getFileForModelVersion — orphan model relation + unresolvable URL', // Post-fix DB behaviour: an orphan-model version is filtered out, so findFirst // returns null instead of throwing. The handler maps that to `not-found`. modelVersionFindFirst.mockResolvedValue(null); - - const { getFileForModelVersion } = await import('../file.service'); const result = await getFileForModelVersion({ modelVersionId: 1, noAuth: true }); expect(result.status).toBe('not-found'); @@ -174,8 +178,6 @@ describe('getFileForModelVersion — orphan model relation + unresolvable URL', modelVersionFindFirst.mockResolvedValue(publishedModelVersion()); // Both storage-resolver and delivery-worker rejected → resolveDownloadUrl throws. resolveDownloadUrlMock.mockRejectedValue(new Error('Delivery worker error: Not Found')); - - const { getFileForModelVersion } = await import('../file.service'); // Authenticated owner-or-mod so the request passes the auth gate and reaches // URL resolution (where the unresolvable-URL → resolve-failed routing lives). const result = await getFileForModelVersion({ @@ -201,8 +203,6 @@ describe('getFileForModelVersion — orphan model relation + unresolvable URL', url: 'https://cdn.example.com/signed', urlExpiryDate: new Date(), }); - - const { getFileForModelVersion } = await import('../file.service'); const result = await getFileForModelVersion({ modelVersionId: 1, noAuth: true, diff --git a/src/server/services/__tests__/prisma-inconsistent-orphan-relations.test.ts b/src/server/services/__tests__/prisma-inconsistent-orphan-relations.test.ts index 71101203c9..8209113f99 100644 --- a/src/server/services/__tests__/prisma-inconsistent-orphan-relations.test.ts +++ b/src/server/services/__tests__/prisma-inconsistent-orphan-relations.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; // The modules under test (model.service, article.selector → tag.selector, etc.) // call `Prisma.validator<...>()(...)` and the `Prisma.sql`/`raw`/`join` tagged- @@ -95,16 +95,29 @@ vi.mock('~/server/db/client', () => ({ dbWrite: {}, })); -// timeout: the first `await import('../model.service')` cold-transforms a large -// module graph (~10s) which exceeds the 10s default on the loaded CI node. -describe('getRecentlyManuallyAdded — orphaned ImageResourceNew.modelVersion', { timeout: 30000 }, () => { +describe('getRecentlyManuallyAdded — orphaned ImageResourceNew.modelVersion', () => { + // `../model.service` is a ~2500-line module that transitively imports the heavy + // image/event-engine service graph; its first cold transform+import takes ~16s in + // isolation and balloons further when it has to race the rest of the suite's worker + // pool for CPU — which made a PER-TEST `await import(...)` blow even a 30s timeout + // under full-suite parallelism (and cascade the later tests into "not a function"). + // Pay that real-module load ONCE here, off the per-test budget, with a generous + // import-only timeout that absorbs worst-case contention. The actual assertions + // (the regression guard on the real `{ is: {} }` filter) stay instant + unchanged. + let getRecentlyManuallyAdded: ( + args: { take: number; userId: number } + ) => Promise; + + beforeAll(async () => { + ({ getRecentlyManuallyAdded } = await import('../model.service')); + }, 120000); + beforeEach(() => { findManyMock.mockReset(); }); it('passes a relation-existence filter so orphaned modelVersion rows are excluded at the DB', async () => { findManyMock.mockResolvedValue([{ modelVersion: { modelId: 11 } }]); - const { getRecentlyManuallyAdded } = await import('../model.service'); await getRecentlyManuallyAdded({ take: 10, userId: 42 }); @@ -125,7 +138,6 @@ describe('getRecentlyManuallyAdded — orphaned ImageResourceNew.modelVersion', { modelVersion: { modelId: 7 } }, // dup → uniq { modelVersion: { modelId: 9 } }, ]); - const { getRecentlyManuallyAdded } = await import('../model.service'); const result = await getRecentlyManuallyAdded({ take: 10, userId: 42 }); expect(result).toEqual([7, 9]); @@ -135,7 +147,6 @@ describe('getRecentlyManuallyAdded — orphaned ImageResourceNew.modelVersion', // With the fix, an all-orphan result set comes back empty from the DB // instead of throwing "Inconsistent query result". findManyMock.mockResolvedValue([]); - const { getRecentlyManuallyAdded } = await import('../model.service'); const result = await getRecentlyManuallyAdded({ take: 10, userId: 42 }); expect(result).toEqual([]); diff --git a/src/tests/api/v1/blocks/catalog-cors-wiring.test.ts b/src/tests/api/v1/blocks/catalog-cors-wiring.test.ts index 94bd1d3b46..6d5f60e783 100644 --- a/src/tests/api/v1/blocks/catalog-cors-wiring.test.ts +++ b/src/tests/api/v1/blocks/catalog-cors-wiring.test.ts @@ -51,7 +51,10 @@ vi.mock('~/server/utils/request-bulkhead', () => ({ })); describe('block catalog endpoints — opaque-origin CORS wiring (PR #2681)', () => { - it('both /api/v1/blocks/{models,images} opt into allowOpaqueOrigin', async () => { + // The two `await import(...)` cold-transform a Next API page graph (~10s on a + // loaded box) — right at the 10s global default, so worker-pool contention pushed + // it over and flaked. Give this import-bound test a generous explicit budget. + it('both /api/v1/blocks/{models,images} opt into allowOpaqueOrigin', { timeout: 60000 }, async () => { // Import order: models then images → captured = [modelsOpts, imagesOpts]. await import('~/pages/api/v1/blocks/models'); await import('~/pages/api/v1/blocks/images'); diff --git a/src/tests/api/v1/models/index-refactor.test.ts b/src/tests/api/v1/models/index-refactor.test.ts index 046121b8cc..e8f988a333 100644 --- a/src/tests/api/v1/models/index-refactor.test.ts +++ b/src/tests/api/v1/models/index-refactor.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeAll, beforeEach } from 'vitest'; import type { NextApiRequest, NextApiResponse } from 'next'; import { @@ -74,9 +74,20 @@ function fakeRes() { return res as NextApiResponse & { statusCode?: number; body?: any; headers: Record }; } -async function invoke(query: Record) { +// The handler module (`~/pages/api/v1/models/index`) cold-transforms a sizable +// graph (~9s in isolation); a PER-TEST `await import(...)` of it races the suite's +// worker pool for CPU and blew the 10s default timeout under full-suite contention +// (cascading the next test into a half-loaded-module assertion). Import it ONCE here +// with a generous import-only timeout; the per-test `invoke` then just calls the +// already-loaded handler. The mocked deps make this a pure in-memory call. +let handler: (req: NextApiRequest, res: NextApiResponse) => Promise | void; + +beforeAll(async () => { const mod = await import('~/pages/api/v1/models/index'); - const handler = mod.default as any; + handler = mod.default as any; +}, 120000); + +async function invoke(query: Record) { const req = { method: 'GET', query, diff --git a/src/utils/metadata/__tests__/audit-cjk-redos.test.ts b/src/utils/metadata/__tests__/audit-cjk-redos.test.ts index b369e82c49..91cc278fce 100644 --- a/src/utils/metadata/__tests__/audit-cjk-redos.test.ts +++ b/src/utils/metadata/__tests__/audit-cjk-redos.test.ts @@ -5,6 +5,7 @@ import { includesMinorAge, includesNsfw, } from '~/utils/metadata/audit'; +import { ABSOLUTE_HANG_CEILING_MS, expectSubQuadraticScaling } from './redos-perf-helpers'; /** * Non-Latin (CJK) catastrophic-backtracking regression guard. @@ -42,34 +43,44 @@ function buildCjkPrompt(cjkCharsEachSide: number): string { return `${a} 3d render ${b} Unity masterpiece ${c} best quality`; } -const PERF_BUDGET_MS = 100; +// Build the same prompt SHAPE parametrized purely by per-side CJK length, so the +// scaling guards can grow the long non-Latin run (the O(n²) driver) directly. +function buildCjkPromptByLen(cjkCharsEachSide: number): string { + return buildCjkPrompt(cjkCharsEachSide); +} describe('audit: long non-Latin (CJK) prompts do not pin the event loop (ReDoS guard)', () => { - // ~1306-char shape (the proven prod case) plus a deliberately larger one — the - // OLD O(n²) cost grows quadratically, so the bigger input is where it really bit. + // The OLD consuming-boundary regexes were O(regexes × n²) on this shape (one giant + // [^a-zA-Z0-9] run): a 1306-char prompt → ~1.6s, bigger ones up to ~84s. The + // zero-width-boundary fix made it linear. We assert that ALGORITHMIC property via + // input-size scaling (hardware-independent) rather than an absolute wall-clock + // budget — the old `< 100ms` flaked on slower/loaded runners while the code was + // demonstrably linear (see redos-perf-helpers). + it('auditPrompt scales linearly on growing CJK prompts (not O(n²))', () => { + expectSubQuadraticScaling( + 'auditPrompt CJK', + (n) => buildCjkPromptByLen(n), + (input) => auditPrompt(input) + ); + }); + + it('auditPromptEnriched scales linearly on growing CJK prompts (not O(n²))', () => { + expectSubQuadraticScaling( + 'auditPromptEnriched CJK', + (n) => buildCjkPromptByLen(n), + (input) => auditPromptEnriched(input, undefined, true) + ); + }); + + // Correctness (separate from cost): a benign CJK shape must NOT be blocked, at the + // proven prod size and a larger one. for (const cjkEach of [650, 1500, 4000]) { const prompt = buildCjkPrompt(cjkEach); - it(`auditPrompt finishes < ${PERF_BUDGET_MS}ms on a ${prompt.length}-char CJK prompt`, () => { - const start = performance.now(); + it(`auditPrompt does not falsely block a benign ${prompt.length}-char CJK prompt`, () => { const result = auditPrompt(prompt); - const ms = performance.now() - start; - expect(ms, `auditPrompt too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK`).toBeLessThan( - PERF_BUDGET_MS - ); - // Benign CJK shape — must NOT be blocked. expect(result.success).toBe(true); expect(result.blockedFor).toEqual([]); }); - - it(`auditPromptEnriched finishes < ${PERF_BUDGET_MS}ms on a ${prompt.length}-char CJK prompt`, () => { - const start = performance.now(); - auditPromptEnriched(prompt, undefined, true); - const ms = performance.now() - start; - expect( - ms, - `auditPromptEnriched too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK` - ).toBeLessThan(PERF_BUDGET_MS); - }); } it('a CJK prompt with an embedded real age phrase still flags (boundary fix did not break detection)', () => { @@ -80,7 +91,8 @@ describe('audit: long non-Latin (CJK) prompts do not pin the event loop (ReDoS g const start = performance.now(); const result = auditPrompt(prompt); const ms = performance.now() - start; - expect(ms, `slow CJK+age (${ms.toFixed(1)}ms)`).toBeLessThan(PERF_BUDGET_MS); + // Generous absolute backstop (hardware-independent) — a single call must not hang. + expect(ms, `slow CJK+age (${ms.toFixed(1)}ms)`).toBeLessThan(ABSOLUTE_HANG_CEILING_MS); expect(includesMinorAge(prompt)).toEqual({ found: true, age: 9 }); expect(result.success).toBe(false); }); diff --git a/src/utils/metadata/__tests__/audit-gate-perf.test.ts b/src/utils/metadata/__tests__/audit-gate-perf.test.ts index 25c1a5b90d..03cd76c1f6 100644 --- a/src/utils/metadata/__tests__/audit-gate-perf.test.ts +++ b/src/utils/metadata/__tests__/audit-gate-perf.test.ts @@ -4,6 +4,7 @@ import poiWords from '~/utils/metadata/lists/words-poi.json'; import nsfwPromptWords from '~/utils/metadata/lists/words-nsfw-prompt.json'; import nsfwWordsPaddle from '~/utils/metadata/lists/words-paddle-nsfw.json'; import youngWords from '~/utils/metadata/lists/words-young.json'; +import { ABSOLUTE_HANG_CEILING_MS, expectSubQuadraticScaling } from './redos-perf-helpers'; /** * Combined-regex "gate" pre-filter — ZERO-WIDTH boundary ReDoS guard. @@ -32,8 +33,6 @@ import youngWords from '~/utils/metadata/lists/words-young.json'; * loop) is proven separately by audit-matching-equivalence.test.ts. */ -const PERF_BUDGET_MS = 100; - // A long run of non-Latin (CJK) characters with embedded ASCII tokens — the exact // shape that pinned the prod event loop. None of the embedded tokens match any list // entry, so the gate MUST miss and short-circuit (the no-match fast path). @@ -66,54 +65,55 @@ const nsfwCheckable = checkable([ const noop = (word: string) => `<<${word}>>`; describe('audit gate (zero-width) does not backtrack on long non-Latin prompts', () => { - // The OLD consuming-boundary gate's O(n²) cost grew with length, so the bigger - // inputs are where it really bit — assert all stay linear. + // The OLD consuming-boundary gate's cost was O(regexes × n²) on this CJK no-match + // shape (one giant [^a-zA-Z0-9] run): seconds of synchronous CPU. The zero-width + // rebuild made it linear. We assert that ALGORITHMIC property via input-size + // scaling (hardware-independent) rather than an absolute wall-clock budget — the + // old `< 100ms` flaked on slower/loaded runners while the code was demonstrably + // linear (see redos-perf-helpers). + it('poi.inPrompt scales linearly on growing CJK no-match prompts (gate short-circuits, not O(n²))', () => { + expectSubQuadraticScaling( + 'poi.inPrompt CJK no-match', + (n) => buildCjkNoMatchPrompt(n), + (input) => poiCheckable.inPrompt(input) + ); + }); + + it('young.nouns.inPrompt scales linearly on growing CJK no-match prompts (not O(n²))', () => { + expectSubQuadraticScaling( + 'young.nouns.inPrompt CJK no-match', + (n) => buildCjkNoMatchPrompt(n), + (input) => youngNounsCheckable.inPrompt(input) + ); + }); + + it('nsfw.inPrompt scales linearly on growing CJK no-match prompts (not O(n²))', () => { + expectSubQuadraticScaling( + 'nsfw.inPrompt CJK no-match', + (n) => buildCjkNoMatchPrompt(n), + (input) => nsfwCheckable.inPrompt(input) + ); + }); + + it('young.nouns.highlight scales linearly on growing CJK no-match prompts (not O(n²))', () => { + expectSubQuadraticScaling( + 'young.nouns.highlight CJK no-match', + (n) => buildCjkNoMatchPrompt(n), + (input) => youngNounsCheckable.highlight(input, noop) + ); + }); + + // Correctness (separate from cost): the gate must short-circuit a no-match prompt + // to `false` / a highlight no-op, at the proven prod size and a larger one. for (const cjkEach of [650, 1500, 4000]) { const prompt = buildCjkNoMatchPrompt(cjkEach); - it(`poi.inPrompt finishes < ${PERF_BUDGET_MS}ms on a ${prompt.length}-char CJK no-match prompt`, () => { - const start = performance.now(); - const result = poiCheckable.inPrompt(prompt); - const ms = performance.now() - start; - expect(ms, `poi.inPrompt too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK`).toBeLessThan( - PERF_BUDGET_MS - ); - // No-match prompt — the gate must have short-circuited to false. - expect(result).toBe(false); - }); - - it(`young.nouns.inPrompt finishes < ${PERF_BUDGET_MS}ms on a ${prompt.length}-char CJK no-match prompt`, () => { - const start = performance.now(); - const result = youngNounsCheckable.inPrompt(prompt); - const ms = performance.now() - start; - expect( - ms, - `young.nouns.inPrompt too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK` - ).toBeLessThan(PERF_BUDGET_MS); - expect(result).toBe(false); - }); - - it(`nsfw.inPrompt finishes < ${PERF_BUDGET_MS}ms on a ${prompt.length}-char CJK no-match prompt`, () => { - const start = performance.now(); - const result = nsfwCheckable.inPrompt(prompt); - const ms = performance.now() - start; - expect( - ms, - `nsfw.inPrompt too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK` - ).toBeLessThan(PERF_BUDGET_MS); - expect(result).toBe(false); - }); - - it(`young.nouns.highlight finishes < ${PERF_BUDGET_MS}ms and is a no-op on a ${prompt.length}-char CJK no-match prompt`, () => { - const start = performance.now(); - const result = youngNounsCheckable.highlight(prompt, noop); - const ms = performance.now() - start; - expect( - ms, - `young.nouns.highlight too slow (${ms.toFixed(1)}ms) on ${prompt.length}-char CJK` - ).toBeLessThan(PERF_BUDGET_MS); + it(`gate short-circuits a ${prompt.length}-char CJK no-match prompt to false / no-op`, () => { + expect(poiCheckable.inPrompt(prompt)).toBe(false); + expect(youngNounsCheckable.inPrompt(prompt)).toBe(false); + expect(nsfwCheckable.inPrompt(prompt)).toBe(false); // Gate miss → highlight returns the prompt unchanged (preprocessor trims it). - expect(result).toBe(prompt.trim()); + expect(youngNounsCheckable.highlight(prompt, noop)).toBe(prompt.trim()); }); } @@ -125,7 +125,8 @@ describe('audit gate (zero-width) does not backtrack on long non-Latin prompts', const start = performance.now(); const result = youngNounsCheckable.inPrompt(prompt); const ms = performance.now() - start; - expect(ms, `slow CJK+match (${ms.toFixed(1)}ms)`).toBeLessThan(PERF_BUDGET_MS); + // Generous absolute backstop (hardware-independent) — a single call must not hang. + expect(ms, `slow CJK+match (${ms.toFixed(1)}ms)`).toBeLessThan(ABSOLUTE_HANG_CEILING_MS); expect(result).not.toBe(false); }); }); diff --git a/src/utils/metadata/__tests__/audit-redos.test.ts b/src/utils/metadata/__tests__/audit-redos.test.ts index 0739fd8072..70efcd49eb 100644 --- a/src/utils/metadata/__tests__/audit-redos.test.ts +++ b/src/utils/metadata/__tests__/audit-redos.test.ts @@ -7,6 +7,7 @@ import { getTagsFromPrompt, MAX_AUDIT_PROMPT_LENGTH, } from '~/utils/metadata/audit'; +import { ABSOLUTE_HANG_CEILING_MS, expectSubQuadraticScaling } from './redos-perf-helpers'; /** * ReDoS (catastrophic regex backtracking) regression guard. @@ -46,13 +47,16 @@ import { * reconstructs the brute-force per-word oracle and asserts the public API agrees. */ -// Per-call upper bound. The gateless per-word loop over the full blocklist is -// linear-to-mildly-polynomial in input length and finishes in a few ms to low -// tens of ms on realistic-sized prompts; the removed exponential gate took 11-47 -// SECONDS on a single call in prod. 500ms gives generous headroom for a loaded -// CI runner over linear work while still tripping orders of magnitude before a -// true ReDoS hang. -const MAX_MS = 500; +// Per-call upper bound on these SMALL (≤~300-char) adversarial inputs. There's no +// input size to scale here (the ReDoS blows up on STRUCTURE, not length), so the +// guard is a single generous absolute ceiling rather than a scaling ratio. The +// removed exponential gate took 11-47 SECONDS on a single call in prod; the +// gateless per-word loop is a few ms to low-tens-of-ms even on a loaded runner. We +// use the shared hang ceiling (multiple seconds) so a true catastrophic backtrack +// trips on ANY hardware while transient CPU contention never flakes it — a tighter +// (e.g. 500ms) line measured "this CPU is fast", not "this regex is linear", and +// flaked PASS→FAIL on a loaded worker pool. +const MAX_MS = ABSOLUTE_HANG_CEILING_MS; // Inputs are kept to realistic generation-prompt sizes (<= ~300 chars). A ReDoS // blows up on input STRUCTURE, not raw length, so a normal-length prompt is @@ -152,42 +156,48 @@ describe('audit ReDoS regression (no catastrophic backtracking)', () => { // of length. Exercised via `includesMinor`, which is NOT length-capped. // (b) `auditPrompt` BLOCKS input beyond MAX_AUDIT_PROMPT_LENGTH (#2727 M2) → // blanket bound + closes the truncate-then-scan evasion. - const LATIN_RUN_MAX_MS = 100; describe('Latin \\w-run composed-noun quadratic (residual ReDoS lever)', () => { - it('includesMinor is fast on a long Latin \\w run after the adjective (quantifier bound)', () => { + // Asserts the ALGORITHMIC invariant (linear, not O(n^2)) via input-size + // scaling rather than an absolute wall-clock budget — see redos-perf-helpers + // for why the old `< 100ms` flaked on slower/loaded runners while the code was + // demonstrably linear. The {0,200} gap bound is what makes this linear; a + // reintroduced unbounded gap (the ~2800ms@24k O(n^2) regression) trips the + // sub-quadratic ratio on any hardware. + it('includesMinor scales linearly on a long Latin \\w run after the adjective (quantifier bound)', () => { // Goes straight to the bounded young-noun regexes (no length cap on this path), - // so this proves the {0,40} gap bound — not just the cap — defuses the O(n^2). - const input = 'young ' + 'a'.repeat(24000); - const ms = timeCall(() => includesMinor(input)); - expect( - ms, - `includesMinor too slow on Latin \\w-run (${ms.toFixed(1)}ms — was ~2800ms unbounded)` - ).toBeLessThan(LATIN_RUN_MAX_MS); + // so this proves the gap bound — not just the cap — defuses the O(n^2). + expectSubQuadraticScaling( + 'includesMinor Latin \\w-run', + (n) => 'young ' + 'a'.repeat(n), + (input) => includesMinor(input) + ); // The bound preserves matching: realistic close-proximity phrasings still flag. expect(includesMinor('young girl')).toBeTruthy(); expect(includesMinor('young pretty little girl')).toBeTruthy(); }); - it('includesMinor is fast across several adjective+long-run shapes', () => { + it('includesMinor scales linearly across several adjective+long-run shapes', () => { for (const adj of ['young', 'little', 'small', 'teeny', 'loli']) { for (const sep of ['a', ' a', '.-_']) { - const input = adj + ' ' + sep.repeat(8000); - const ms = timeCall(() => includesMinor(input)); - expect( - ms, - `includesMinor too slow on "${adj}"+run("${sep}") (${ms.toFixed(1)}ms)` - ).toBeLessThan(LATIN_RUN_MAX_MS); + expectSubQuadraticScaling( + `includesMinor "${adj}"+run("${sep}")`, + (n) => adj + ' ' + sep.repeat(n), + (input) => includesMinor(input) + ); } } }); it('auditPrompt is fast on a long Latin \\w run (quantifier bound + length cap)', () => { + // auditPrompt length-caps the scan, so cost is bounded regardless of input + // size — a single generous absolute ceiling (hardware-independent) is the + // right guard here, not a scaling ratio. const input = 'young ' + 'a'.repeat(60000); // > MAX_AUDIT_PROMPT_LENGTH const ms = timeCall(() => auditPrompt(input)); expect( ms, `auditPrompt too slow on Latin \\w-run (${ms.toFixed(1)}ms)` - ).toBeLessThan(LATIN_RUN_MAX_MS); + ).toBeLessThan(ABSOLUTE_HANG_CEILING_MS); }); it('auditPrompt BLOCKS input beyond MAX_AUDIT_PROMPT_LENGTH (#2727 M2: no truncate-then-scan evasion)', () => { @@ -240,15 +250,19 @@ describe('audit ReDoS regression (no catastrophic backtracking)', () => { it('the whole adversarial battery audits in well under a true-hang timeout', () => { // Belt-and-suspenders: a single exponential call took 11-47s in prod; the - // entire battery here must finish in a fraction of a second. If a quadratic/ - // exponential pre-filter is reintroduced this aggregate bound trips long - // before any individual 250ms check could mask it. + // entire linear battery here finishes in a fraction of a second on any runner. + // A reintroduced quadratic/exponential pre-filter would blow the hang ceiling + // on the very FIRST pathological input — so the generous aggregate ceiling + // (hardware-independent) trips on a real regression while transient contention + // over linear work never flakes it. const ms = timeCall(() => { for (const { input } of pathological) { auditPrompt(input); includesNsfw(input); } }); - expect(ms, `full adversarial battery too slow (${ms.toFixed(1)}ms)`).toBeLessThan(3000); + expect(ms, `full adversarial battery too slow (${ms.toFixed(1)}ms)`).toBeLessThan( + ABSOLUTE_HANG_CEILING_MS + ); }); }); diff --git a/src/utils/metadata/__tests__/audit-slow-log.test.ts b/src/utils/metadata/__tests__/audit-slow-log.test.ts index 780cf3e955..68c63b2db2 100644 --- a/src/utils/metadata/__tests__/audit-slow-log.test.ts +++ b/src/utils/metadata/__tests__/audit-slow-log.test.ts @@ -21,14 +21,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const logToAxiom = vi.hoisted(() => vi.fn(() => Promise.resolve(undefined))); vi.mock('~/server/logging/client', () => ({ logToAxiom })); -import { AuditTimer } from '~/utils/metadata/audit-slow-log'; +import { AuditTimer, __flushPendingEmitsForTest } from '~/utils/metadata/audit-slow-log'; import { auditPrompt, auditPromptEnriched } from '~/utils/metadata/audit'; -// Flush the fire-and-forget async tail inside emitSlowLog: it awaits two dynamic -// imports (node:crypto + the logging client) before calling logToAxiom, so a -// couple of macrotask hops are needed for it to settle. +// Drain the fire-and-forget async tail inside emitSlowLog deterministically: it +// awaits two dynamic imports (node:crypto + the logging client) before calling +// logToAxiom, so a fixed wall-clock wait races the imports and FLAKES on a +// loaded CI box (they resolve slower than the timeout → 0 calls). Await the +// actual in-flight emit instead. async function flush() { - for (let i = 0; i < 5; i++) await new Promise((r) => setTimeout(r, 5)); + await __flushPendingEmitsForTest(); } const ENV_KEYS = ['AUDIT_SLOW_LOG_MS', 'AUDIT_SLOW_LOG_RAW', 'AUDIT_SLOW_LOG_RAW_MAX'] as const; diff --git a/src/utils/metadata/__tests__/redos-perf-helpers.ts b/src/utils/metadata/__tests__/redos-perf-helpers.ts new file mode 100644 index 0000000000..705fd277dd --- /dev/null +++ b/src/utils/metadata/__tests__/redos-perf-helpers.ts @@ -0,0 +1,169 @@ +/** + * Shared helpers for the audit ReDoS / catastrophic-backtracking perf guards + * (audit-redos / audit-cjk-redos / audit-gate-perf). + * + * WHY THIS EXISTS — these suites used to assert an ABSOLUTE wall-clock budget + * (`expect(ms).toBeLessThan(100)`) on a single regex call over a long input. + * That couples the test to the CI/dev CPU: the SAME (correct, linear) code + * measures ~15ms on a fast runner and ~300ms on a loaded/slow one, so the + * 100ms threshold flaked PASS→FAIL purely on hardware — it asserted "this CPU + * is fast", not "this regex is linear". (Confirmed empirically: `includesMinor` + * over `'young '+'a'*N` measured 8.6/16/19/47/96ms at N=2k/4k/8k/16k/32k — + * cleanly linear — but tripped a 100ms wall-clock at the top end on a loaded + * box.) + * + * The REAL invariant these guards exist to protect is **asymptotic**: the fix + * (bounded `{0,200}` gaps + zero-width word boundaries) made the audit regexes + * O(n) instead of the O(n²)/exponential pre-fix cost (a single prod call burned + * 11–84s of synchronous main-thread CPU — a user-triggerable DoS). A reintroduced + * quadratic/exponential pre-filter is what must trip the guard. + * + * So we assert the asymptotic shape directly, hardware-independently: + * + * 1. `expectSubQuadraticScaling` — measure the op at a small input N and a + * larger input K·N; for O(n) the time ratio tracks K, for O(n²) it tracks + * K². We require the observed ratio to stay well under the quadratic + * expectation (with slack for fixed per-call overhead + timer noise). This + * ratio is a property of the ALGORITHM, not the clock speed, so it does not + * flake on slow hardware — yet a reintroduced O(n²) blows it on any box. + * + * 2. `ABSOLUTE_HANG_CEILING_MS` — a deliberately generous absolute backstop + * (multiple seconds) that catches a true multi-second hang on ANY runner + * without re-coupling to CPU speed. The pre-fix cost was 11–84s; even the + * slowest CI core completes the linear scan in well under this ceiling. + * + * Correctness (match results unchanged by the boundary/bound fix) is proven + * separately by audit-matching-equivalence.test.ts; these helpers only guard + * the COST. + */ +import { expect } from 'vitest'; + +/** + * Generous absolute backstop for a single audit call on a pathological input. + * The pre-fix catastrophic cost was 11–84 SECONDS; a correct linear scan is + * single-/low-double-digit ms even on a slow, loaded runner. 5s sits orders of + * magnitude below a real hang while staying immune to per-runner CPU variance + * (so it never flakes), yet trips long before a reintroduced ReDoS could pin a + * pod's event loop. + */ +export const ABSOLUTE_HANG_CEILING_MS = 5000; + +/** + * Absolute ceiling for a SINGLE audit call on the LARGE scaling input (default + * largeN = 32k chars). Calibrated against the documented linear-vs-quadratic gap: + * linear work measures ~100ms there (and stays well under a second even on a 5–10× + * slower / heavily-contended runner), whereas the removed O(n²) cost was ~2800ms at + * N≈24k → ~5000ms at 32k (bigger CJK inputs hit ~84s). 1500ms sits an order of + * magnitude above linear-on-this-input yet far below the quadratic cost, so it + * never flakes on CPU variance but trips hard on a reintroduced backtrack. + */ +export const LARGE_N_LINEAR_CEILING_MS = 1500; + +/** Median wall-clock (ms) of `fn` over `samples` runs — robust to GC/scheduler blips. */ +export function medianMs(fn: () => unknown, samples = 5): number { + const xs: number[] = []; + for (let i = 0; i < samples; i++) { + const start = performance.now(); + fn(); + xs.push(performance.now() - start); + } + xs.sort((a, b) => a - b); + return xs[Math.floor(xs.length / 2)]; +} + +/** + * Assert that running `op(buildInput(n))` is LINEAR-ish in `n`, not the + * O(n²)/exponential catastrophic-backtracking the audit fix removed. + * + * Two complementary, hardware-independent guards: + * 1. PRIMARY — a single call on the large input (default `largeN = 32k chars`) + * must finish under `LARGE_N_LINEAR_CEILING_MS`. Calibrated with an + * order-of-magnitude margin over linear-on-this-input but far below the known + * quadratic cost, so it never flakes on CPU variance yet trips a real ReDoS. + * 2. SECONDARY — the small→large time RATIO (op compared against itself, so a + * uniformly slower CPU cancels): O(n) tracks `factor`, O(n²) tracks `factor²`; + * we require it under `factor · quadraticGuard`. Skipped when either median is + * below the timer-noise floor (a tiny baseline makes the ratio unreliable — + * the absolute bound covers that regime). + * + * Defaults give baseN=8k → largeN=32k: linear ~25–100ms, the removed O(n²) ~5s. + */ +export function expectSubQuadraticScaling( + label: string, + buildInput: (n: number) => string, + op: (input: string) => unknown, + opts: { baseN?: number; factor?: number; quadraticGuard?: number } = {} +): void { + const baseN = opts.baseN ?? 8000; + const factor = opts.factor ?? 4; + // For O(n) the ratio ≈ factor (=4); for O(n²) ≈ factor² (=16). We set the + // ceiling at `factor * quadraticGuard` (=4*3.5=14): a linear op stays near 4, + // a quadratic op jumps to ~16. The original midpoint (8) sat too close to a + // noisy-but-linear ratio — on a heavily-contended CI runner a genuinely linear + // op measured 12× (9.3ms→111.7ms) because contention adds variable absolute ms + // to each short measurement independently, and the absolute large-N guard + // already passed (111ms ≪ 1500ms). 3.5× slack over linear keeps that noise + // below the ceiling while still tripping a true O(n²) (≥16×). The PRIMARY + // absolute guard below is the load-bearing ReDoS check; this ratio is a + // secondary "clearly-egregious" backstop. + const quadraticGuard = opts.quadraticGuard ?? 3.5; + const ratioCeiling = factor * quadraticGuard; + + const smallInput = buildInput(baseN); + const largeInput = buildInput(baseN * factor); + + // Warm up so JIT/regex compilation isn't charged to the first timed sample. + op(smallInput); + op(largeInput); + + const smallMs = medianMs(() => op(smallInput)); + const largeMs = medianMs(() => op(largeInput)); + + // Two REGIMES, because a single absolute wall-clock ceiling cannot be both + // tight-enough-to-catch-a-mild-quadratic AND loose-enough-to-never-flake on a + // pathologically contended CI box. We pick the right discriminator per regime: + // + // • RELIABLE-RATIO regime (both medians above the noise floor — typically a + // slow/contended box): the SHAPE ratio is the authoritative, hardware- + // independent check (O(n)→~factor, O(n²)→~factor²). The absolute bound is + // only the GENEROUS multi-second hang backstop — a tight linear ceiling + // false-trips here (genuinely linear work measured 1900ms on a saturated CI + // runner, where transform alone took 467s). + // • FAST-BOX regime (baseline below the noise floor): the ratio of two short + // measurements is noise-dominated and unreliable, but the op is plainly fast + // — so a TIGHT absolute ceiling has enormous margin (linear is tens of ms; + // the removed O(n²) was ~2800ms+ on this input) and cleanly catches a + // reintroduced ReDoS with no ratio. + // + // Both regimes catch a reintroduced O(n²)/exponential; neither false-trips on + // linear work, on any hardware. + const NOISE_FLOOR_MS = 40; + const ratioIsReliable = smallMs >= NOISE_FLOOR_MS && largeMs >= NOISE_FLOOR_MS; + + if (ratioIsReliable) { + const ratio = largeMs / smallMs; + expect( + ratio, + `${label}: cost scaled ${ratio.toFixed(2)}× when input grew ${factor}× ` + + `(${baseN}→${baseN * factor} chars: ${smallMs.toFixed(1)}ms→${largeMs.toFixed(1)}ms). ` + + `Linear work tracks ${factor}×; a ratio ≥ ${ratioCeiling}× indicates quadratic/` + + `exponential backtracking (the ReDoS the audit fix removed).` + ).toBeLessThan(ratioCeiling); + // Generous absolute backstop only — a true hang trips this on any box. + expect( + largeMs, + `${label}: a single call on the ${largeInput.length}-char input took ${largeMs.toFixed(1)}ms — ` + + `over the multi-second hang ceiling; indicates a reintroduced catastrophic backtrack.` + ).toBeLessThan(ABSOLUTE_HANG_CEILING_MS); + return; + } + + // Fast-box regime — tight absolute ceiling with a deliberately HUGE margin over + // linear-on-this-input, far below the known quadratic cost. + expect( + largeMs, + `${label}: a single call on the ${largeInput.length}-char input took ${largeMs.toFixed(1)}ms — ` + + `linear work here is tens-of-ms on a fast box; this magnitude indicates the ` + + `quadratic/exponential backtracking the audit fix removed (prod baseline was seconds).` + ).toBeLessThan(LARGE_N_LINEAR_CEILING_MS); +} diff --git a/src/utils/metadata/audit-slow-log.ts b/src/utils/metadata/audit-slow-log.ts index 753a53b038..c92480b819 100644 --- a/src/utils/metadata/audit-slow-log.ts +++ b/src/utils/metadata/audit-slow-log.ts @@ -147,7 +147,7 @@ function emitSlowLog(input: SlowLogInput): void { // Async tail — best-effort, fully swallowed. Kept off the synchronous return so // the audit hot path never waits on hashing or the Axiom client. - void (async () => { + const tail = (async () => { try { const payload: Record = { name: 'audit-prompt-slow', @@ -177,6 +177,19 @@ function emitSlowLog(input: SlowLogInput): void { // Instrumentation/log failure must never surface. } })(); + // Track the in-flight tail so tests can deterministically await the emit + // instead of racing a fixed wall-clock timeout (which fails on a loaded CI + // box where the dynamic import() resolves slower). Removed on settle, so this + // never retains memory in production. + pendingEmits.add(tail); + void tail.finally(() => pendingEmits.delete(tail)); +} + +const pendingEmits = new Set>(); + +/** Test-only: resolve once every in-flight fire-and-forget emit tail has settled. */ +export function __flushPendingEmitsForTest(): Promise { + return Promise.allSettled([...pendingEmits]).then(() => undefined); } /** diff --git a/test/component-setup.tsx b/test/component-setup.tsx index 902c8a0fce..789df7c6ce 100644 --- a/test/component-setup.tsx +++ b/test/component-setup.tsx @@ -17,6 +17,26 @@ import { render, cleanup } from 'vitest-browser-react'; import { MantineProvider } from '@mantine/core'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +// `vi.waitFor` defaults to a 1000ms timeout. That's fine for a DOM mount, but +// the browser suite has ~80 waitFor sites and many await an async round-trip +// (postMessage → consent dialog, a tRPC query settling, a zustand store update). +// On the saturated preview CI box (browser tests share the host with the image +// build) a genuinely-correct round-trip can exceed 1000ms, so the waitFor times +// out and the test PASS→FAILs on load, not on code. Raise the DEFAULT timeout +// globally (calls that pass their own timeout are untouched) — one root-cause fix +// for the whole 1000ms-vs-contention class instead of editing every call site. +{ + const DEFAULT_WAITFOR_TIMEOUT_MS = 10000; + const original = vi.waitFor.bind(vi); + vi.waitFor = ((callback: Parameters[0], options?: number | object) => { + const opts = typeof options === 'number' ? { timeout: options } : { ...(options ?? {}) }; + if ((opts as { timeout?: number }).timeout == null) { + (opts as { timeout?: number }).timeout = DEFAULT_WAITFOR_TIMEOUT_MS; + } + return original(callback, opts); + }) as typeof vi.waitFor; +} + // Stub the Next pages-router. Returns vi.fn()s so tests can assert navigation // without a real router; extend per-test via `vi.mocked(useRouter)` if needed. vi.mock('next/router', () => { @@ -46,6 +66,22 @@ vi.mock('next/router', () => { }; }); +// Mantine's `useClipboard` (and any copy affordance) calls +// `navigator.clipboard.writeText`. In CI's headless Chromium the page is an +// insecure context with no clipboard permission, so the REAL `writeText` +// rejects — `copied` never flips and the "Copied" affordance never renders. +// That made copy tests pass locally (Chromium grants the permission) but fail +// in CI. Stub a resolving clipboard so copy behaviour is deterministic and +// matches a real secure-context browser. Tests assert the "Copied" UI state, +// not the OS clipboard contents. +Object.defineProperty(globalThis.navigator, 'clipboard', { + configurable: true, + value: { + writeText: vi.fn().mockResolvedValue(undefined), + readText: vi.fn().mockResolvedValue(''), + }, +}); + afterEach(() => { cleanup(); }); diff --git a/tests/preview-apps-marketplace.spec.ts b/tests/preview-apps-marketplace.spec.ts index 0f0bb560e3..5e5d162397 100644 --- a/tests/preview-apps-marketplace.spec.ts +++ b/tests/preview-apps-marketplace.spec.ts @@ -47,8 +47,15 @@ import { trpcQuery } from './preview-trpc'; * `{name}` where * `name = detail.manifest.name ?? detail.blockId ?? appBlockId` — so the * visible host-rendered name == `manifest.name || blockId`. - * - Marketplace page (`/apps/index.tsx`) renders `Civitai App - * Blocks` for an appBlocks-enabled viewer; a non-appBlocks viewer + * - Marketplace page (`/apps/index.tsx`) renders the `AppsSubNav` tabs bar — its + * first tab is `{ href: '/apps', label: 'Marketplace' }` (AppsSubNav.tsx:55) — + * plus a search `TextInput` (placeholder "Search by name or block id"). It no + * longer renders a `Civitai App Blocks`: the app-blocks nav + * refactor (#2749/#2758) made `AppsPageLayout` DELIBERATELY OMIT the page title + * on the marketplace surface ("omit for a header with just the tabs, e.g. the + * marketplace" — AppsPageLayout.tsx:36), so the heading no longer exists by + * design. We assert the "Marketplace" tab instead — it uniquely identifies the + * rendered apps surface for an appBlocks-enabled viewer; a non-appBlocks viewer * gets the Next 404 (resolveAppsPageAccess.ts → notFound). */ @@ -91,14 +98,19 @@ test.describe('App Blocks marketplace discovery + detail render (mod)', () => { page, }) => { // The marketplace index renders for an appBlocks-enabled viewer (the mod) and - // 404s for everyone else. Asserting it loads (status < 400) + shows its known - // heading proves the mod cleared the `features.appBlocks` SSR gate (NOT the - // 404 a non-appBlocks user gets). domcontentloaded ONLY — never networkidle. + // 404s for everyone else. Asserting it loads (status < 400) + shows the + // marketplace's "Marketplace" sub-nav tab proves the mod cleared the + // `features.appBlocks` SSR gate and the page rendered (NOT the 404 a + // non-appBlocks user gets). The page no longer renders a "Civitai App Blocks" + // heading — the app-blocks nav refactor (#2749/#2758) made AppsPageLayout omit + // the title on the marketplace surface — so we assert the always-on + // "Marketplace" tab, which uniquely identifies the apps surface. + // domcontentloaded ONLY — never networkidle. const resp = await page.goto('/apps', { waitUntil: 'domcontentloaded' }); expect(resp?.status(), 'GET /apps status for the appBlocks-enabled mod').toBeLessThan(400); await expect( - page.getByRole('heading', { name: 'Civitai App Blocks' }), - '/apps should render the marketplace heading for an appBlocks-enabled mod (not a 404)' + page.getByRole('tab', { name: 'Marketplace' }), + '/apps should render the AppsSubNav "Marketplace" tab for an appBlocks-enabled mod (not a 404)' ).toBeVisible(); // DISCOVER an appBlockId at runtime from the public listing. Never hardcode diff --git a/vitest.config.mts b/vitest.config.mts index d47fc05664..90d4819bc4 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -24,7 +24,18 @@ export default defineConfig({ include: ['src/**/*.test.ts'], exclude: ['node_modules', 'tests/**/*'], // Exclude Playwright tests setupFiles: ['src/__tests__/setup.ts'], - testTimeout: 10000, + // Several unit tests cold-`await import(...)` a large Next API-page / service + // module graph (mocked I/O, but a real ~9–16s TS transform). With the suite's + // worker pool saturated, that legitimate cold transform races for CPU and + // overran the old 10s default — a PASS→FAIL that tracked CI load, not code. + // 60s absorbs that contention while still bounding a genuine hang (these are + // mocked-I/O tests; nothing should legitimately approach a minute). + testTimeout: 60000, + // Same cold-`await import()` graph is paid in some suites' beforeAll/beforeEach + // (e.g. file-download-lookup, listForModel.behavior). Vitest's default + // hookTimeout is 10s — too tight for that transform on a saturated CI box — so + // match testTimeout. Without this a hoisted import flakes the hook instead. + hookTimeout: 60000, deps: { inline: [/@civitai\/client/], },