diff --git a/.claude/skills/dev-server/cli.mjs b/.claude/skills/dev-server/cli.mjs index c1a71faebd..0af1896f6c 100644 --- a/.claude/skills/dev-server/cli.mjs +++ b/.claude/skills/dev-server/cli.mjs @@ -663,6 +663,11 @@ async function cmdTest(sub, rest) { const inline = rest[typecheckAt].split('=')[1]; body.typecheckConcurrency = Number(inline !== undefined ? inline : rest[typecheckAt + 1]); } + const cacheAt = rest.findIndex((a) => /^--cache(=|$)/.test(a)); + if (cacheAt !== -1) { + const inline = rest[cacheAt].split('=')[1]; + body.cacheMode = inline !== undefined ? inline : rest[cacheAt + 1]; + } const capAt = rest.findIndex((a) => /^--max-workers(=|$)/.test(a)); if (capAt !== -1) { const inline = rest[capAt].split('=')[1]; @@ -1062,6 +1067,7 @@ Commands: test config [n] Show or set the concurrency limit (0 pauses the queue) [--max-workers |none] also caps each run's vitest pool [--typecheck ] sets the typecheck lane's limit + [--cache off|shadow|on] result cache: on skips unchanged tests wt stale List worktrees whose PR merged (read-only) wt rm Remove a worktree safely (unlinks junctions first) [--stop-server] [--force] diff --git a/.claude/skills/dev-server/scripts/daemon.mjs b/.claude/skills/dev-server/scripts/daemon.mjs index a7a3354790..5279f13b4c 100644 --- a/.claude/skills/dev-server/scripts/daemon.mjs +++ b/.claude/skills/dev-server/scripts/daemon.mjs @@ -89,6 +89,7 @@ function loadSkillConfig() { testConcurrency: 1, typecheckConcurrency: 1, testMaxWorkers: null, + testCacheMode: 'off', prodGroups: [], }; @@ -178,6 +179,11 @@ function loadSkillConfig() { else if (value) console.error(`Ignoring TYPECHECK_CONCURRENCY=${value} (want an integer >= 0)`); break; } + case 'TEST_CACHE_MODE': + // Degrades rather than throws, like every setting the module-scope queue consumes. + if (['off', 'shadow', 'on'].includes(value)) config.testCacheMode = value; + else if (value) console.error(`Ignoring TEST_CACHE_MODE=${value} (want off, shadow or on)`); + break; case 'TEST_MAX_WORKERS': { // Same reasoning as TEST_CONCURRENCY above — this feeds a constructor that throws, and // the queue is built at module scope, so a typo would stop the daemon binding at all. @@ -2016,6 +2022,7 @@ const testQueue = new TestQueue({ typecheck: skillConfig.typecheckConcurrency, }, maxWorkers: skillConfig.testMaxWorkers, + cacheMode: skillConfig.testCacheMode, }); // A tracked session owns its port whatever its status says. Status is a report the daemon @@ -2634,6 +2641,7 @@ async function main() { testQueue.setConcurrency(parsed.typecheckConcurrency, 'typecheck'); } if (parsed.maxWorkers !== undefined) testQueue.setMaxWorkers(parsed.maxWorkers); + if (parsed.cacheMode !== undefined) testQueue.setCacheMode(parsed.cacheMode); } catch (err) { res.writeHead(400); res.end(JSON.stringify({ error: err.message })); @@ -2645,6 +2653,7 @@ async function main() { concurrency: testQueue.concurrency, typecheckConcurrency: testQueue.concurrencyFor('typecheck'), maxWorkers: testQueue.maxWorkers, + cacheMode: testQueue.cacheMode, paused: testQueue.paused, queued: testQueue.order.length, running: testQueue.running.size, diff --git a/.claude/skills/dev-server/scripts/test-queue.mjs b/.claude/skills/dev-server/scripts/test-queue.mjs index 1fe733737c..3f9ca290d9 100644 --- a/.claude/skills/dev-server/scripts/test-queue.mjs +++ b/.claude/skills/dev-server/scripts/test-queue.mjs @@ -13,7 +13,7 @@ import { EventEmitter } from 'events'; import { spawn, execFileSync } from 'child_process'; -import { closeSync, openSync, readSync, unlinkSync } from 'fs'; +import { closeSync, existsSync, openSync, readSync, unlinkSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { randomUUID } from 'crypto'; @@ -211,6 +211,26 @@ export function workerCapArgv(maxWorkers, args) { return [`--max-workers=${maxWorkers}`]; } +export const CACHE_MODES = ['off', 'shadow', 'on']; + +// The WORKTREE's copy, not the daemon's: the sequencer and fs tracker come from that tree's +// vitest config, and all three must share one key definition (scripts/test-cache/core.mjs) or +// every lookup misses. A tree without the files simply runs uncached. +export const cacheReporterPath = (worktree) => join(worktree, 'scripts', 'test-cache', 'reporter.mjs'); + +/** + * The reporter arguments a queued unit run should carry. The reporter is passed on the command line + * rather than from the config because a caller's own `--reporter` replaces config reporters — and + * that would drop the false-skip check while the sequencer went on skipping. Naming any + * `--reporter` also replaces vitest's default, so a caller who named none gets `default` back. + */ +export function cacheReporterArgv(cacheMode, args, reporterPath) { + if (cacheMode === 'off') return []; + if (!reporterPath || !existsSync(reporterPath)) return []; + const named = args.some((a) => /^--reporter(?:=|$)/.test(String(a))); + return [...(named ? [] : ['--reporter=default']), `--reporter=${reporterPath}`]; +} + export function defaultStartRun({ worktree, args, @@ -218,12 +238,20 @@ export function defaultStartRun({ onExit, maxWorkers = null, kind = DEFAULT_KIND, + cacheMode = 'off', }) { const emitter = new EventEmitter(); const isWindows = process.platform === 'win32'; const pnpm = isWindows ? 'pnpm.cmd' : 'pnpm'; const { script, capWorkers } = RUN_KINDS[normalizeKind(kind)]; - const argv = ['run', script, ...args, ...(capWorkers ? workerCapArgv(maxWorkers, args) : [])]; + const argv = [ + 'run', + script, + ...args, + ...(capWorkers ? workerCapArgv(maxWorkers, args) : []), + // Unit runs only: the reporter is a vitest reporter, and tsc would reject the flag. + ...(capWorkers ? cacheReporterArgv(cacheMode, args, cacheReporterPath(worktree)) : []), + ]; onLog('info', `> ${pnpm} ${argv.join(' ')}`); @@ -245,7 +273,12 @@ export function defaultStartRun({ // enqueue a second run and wait for it, while this one holds the slot that run needs — a // deadlock on every full-suite run, not a race. Concurrency is not the fix: each logical run // would need two slots, so N agents starting together still fill them all with waiters. - env: { ...process.env, CIVITAI_TEST_QUEUE: '0' }, + env: { + ...process.env, + CIVITAI_TEST_QUEUE: '0', + // Read by the worktree's vitest config, sequencer, tracker and reporter alike. + CIVITAI_TEST_CACHE: RUN_KINDS[normalizeKind(kind)].capWorkers ? cacheMode : 'off', + }, // The same fd twice: one file description, one shared offset, so the two streams append in // the order they were actually written. See createOutputCapture. stdio: ['ignore', capture.writeFd, capture.writeFd], @@ -354,6 +387,7 @@ export class TestQueue { const { concurrency = DEFAULT_CONCURRENCY, maxWorkers = DEFAULT_MAX_WORKERS, + cacheMode = 'off', startRun = defaultStartRun, now = () => Date.now(), abandonAfterMs = DEFAULT_ABANDON_AFTER_MS, @@ -364,6 +398,7 @@ export class TestQueue { this.limits = normalizeLimits(concurrency); this.maxWorkers = normalizeMaxWorkers(maxWorkers); + this.cacheMode = normalizeCacheMode(cacheMode); this.startRun = startRun; this.now = now; this.abandonAfterMs = abandonAfterMs; @@ -489,6 +524,12 @@ export class TestQueue { * Takes effect on the NEXT run to start, never on one already running — the width is fixed when * vitest is spawned. Nothing here kills a run to resize it. */ + /** Like the worker cap: applies to the next run to start, never to one already running. */ + setCacheMode(value) { + this.cacheMode = normalizeCacheMode(value); + return this.cacheMode; + } + setMaxWorkers(value) { this.maxWorkers = normalizeMaxWorkers(value); return this.maxWorkers; @@ -651,6 +692,7 @@ export class TestQueue { onExit, maxWorkers: this.maxWorkers, kind: run.kind, + cacheMode: this.cacheMode, }); } catch (err) { // A runner that reported an exit and then threw has already produced a verdict; overwriting @@ -740,6 +782,7 @@ export class TestQueue { running: this.runningFor(run.kind), concurrency: this.limits[run.kind], maxWorkers: this.maxWorkers, + cacheMode: this.cacheMode, paused: this.limits[run.kind] === 0, enqueuedAt: run.enqueuedAt, startedAt: run.startedAt, @@ -761,6 +804,14 @@ export class TestQueue { * workers, and vitest's own resolution treats a falsy value as "unset" — so a 0 that slipped * through here would silently restore the uncapped pool the setting exists to prevent. */ +function normalizeCacheMode(value) { + const mode = value === undefined || value === null || value === '' ? 'off' : String(value); + if (!CACHE_MODES.includes(mode)) { + throw new Error(`cacheMode must be one of ${CACHE_MODES.join(', ')}, got: ${value}`); + } + return mode; +} + function normalizeMaxWorkers(value) { if (value === null || value === undefined || value === '') return null; const parsed = typeof value === 'number' ? value : parseInt(value, 10); diff --git a/CLAUDE.md b/CLAUDE.md index 6322ce9aff..97551091bd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -175,6 +175,21 @@ as running all of it. Two source files gave the identical number. count: when 17 tests fail across 7 files, `git stash` and re-run those same files to see whether they already failed on `main`. On Windows several do — see the portability notes below. +#### Queued unit runs can be served from a result cache + +When the dev-server queue has the cache on (`cli.mjs test config --cache on`), a queued `test:unit:run` +skips every test file whose inputs are unchanged since it last passed — its source, every module it imports, +every file it read, and the lockfile/configs. Results are shared between worktrees. It prints what it did: + +``` +[test-cache] 1880 test files: 41 ran, 1839 skipped as unchanged since they last passed. 92 unchanged file(s) re-run to verify; false skips: 0. +``` + +A random ~5% of the unchanged files run anyway. If one of those fails, the cache predicted a pass it could not +deliver — a **false skip** — and it trips itself off (`TRIPPED.json` in the cache dir) until a human looks. +**Known blind spot:** environment variables are not part of the key. Never on in CI; a run that filters +files (a filename, directory or substring) is never trimmed. Code and details: `scripts/test-cache/`. + #### Worker count: uncapped by default, `VITEST_MAX_WORKERS` / `--max-workers` to size it A suite uses Vitest's own worker count (`cpus - 1` in run mode, `floor(cpus / 2)` in watch; the browser pool `min(12, cpus - 1)`). @@ -698,6 +713,11 @@ as a pass to anyone checking an exit code or skimming a summary. **Validate any that file collected a nonzero count** — it was 308 tests on one base. If it reports 0, the run tells you nothing about your change, whatever the summary says. +On a queued run with the result cache on, that file can be absent from the output for a legitimate reason: +it was skipped as unchanged since it last passed. The `[test-cache]` line names how many files were skipped. +To apply this check, confirm the file either ran with a nonzero count or is not in your diff's reach — +or run it by its full filename, which never goes through the cache. + **A fresh worktree also has no `.envrc`.** It's gitignored, so it never comes with the checkout, and you silently get system Node instead of the flake's pinned version. Measured (when the flake still shipped node 22): system Node **26.5.0** against the flake's **22.22.2** produced 7 spurious `window.localStorage is undefined` failures diff --git a/scripts/__tests__/dev-server-test-queue-spawn.test.ts b/scripts/__tests__/dev-server-test-queue-spawn.test.ts index c0e0cb50fc..037e53d06e 100644 --- a/scripts/__tests__/dev-server-test-queue-spawn.test.ts +++ b/scripts/__tests__/dev-server-test-queue-spawn.test.ts @@ -11,7 +11,7 @@ vi.mock('child_process', async (importOriginal) => ({ // Lives under scripts/ because the daemon is not part of the app's module graph — same arrangement // as the rest of the queue's tests. -const { defaultStartRun, TestQueue } = await import( +const { defaultStartRun, TestQueue, cacheReporterArgv, cacheReporterPath } = await import( '../../.claude/skills/dev-server/scripts/test-queue.mjs' ); @@ -161,3 +161,80 @@ describe("the queue caps each run's vitest pool", () => { expect(() => new TestQueue({ concurrency: 1, maxWorkers: 0 })).toThrow(/integer >= 1/); }); }); + +/** + * The cache rides on the fleet's real queued runs, so the one thing its wiring must not do is change + * them. Naming any `--reporter` replaces vitest's default — so a caller who named none has to get + * `default` back, or turning the cache on silently strips every queued run's normal output. + */ +describe('result cache on queued runs', () => { + const argvOf = (call: number) => spawn.mock.calls[call][1] as string[]; + const envOf = (call: number) => + (spawn.mock.calls[call][2] as { env: Record }).env; + // The repo's own tree, so the reporter file really exists and the argv is not vacuously empty. + const worktree = process.cwd(); + const reporter = cacheReporterPath(worktree); + const start = (opts: Record) => { + const handle = defaultStartRun({ + worktree, + args: [], + onLog: () => undefined, + onExit: () => undefined, + ...opts, + }) as EventEmitter & { dispose: () => void }; + handle.dispose(); + }; + + it('adds nothing and tells the run the cache is off while it is off', () => { + start({ cacheMode: 'off' }); + expect(argvOf(0)).toEqual(['run', 'test:unit:run']); + expect(envOf(0).CIVITAI_TEST_CACHE).toBe('off'); + }); + + it('keeps the default reporter beside the cache one when the caller named none', () => { + start({ cacheMode: 'on' }); + expect(argvOf(0)).toEqual([ + 'run', + 'test:unit:run', + '--reporter=default', + `--reporter=${reporter}`, + ]); + expect(envOf(0).CIVITAI_TEST_CACHE).toBe('on'); + }); + + it('adds only the cache reporter when the caller chose their own', () => { + start({ cacheMode: 'shadow', args: ['--reporter=json'] }); + expect(argvOf(0)).toEqual([ + 'run', + 'test:unit:run', + '--reporter=json', + `--reporter=${reporter}`, + ]); + }); + + // tsc would reject a vitest reporter, and the cache has nothing to skip in a typecheck. + it('leaves a typecheck untouched', () => { + start({ cacheMode: 'on', kind: 'typecheck' }); + expect(argvOf(0)).toEqual(['run', 'typecheck']); + expect(envOf(0).CIVITAI_TEST_CACHE).toBe('off'); + }); + + // A tree without the cache files runs uncached rather than as a vitest that cannot load its + // reporter and fails every queued suite. + it('adds nothing when the reporter file is absent', () => { + expect(cacheReporterArgv('on', [], '/nowhere/reporter.mjs')).toEqual([]); + }); + + it("hands the queue's cache mode to the runner it starts", () => { + const startRun = vi.fn<(opts: { cacheMode: string }) => EventEmitter>(() => new EventEmitter()); + const queue = new TestQueue({ concurrency: 1, cacheMode: 'on', startRun }) as unknown as { + request: (run: { worktree: string; args: string[] }) => unknown; + }; + queue.request({ worktree: '/repo', args: [] }); + expect(startRun.mock.calls[0][0]).toMatchObject({ cacheMode: 'on' }); + }); + + it('refuses an unknown cache mode', () => { + expect(() => new TestQueue({ concurrency: 1, cacheMode: 'yes' })).toThrow(/cacheMode must be/); + }); +}); diff --git a/scripts/__tests__/test-cache-core.test.ts b/scripts/__tests__/test-cache-core.test.ts new file mode 100644 index 0000000000..1be0762da8 --- /dev/null +++ b/scripts/__tests__/test-cache-core.test.ts @@ -0,0 +1,262 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { describe, expect, it } from 'vitest'; + +import * as Core from '../test-cache/core.mjs'; + +type Node = { id: string; importedModules: Set }; +type Fingerprint = (rel: string) => string; + +const { + closureOf, + toRel, + isCoveredElsewhere, + alwaysRuns, + keyFor, + makeFingerprinter, + mode, + shadowCandidates, + changedSince, + tripped, +} = Core as unknown as { + closureOf: ( + g: { getModuleById: (id: string) => Node | undefined }, + id: string + ) => Set | null; + toRel: (id: string, root: string) => string | null; + isCoveredElsewhere: (rel: string | null) => boolean; + alwaysRuns: (source: string) => boolean; + keyFor: (a: { + salt: string; + project: string; + testRel: string; + entries: string[]; + fingerprint: Fingerprint; + }) => string; + makeFingerprinter: (root: string) => Fingerprint; + mode: (env: Record) => string; + shadowCandidates: (rel: string) => string[]; + changedSince: (root: string, rel: string, sinceMs: number) => boolean; + tripped: (dir: string) => { at: string } | null; +}; + +function graphOf(edges: Record) { + const nodes = new Map(); + const node = (id: string) => { + if (!nodes.has(id)) nodes.set(id, { id, importedModules: new Set() }); + return nodes.get(id)!; + }; + for (const [from, tos] of Object.entries(edges)) + for (const to of tos) node(from).importedModules.add(node(to)); + return { getModuleById: (id: string) => nodes.get(id) }; +} + +describe('the dependency set a key is built from', () => { + it('follows imports transitively and terminates on a cycle', () => { + const graph = graphOf({ 't.test.ts': ['a.ts'], 'a.ts': ['b.ts'], 'b.ts': ['a.ts', 'c.ts'] }); + expect([...closureOf(graph, 't.test.ts')!].sort()).toEqual(['a.ts', 'b.ts', 'c.ts']); + }); + + // null, not an empty set: an empty dependency set is a key no change can ever invalidate. + it('reports a file missing from the graph as unknown rather than dependency-free', () => { + expect(closureOf(graphOf({}), 'missing.test.ts')).toBeNull(); + }); +}); + +describe('keys are portable between worktrees', () => { + // One tree's green run covers every other tree whose files are identical. A key that embedded + // the worktree path would never hit across trees. + it('gives the same path for the same file in two worktrees, URL or path', () => { + expect(toRel('C:/Dev/wt/one/src/a.ts?v=123', 'C:\\Dev\\wt\\one')).toBe('src/a.ts'); + expect(toRel('file:///C:/Dev/wt/two/src/a.ts', 'C:/Dev/wt/two/')).toBe('src/a.ts'); + }); + + // A read outside the repo (a temp file the test wrote itself) is not an input anyone else shares. + it('drops absolute paths outside the repo', () => { + expect(toRel('D:/elsewhere/x.json', 'C:/Dev/wt/one')).toBeNull(); + expect(isCoveredElsewhere(null)).toBe(true); + }); + + it('leaves node_modules and builtins to the lockfile', () => { + expect(isCoveredElsewhere('node_modules/.vite/vitest/x/deps_ssr/zod.js')).toBe(true); + expect(isCoveredElsewhere('crypto')).toBe(true); + expect(isCoveredElsewhere('src/server/services/image.service.ts')).toBe(false); + }); +}); + +describe('the key', () => { + const root = mkdtempSync(join(tmpdir(), 'test-cache-')); + mkdirSync(join(root, 'src/sub'), { recursive: true }); + writeFileSync(join(root, 't.test.ts'), 'test'); + writeFileSync(join(root, 'src/a.ts'), 'a'); + const key = (entries: string[]) => + keyFor({ + salt: 's', + project: 'unit', + testRel: 't.test.ts', + entries, + fingerprint: makeFingerprinter(root), + }); + + it('does not depend on the order dependencies were discovered in', () => { + expect(key(['src/a.ts', 'src'])).toBe(key(['src', 'src/a.ts'])); + }); + + it('changes when a dependency changes', () => { + const before = key(['src/a.ts']); + writeFileSync(join(root, 'src/a.ts'), 'a2'); + expect(key(['src/a.ts'])).not.toBe(before); + }); + + // A convention guard lists a directory recursively in ONE call. A file added three levels down + // changes what that call returns, so it has to change the key. + it('changes when a file appears anywhere below a directory that was read', () => { + const before = key(['src']); + writeFileSync(join(root, 'src/sub/new.ts'), 'n'); + expect(key(['src'])).not.toBe(before); + }); +}); + +describe('tests that always run', () => { + it.each([ + ["import { execSync } from 'node:child_process';"], + ["import * as cp from 'child_process';"], + ["const { Worker } = require('worker_threads');"], + ['const m = await import(`./pages/${name}`);'], + ['const m = await import(target);'], + // The form this repo uses, which the first version of the pattern let through. + ['return import(/* @vite-ignore */ file);'], + ["const files = globSync('src/**/*.ts');"], + ["import fg from 'fast-glob';"], + ])('recognises %s', (source) => { + expect(alwaysRuns(source)).toBe(true); + }); + + // Plain file reads are no longer a reason to always run: the tracker records them into the key. + // A literal dynamic import is in the module graph. + // `regex.exec(` is not a process: the call-name pattern this replaced matched it in + // src/__tests__/setup.ts, which every test loads, and nothing was cacheable at all. + it.each([ + ["readFileSync('src/x.ts', 'utf8');"], + ["await import('./dep');"], + ['const m = /^a(b)$/.exec(input);'], + // `'cluster'` is an ordinary value in this repo's Redis and telemetry code, reached by every + // test's setup. Matching the bare word made all 1880 unit tests uncacheable. + ["const opts = { client: 'cluster' };"], + ["// Module not found: Can't resolve 'cluster'"], + ])('leaves %s cacheable', (source) => { + expect(alwaysRuns(source)).toBe(false); + }); +}); + +// CI must always run everything: it is the check that still runs when nothing local does. +describe('when the cache is active', () => { + it('is off in CI whatever the mode says', () => { + expect(mode({ CI: 'true', CIVITAI_TEST_CACHE: 'on' })).toBe('off'); + }); + + it('is off unless a known mode is named', () => { + expect(mode({})).toBe('off'); + expect(mode({ CIVITAI_TEST_CACHE: 'yes' })).toBe('off'); + expect(mode({ CIVITAI_TEST_CACHE: 'on' })).toBe('on'); + }); +}); + +describe('what a key must see besides content', () => { + // `./foo` resolving to `foo/index.ts` is overtaken by a new `foo.ts` without any file in the + // dependency list changing. + it('names the files that would shadow an index module or an extension', () => { + const c = shadowCandidates('src/foo/index.ts'); + expect(c).toContain('src/foo.ts'); + expect(c).toContain('src/foo/index.tsx'); + expect(c).not.toContain('src/foo/index.ts'); + }); + + it('changes when a shadowing file appears', () => { + const root = mkdtempSync(join(tmpdir(), 'test-cache-shadow-')); + mkdirSync(join(root, 'src/foo'), { recursive: true }); + writeFileSync(join(root, 't.test.ts'), 't'); + writeFileSync(join(root, 'src/foo/index.ts'), 'i'); + const k = () => + keyFor({ + salt: 's', + project: 'unit', + testRel: 't.test.ts', + entries: ['src/foo/index.ts'], + fingerprint: makeFingerprinter(root), + }); + const before = k(); + writeFileSync(join(root, 'src/foo.ts'), 'f'); + expect(k()).not.toBe(before); + }); + + // The key is computed at the END of a run; an input written after the run started is not what + // ran. Measured by review: without this an edit made during a queued suite was recorded as passing. + describe('what changed during a run', () => { + // The tree is built, THEN the "run" starts, so each assertion below moves exactly one thing + // past `since`. Not backdated with utimes: that bumps ctime to now, and ctime is checked on + // purpose — it is what catches an edit made with a backdated mtime. + const pause = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); + const setup = () => { + const root = mkdtempSync(join(tmpdir(), 'test-cache-mtime-')); + mkdirSync(join(root, 'd/deep'), { recursive: true }); + mkdirSync(join(root, 'h'), { recursive: true }); + writeFileSync(join(root, 'a.ts'), 'a'); + writeFileSync(join(root, 'h/gone.ts'), 'g'); + pause(50); + const since = Date.now(); + pause(50); + return { root, since }; + }; + + it('reports nothing for inputs untouched since the run started', () => { + const { root, since } = setup(); + expect(changedSince(root, 'a.ts', since)).toBe(false); + expect(changedSince(root, 'd', since)).toBe(false); + expect(changedSince(root, 'h/gone.ts', since)).toBe(false); + }); + + it('sees a file edited after the run started', () => { + const { root, since } = setup(); + writeFileSync(join(root, 'a.ts'), 'a2'); + expect(changedSince(root, 'a.ts', since)).toBe(true); + }); + + // A convention guard lists a directory recursively; a file added three levels down counts. + it('sees a directory whose subtree gained a file', () => { + const { root, since } = setup(); + writeFileSync(join(root, 'd/deep/new.ts'), 'n'); + expect(changedSince(root, 'd', since)).toBe(true); + }); + + // The test ran WITH the file. Recording its absence would skip it green next time — confirmed + // by review as a false skip before this existed. + // Every test probes `__snapshots__/.snap`, a directory that usually never existed. + // Reading a missing parent as "changed" refused every record in the repo. + it('does not count a probe into a directory that never existed', () => { + const { root, since } = setup(); + expect(changedSince(root, 'd/__snapshots__/x.test.ts.snap', since)).toBe(false); + }); + + it('sees a whole directory removed after the run started', () => { + const { root, since } = setup(); + rmSync(join(root, 'd'), { recursive: true }); + expect(changedSince(root, 'd/deep/x.ts', since)).toBe(true); + }); + + it('sees a file deleted after the run started', () => { + const { root, since } = setup(); + rmSync(join(root, 'h/gone.ts')); + expect(changedSince(root, 'h/gone.ts', since)).toBe(true); + }); + }); + + // A half-written marker must read as tripped, never as "safe to skip". + it('treats an unreadable trip marker as tripped', () => { + const dir = mkdtempSync(join(tmpdir(), 'test-cache-trip-')); + expect(tripped(dir)).toBeNull(); + writeFileSync(join(dir, 'TRIPPED.json'), '{"at": "2026'); + expect(tripped(dir)).not.toBeNull(); + }); +}); diff --git a/scripts/__tests__/test-cache-e2e.test.ts b/scripts/__tests__/test-cache-e2e.test.ts new file mode 100644 index 0000000000..58ca8d59f8 --- /dev/null +++ b/scripts/__tests__/test-cache-e2e.test.ts @@ -0,0 +1,56 @@ +import { spawnSync } from 'child_process'; +import { mkdtempSync, readFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join, resolve } from 'path'; +import { describe, expect, it } from 'vitest'; + +/** + * The one test that runs the cache for real: vitest, the real sequencer, reporter and fs tracker, + * over a one-file fixture — twice. Unit tests with fake vitest objects stayed green through two + * regressions that made the cache record NOTHING on the real repo, because the fakes modelled + * neither the snapshot probe every file makes nor a setup closure mentioning 'cluster'. This is + * what reddens when the cache is inert. + */ +const repo = resolve(__dirname, '../..'); +const vitestBin = join(repo, 'node_modules/vitest/vitest.mjs'); +const config = 'scripts/test-cache/__e2e__/vitest.e2e.config.mts'; +const reporter = join(repo, 'scripts/test-cache/reporter.mjs'); + +function runOnce(cacheDir: string) { + const r = spawnSync( + process.execPath, + [vitestBin, 'run', '--config', config, '--reporter=default', `--reporter=${reporter}`], + { + cwd: repo, + encoding: 'utf8', + // Bounded: a wedged child fails this test with a timeout message, never hangs the runner. + timeout: 120_000, + env: { + ...process.env, + CI: '', + CIVITAI_TEST_CACHE: 'on', + CIVITAI_TEST_CACHE_DIR: cacheDir, + CIVITAI_TEST_CACHE_SAMPLE: '0', + }, + } + ); + const ledger = readFileSync(join(cacheDir, 'ledger.jsonl'), 'utf8').trim().split('\n'); + return { status: r.status, last: JSON.parse(ledger[ledger.length - 1]) }; +} + +describe('the cache, run for real', () => { + it('records a passing file, then skips it when nothing changed', () => { + const cacheDir = mkdtempSync(join(tmpdir(), 'test-cache-e2e-')); + + const cold = runOnce(cacheDir); + expect(cold.status).toBe(0); + expect({ recorded: cold.last.recorded, notRecorded: cold.last.notRecorded }).toEqual({ + recorded: 1, + notRecorded: {}, + }); + + const warm = runOnce(cacheDir); + expect(warm.status).toBe(0); + expect({ ran: warm.last.ran, skipped: warm.last.skipped }).toEqual({ ran: 0, skipped: 1 }); + }, 300_000); +}); diff --git a/scripts/__tests__/test-cache-reporter.test.ts b/scripts/__tests__/test-cache-reporter.test.ts new file mode 100644 index 0000000000..3b54a8f855 --- /dev/null +++ b/scripts/__tests__/test-cache-reporter.test.ts @@ -0,0 +1,284 @@ +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import TestCacheReporter from '../test-cache/reporter.mjs'; +import TestCacheSequencer from '../test-cache/sequencer.mjs'; + +/** + * The reporter and sequencer driven with fake vitest objects, so each guard can be reverted and + * seen to fail. Every case below was a false skip, or a hole in the check that catches one, found + * by adversarial review of this cache and reproduced on a fixture before it was fixed. + */ + +type Node = { id: string; importedModules: Set }; +type State = { + hits: Set; + sampled: Set; + skipped: string[]; + startedAt: number; + bailed?: string | null; +}; + +const fwd = (p: string) => p.replace(/\\/g, '/'); +let root: string; +let cacheDir: string; + +function graphOf(edges: Record) { + const nodes = new Map(); + const node = (id: string) => { + if (!nodes.has(id)) nodes.set(id, { id, importedModules: new Set() }); + return nodes.get(id)!; + }; + for (const [from, tos] of Object.entries(edges)) { + node(from); + for (const to of tos) node(from).importedModules.add(node(to)); + } + return { getModuleById: (id: string) => nodes.get(id) }; +} + +function write(rel: string, content = rel) { + mkdirSync(join(root, rel, '..'), { recursive: true }); + writeFileSync(join(root, rel), content); +} + +function testModule( + rel: string, + { passed = true, deps = [] as string[], reads = [] as string[], setupFiles = [] as string[] } = {} +) { + const abs = fwd(join(root, rel)); + const graph = graphOf({ [abs]: deps.map((d) => fwd(join(root, d))) }); + return { + moduleId: abs, + project: { + name: 'unit', + config: { setupFiles: setupFiles.map((s) => fwd(join(root, s))) }, + vite: { environments: { ssr: { moduleGraph: graph } } }, + }, + meta: () => ({ testCacheReads: reads.map((r) => join(root, r)) }), + diagnostic: () => ({}), + state: () => (passed ? 'passed' : 'failed'), + children: { + *allTests() { + yield { result: () => ({ state: passed ? 'passed' : 'failed' }) }; + }, + }, + }; +} + +function run( + modules: ReturnType[], + { errors = [] as unknown[], state = {} as Partial } = {} +) { + (globalThis as { __civitaiTestCache?: State }).__civitaiTestCache = { + hits: new Set(), + sampled: new Set(), + skipped: [], + // Well in the future, so nothing counts as changed during the run unless a case says so. + startedAt: Date.now() + 60_000, + ...state, + }; + const reporter = new TestCacheReporter(); + reporter.onInit({ config: { root }, version: 'test' } as never); + reporter.onTestRunEnd(modules as never, errors as never, 'passed' as never); +} + +/** How many stored records name `rel` as their test file's own entry. */ +function records(rel: string) { + const recDir = join(cacheDir, 'rec'); + if (!existsSync(recDir)) return 0; + let n = 0; + for (const d of readdirSync(recDir)) { + for (const f of readdirSync(join(recDir, d))) { + const body = JSON.parse(readFileSync(join(recDir, d, f), 'utf8')); + if (body.entries.includes(rel)) n += 1; + } + } + return n; +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'test-cache-root-')); + cacheDir = mkdtempSync(join(tmpdir(), 'test-cache-dir-')); + vi.stubEnv('CIVITAI_TEST_CACHE', 'on'); + vi.stubEnv('CIVITAI_TEST_CACHE_DIR', cacheDir); + vi.stubEnv('CI', ''); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + vi.spyOn(console, 'log').mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + delete (globalThis as { __civitaiTestCache?: State }).__civitaiTestCache; +}); + +describe('recording a pass', () => { + // The positive control every case below leans on: without it, "not recorded" passes vacuously — + // which is exactly how an earlier round of these fixes certified itself while recording nothing. + it('records an ordinary passing file', () => { + write('a.test.ts'); + write('dep.ts'); + run([testModule('a.test.ts', { deps: ['dep.ts'] })]); + expect(records('a.test.ts')).toBe(1); + }); + + /** + * The control above, shaped like a REAL test file — the shape the fakes first left out. Every real + * test probes a snapshot path in a `__snapshots__` directory that usually never existed, and its + * setup closure reaches Redis code where `'cluster'` is an ordinary value. Review measured each of + * those, separately, making every one of the repo's 1880 unit tests unrecordable while the plain + * control stayed green. + */ + it('records a file shaped like a real one: snapshot probe, and a setup closure mentioning cluster', () => { + write('src/a.test.ts'); + write('src/__tests__/setup.ts'); + write('packages/redis/client.ts', "export const opts = { client: 'cluster' };"); + const abs = (r: string) => fwd(join(root, r)); + const m = testModule('src/a.test.ts', { + reads: ['src/__snapshots__/a.test.ts.snap'], + setupFiles: ['src/__tests__/setup.ts'], + }); + const g = m.project.vite.environments.ssr.moduleGraph as ReturnType; + const setupGraph = graphOf({ + [abs('src/a.test.ts')]: [], + [abs('src/__tests__/setup.ts')]: [abs('packages/redis/client.ts')], + }); + m.project.vite.environments.ssr.moduleGraph = { + getModuleById: (id: string) => g.getModuleById(id) ?? setupGraph.getModuleById(id), + }; + run([m]); + expect(records('src/a.test.ts')).toBe(1); + }); + + it('refuses a file whose input changed after the run started', () => { + write('a.test.ts'); + write('dep.ts'); + run([testModule('a.test.ts', { deps: ['dep.ts'] })], { state: { startedAt: 0 } }); + expect(records('a.test.ts')).toBe(0); + }); + + // The test ran WITH the module; recording its absence would skip it green next time. + it('refuses a file whose imported module is gone by the end of the run', () => { + write('a.test.ts'); + run([testModule('a.test.ts', { deps: ['deleted.ts'] })]); + expect(records('a.test.ts')).toBe(0); + }); + + it('refuses a file whose HELPER makes a computed import', () => { + write('a.test.ts'); + write('helper.ts', 'export const load = (f: string) => import(/* @vite-ignore */ f);'); + run([testModule('a.test.ts', { deps: ['helper.ts'] })]); + expect(records('a.test.ts')).toBe(0); + }); + + it('refuses a file whose helper reaches child_process by any syntax', () => { + write('a.test.ts'); + write('helper.ts', "const cp = process.getBuiltinModule('node:child_process');"); + run([testModule('a.test.ts', { deps: ['helper.ts'] })]); + expect(records('a.test.ts')).toBe(0); + }); +}); + +describe('unhandled errors', () => { + it('blocks only the file the error is attributed to', () => { + write('a.test.ts'); + write('b.test.ts'); + run([testModule('a.test.ts'), testModule('b.test.ts')], { + errors: [{ VITEST_TEST_PATH: fwd(join(root, 'a.test.ts')) }], + }); + expect(records('a.test.ts')).toBe(0); + expect(records('b.test.ts')).toBe(1); + }); + + it('blocks every file when an error cannot be attributed', () => { + write('a.test.ts'); + write('b.test.ts'); + run([testModule('a.test.ts'), testModule('b.test.ts')], { errors: [{ message: 'somewhere' }] }); + expect(records('a.test.ts')).toBe(0); + expect(records('b.test.ts')).toBe(0); + }); +}); + +describe('setup files', () => { + // The fs tracker is instrumentation whose own code imports child_process. Walking it marked every + // test uncacheable; it is covered by the salt instead. + it('does not let the cache tracker itself make a file uncacheable', () => { + write('a.test.ts'); + run([testModule('a.test.ts', { setupFiles: ['scripts/test-cache/fs-tracker.mjs'] })]); + expect(records('a.test.ts')).toBe(1); + }); + + it('refuses a file whose ordinary setup file is missing from the graph', () => { + write('a.test.ts'); + run([testModule('a.test.ts', { setupFiles: ['src/__tests__/setup.ts'] })]); + expect(records('a.test.ts')).toBe(0); + }); +}); + +describe('a false skip', () => { + it('trips the cache and forgets the file, so clearing the trip cannot revive the skip', () => { + write('a.test.ts'); + run([testModule('a.test.ts')]); + expect(records('a.test.ts')).toBe(1); + + const hit = `unit\0${fwd(join(root, 'a.test.ts'))}`; + run([testModule('a.test.ts', { passed: false })], { state: { hits: new Set([hit]) } }); + + expect(existsSync(join(cacheDir, 'TRIPPED.json'))).toBe(true); + expect(records('a.test.ts')).toBe(0); + }); + + // stdout is reserved for a caller's own `--reporter=json`. + it('says everything on stderr, nothing on stdout', () => { + write('a.test.ts'); + run([testModule('a.test.ts')]); + expect(console.error).toHaveBeenCalled(); + expect(console.log).not.toHaveBeenCalled(); + }); +}); + +describe('when the sequencer declines to skip', () => { + const bailFor = async (ctx: Record) => { + const seq = new TestCacheSequencer({ config: {}, reporters: [], ...ctx } as never); + await seq.sort([]); + return (globalThis as { __civitaiTestCache?: State }).__civitaiTestCache?.bailed; + }; + const withReporter = [new TestCacheReporter()]; + + it('declines on a name filter', async () => { + expect(await bailFor({ config: { testNamePattern: /x/ }, reporters: withReporter })).toBe( + 'name filter' + ); + }); + + it('declines when the run names files', async () => { + expect(await bailFor({ filenamePattern: ['a.test.ts'], reporters: withReporter })).toBe( + 'file filter' + ); + }); + + // Nothing would record or check the sample, so a skip would be unobserved. + it('declines when the cache reporter is not loaded', async () => { + expect(await bailFor({ reporters: [] })).toBe('cache reporter not loaded'); + }); + + it('does not decline a whole-suite run with the reporter loaded', async () => { + expect( + await bailFor({ config: { root }, reporters: withReporter, version: 'test' }) + ).toBeNull(); + }); +}); + +describe('the fs tracker', () => { + // Wrapping a function drops properties that live ON it. `fs.realpathSync.native` is one, and + // next's lib/realpath.js calls it off-Windows — review confirmed the tracker stripped it. + it('keeps realpathSync.native callable after wrapping fs', async () => { + vi.stubEnv('CIVITAI_TEST_CACHE', 'on'); + await import('../test-cache/fs-tracker.mjs'); + const fs = await import('node:fs'); + expect(typeof fs.realpathSync.native).toBe('function'); + expect(fs.realpathSync.native(root)).toBeTruthy(); + }); +}); diff --git a/scripts/test-cache/__e2e__/dep.ts b/scripts/test-cache/__e2e__/dep.ts new file mode 100644 index 0000000000..64a32fd291 --- /dev/null +++ b/scripts/test-cache/__e2e__/dep.ts @@ -0,0 +1 @@ +export const answer = 42; diff --git a/scripts/test-cache/__e2e__/probe.e2e.ts b/scripts/test-cache/__e2e__/probe.e2e.ts new file mode 100644 index 0000000000..618531ff30 --- /dev/null +++ b/scripts/test-cache/__e2e__/probe.e2e.ts @@ -0,0 +1,9 @@ +import { expect, it } from 'vitest'; + +import { answer } from './dep'; + +// Every real test file probes `__snapshots__/.snap` in a directory that usually never +// existed; that probe once refused every record. This file gets it for free, like any test. +it('passes', () => { + expect(answer).toBe(42); +}); diff --git a/scripts/test-cache/__e2e__/redis-like.ts b/scripts/test-cache/__e2e__/redis-like.ts new file mode 100644 index 0000000000..c1004a81bb --- /dev/null +++ b/scripts/test-cache/__e2e__/redis-like.ts @@ -0,0 +1,3 @@ +// Shaped like packages/civitai-redis: `'cluster'` as an ordinary value, which a too-broad spawn check +// once treated as the cluster module and refused every test in the repo. +export const redisOptions = { client: 'cluster' }; diff --git a/scripts/test-cache/__e2e__/setup.ts b/scripts/test-cache/__e2e__/setup.ts new file mode 100644 index 0000000000..e75b964bbb --- /dev/null +++ b/scripts/test-cache/__e2e__/setup.ts @@ -0,0 +1,3 @@ +import { redisOptions } from './redis-like'; + +export const configured = redisOptions.client; diff --git a/scripts/test-cache/__e2e__/vitest.e2e.config.mts b/scripts/test-cache/__e2e__/vitest.e2e.config.mts new file mode 100644 index 0000000000..a0458ff6a6 --- /dev/null +++ b/scripts/test-cache/__e2e__/vitest.e2e.config.mts @@ -0,0 +1,19 @@ +import path from 'path'; +import { defineConfig } from 'vitest/config'; + +import TestCacheSequencer from '../sequencer.mjs'; + +// Drives the REAL sequencer, reporter and fs tracker over one fixture, for +// scripts/__tests__/test-cache-e2e.test.ts. Its files end `.e2e.ts`, which the unit project's +// `*.test.ts` include never collects. +const root = path.resolve(__dirname, '../../..'); +export default defineConfig({ + test: { + root, + name: 'unit-e2e', + include: ['scripts/test-cache/__e2e__/*.e2e.ts'], + setupFiles: ['scripts/test-cache/fs-tracker.mjs', 'scripts/test-cache/__e2e__/setup.ts'], + pool: 'forks', + sequence: { sequencer: TestCacheSequencer }, + }, +}); diff --git a/scripts/test-cache/core.mjs b/scripts/test-cache/core.mjs new file mode 100644 index 0000000000..29ad329fd6 --- /dev/null +++ b/scripts/test-cache/core.mjs @@ -0,0 +1,351 @@ +/** + * The one definition of a test file's cache key, shared by the sequencer (which decides BEFORE a + * run what to skip) and the reporter (which records AFTER a run what passed). If the two computed + * a key differently, every lookup would miss — or worse, hit on the wrong thing — so neither + * computes it itself. + * + * A key covers: the test file, every first-party module it depends on, every file or directory it + * read at runtime, and the inputs nothing else records (lockfile, configs, node, platform, vitest, + * and this cache's own code). Anything under node_modules is left to the lockfile. + * + * Node builtins only: the queue runs the reporter from the primary checkout against any worktree. + */ + +import { createHash } from 'node:crypto'; +import { isBuiltin } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const CACHE_FORMAT = 2; +export const MODES = ['off', 'shadow', 'on']; +export const RECORDS_PER_TEST = 8; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +// Inputs every test depends on that no import edge or file read records. This cache's own code is +// among them: a fix to how reads are captured must invalidate everything recorded without it. +// `node_modules/.pnpm/lock.yaml` is what pnpm actually INSTALLED, which a rebased tree that skipped +// `pnpm install` does not share with its lockfile. The package.json files carry `exports` maps, and a +// symlinked workspace package resolves through them — the lockfile does not record `exports`. +const GLOBAL_INPUTS = [ + 'pnpm-lock.yaml', + 'node_modules/.pnpm/lock.yaml', + 'package.json', + 'vitest.config.mts', + 'tsconfig.json', +]; +const WORKSPACE_DIRS = ['packages', 'apps']; +const OWN_CODE = ['core.mjs', 'fs-tracker.mjs', 'sequencer.mjs', 'reporter.mjs']; + +// What stays uncacheable even with file reads tracked: a child process reads what it likes, and a +// dynamic import whose specifier is computed is invisible to the module graph. 19 files, 0.7% of +// modelled worker time, measured 2026-09-19. +const ALWAYS_RUN_SOURCE = [ + // IMPORTING a process or thread module, not calling one by name: a call pattern cannot see + // `cp.execSync(` or a destructured alias, and `exec(` matched every `regex.exec(` in the repo — + // measured: src/__tests__/setup.ts tripped it and nothing was cacheable at all. + // The module name in any IMPORT-SHAPED position: `from`, `import(`, `require(`, + // `getBuiltinModule(`, or a call on a call (`createRequire(...)('child_process')`). Review found + // each of those walking past a narrower pattern. NOT the bare quoted word: `'cluster'` is an + // ordinary value in this repo's Redis and telemetry code, which every test's setup reaches — + // measured, that made 1880 of 1880 unit tests uncacheable. + /(?:\bfrom|\bimport\s*\(|\brequire\s*\(|\bgetBuiltinModule\s*\(|\)\s*\()\s*['"`](?:node:)?(?:child_process|worker_threads|cluster)['"`]/, + // Wrappers that spawn for you. None is a direct dependency today; this keeps one from arriving + // unnoticed, since node_modules is never scanned. + /['"`](?:execa|cross-spawn|tinyexec|nano-spawn|zx)['"`]/, + // Comments allowed between `import(` and the specifier: `import(/* @vite-ignore */ file)` is the + // form this repo actually uses, and the first version of this pattern let it through. + /import\(\s*(?:\/\*[\s\S]*?\*\/\s*)*(?:`[^`]*\$\{|[A-Za-z_$])/, + // A glob's result changes when a MATCHING file is added anywhere, and a pattern is not a path + // whose state can be fingerprinted. + /\b(?:globSync|glob|globby|fastGlob|fg)\(|from ['"](?:fast-glob|globby|glob|tinyglobby)['"]/, +]; + +export function mode(env = process.env) { + if (env.CI) return 'off'; + const m = env.CIVITAI_TEST_CACHE; + return MODES.includes(m) ? m : 'off'; +} + +export const sha = (data) => createHash('sha256').update(data).digest('hex'); + +export function toRel(id, root) { + let p = String(id); + if (p.startsWith('file://')) p = fileURLToPath(p); + p = p.split('?')[0].replace(/\\/g, '/'); + const r = root.replace(/\\/g, '/').replace(/\/$/, ''); + if (p.toLowerCase().startsWith(r.toLowerCase() + '/')) return p.slice(r.length + 1); + return isAbsolute(p) || /^[A-Za-z]:\//.test(p) ? null : p; +} + +/** A dependency the lockfile already covers, or that is not a file in this repo at all. */ +export function isCoveredElsewhere(rel) { + if (rel === null) return true; + // `isBuiltin`, not "has no slash and no dot": that heuristic also matched a top-level directory + // like `src`, which the convention guards list — dropping the one read that sees a new file. + if (isBuiltin(rel)) return true; + if (rel.startsWith('node:')) return true; + if (rel.startsWith('node_modules/') || rel.includes('/node_modules/')) return true; + if (rel.startsWith('.git/')) return true; + return false; +} + +export function alwaysRuns(source) { + return ALWAYS_RUN_SOURCE.some((re) => re.test(source)); +} + +const RESOLVABLE_EXTS = ['ts', 'tsx', 'mts', 'cts', 'js', 'jsx', 'mjs', 'cjs', 'json']; + +/** + * Paths whose APPEARANCE would change what an import of `rel` resolves to. `./foo` resolving to + * `foo/index.ts` is overtaken by a new `foo.ts`; `foo.ts` is overtaken by an extension ahead of it. + * Neither edits a file already in the dependency list, so without these the key cannot see it. + * Only these siblings, not the whole directory: a listing would invalidate every test near any + * new file. + */ +export function shadowCandidates(rel) { + const m = /^(.*?)([^/]+)\.([^./]+)$/.exec(rel); + if (!m) return []; + const [, dir, name, ext] = m; + if (!RESOLVABLE_EXTS.includes(ext)) return []; + const out = RESOLVABLE_EXTS.filter((e) => e !== ext).map((e) => `${dir}${name}.${e}`); + if (name === 'index' && dir) { + const parent = dir.replace(/\/$/, ''); + for (const e of RESOLVABLE_EXTS) out.push(`${parent}.${e}`); + } + return out; +} + +/** Transitive imports in one vite environment's module graph. null when the root is absent. */ +export function closureOf(graph, id) { + const root = graph.getModuleById(id); + if (!root) return null; + const out = new Set(); + const seen = new Set([root]); + const stack = [root]; + while (stack.length) { + for (const dep of stack.pop().importedModules) { + if (!dep?.id || seen.has(dep)) continue; + seen.add(dep); + out.add(dep.id); + stack.push(dep); + } + } + return out; +} + +/** + * What a path looks like right now, as a string that changes exactly when the path does: a file's + * content hash, a directory's sorted listing, or its absence. A directory counts because a test + * that lists one (every convention guard does) depends on which files are in it. + */ +export function makeFingerprinter(root) { + const memo = new Map(); + return (rel) => { + if (memo.has(rel)) return memo.get(rel); + const abs = join(root, rel); + let fp; + try { + const st = statSync(abs); + fp = st.isDirectory() ? `dir:${sha(listTree(abs).join('\n'))}` : `file:${sha(readFileSync(abs))}`; + } catch { + fp = 'missing'; + } + memo.set(rel, fp); + return fp; + }; +} + +/** + * A directory's whole subtree, not its top level: `readdirSync(dir, { recursive: true })` is one + * recorded call, and a file added three levels down changes what it returns. + */ +function listTree(abs) { + const out = []; + const walk = (d, prefix) => { + for (const e of readdirSync(d, { withFileTypes: true })) { + if (e.name === 'node_modules' || e.name === '.git') continue; + const rel = prefix ? `${prefix}/${e.name}` : e.name; + out.push(e.isDirectory() ? `${rel}/` : rel); + if (e.isDirectory()) walk(join(d, e.name), rel); + } + }; + walk(abs, ''); + return out.sort(); +} + +function workspaceManifests(root) { + const out = []; + for (const ws of WORKSPACE_DIRS) { + let names = []; + try { + names = readdirSync(join(root, ws)); + } catch { + continue; + } + for (const n of names.sort()) out.push(`${ws}/${n}/package.json`); + } + return out; +} + +export function globalSalt(root, vitestVersion, fingerprint = makeFingerprinter(root)) { + const own = OWN_CODE.map((f) => { + try { + return `${f}:${sha(readFileSync(join(HERE, f)))}`; + } catch { + return `${f}:missing`; + } + }); + return sha( + JSON.stringify([ + CACHE_FORMAT, + process.version, + process.platform, + vitestVersion, + GLOBAL_INPUTS.map((f) => [f, fingerprint(f)]), + workspaceManifests(root).map((f) => [f, fingerprint(f)]), + own, + ]) + ); +} + +/** The key. `entries` is every repo-relative path the file depends on; order does not matter. */ +export function keyFor({ salt, project, testRel, entries, fingerprint }) { + const deps = [...new Set(entries)].filter((rel) => !isCoveredElsewhere(rel)).sort(); + const parts = deps.map((rel) => `${rel}\0${fingerprint(rel)}`); + // Only the shadow candidates that EXIST, so the common case (none) adds nothing to fingerprint. + const shadows = [...new Set(deps.flatMap(shadowCandidates))] + .filter((rel) => fingerprint(rel) !== 'missing') + .sort(); + return sha([salt, project, testRel, fingerprint(testRel), ...parts, '--shadows--', ...shadows].join('\n')); +} + +/** + * Whether `rel` was modified at or after `sinceMs`. A directory counts as modified when any + * directory in its subtree was — that is what moves when a file is added, removed or renamed. + * Used to refuse recording a pass whose inputs changed while the run was in flight: the key is + * computed from disk at the END of a run, and an agent editing during a queued suite would + * otherwise have its edit recorded as the version that passed. Measured by review: exactly that. + */ +export function changedSince(root, rel, sinceMs) { + const abs = join(root, rel); + let st; + try { + st = statSync(abs); + } catch { + // Absent now. If it was deleted or renamed DURING the run, the test ran with it and the key + // would record its absence — review confirmed that as a false skip. A removal moves the mtime of + // the NEAREST SURVIVING ancestor, so ask that one. Not "parent missing means changed": every + // test probes `__snapshots__/.snap` in a directory that usually never existed, and that + // reading refused every record in the repo. + let parent = rel; + for (;;) { + parent = parent.includes('/') ? parent.slice(0, parent.lastIndexOf('/')) : '.'; + try { + const p = statSync(join(root, parent)); + return p.mtimeMs >= sinceMs || p.ctimeMs >= sinceMs; + } catch { + if (parent === '.') return true; + } + } + } + if (st.mtimeMs >= sinceMs || st.ctimeMs >= sinceMs) return true; + if (!st.isDirectory()) return false; + for (const e of readdirSync(abs, { withFileTypes: true })) { + if (!e.isDirectory() || e.name === 'node_modules' || e.name === '.git') continue; + if (changedSince(root, `${rel}/${e.name}`, sinceMs)) return true; + } + return false; +} + +export const identity = (project, testRel) => sha(`${project}\0${testRel}`); + +// ------------------------------------------------------------------------------------------ store + +export function cacheDir(root) { + if (process.env.CIVITAI_TEST_CACHE_DIR) return process.env.CIVITAI_TEST_CACHE_DIR; + // The COMMON git dir is shared by every worktree, so one tree's green result is reusable by + // another, and it is never inside a checkout's own files. + const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root }) + .toString() + .trim(); + return join(isAbsolute(common) ? common : join(root, common), 'civitai-test-cache', `v${CACHE_FORMAT}`); +} + +/** + * Every recorded pass for one test file, newest first. More than one so two worktrees on different + * versions of the same code can BOTH stay fast, instead of evicting each other's record. + */ +export function recordsFor(dir, project, testRel) { + const d = join(dir, 'rec', identity(project, testRel)); + if (!existsSync(d)) return []; + const out = []; + for (const name of readdirSync(d)) { + if (!name.endsWith('.json')) continue; + try { + out.push({ ...JSON.parse(readFileSync(join(d, name), 'utf8')), mtime: statSync(join(d, name)).mtimeMs }); + } catch { + /* a half-written or foreign file is not a record */ + } + } + return out.sort((a, b) => b.mtime - a.mtime); +} + +export function writeRecord(dir, project, testRel, record) { + const d = join(dir, 'rec', identity(project, testRel)); + mkdirSync(d, { recursive: true }); + const final = join(d, `${record.key}.json`); + const tmp = `${final}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(record)); + renameSync(tmp, final); + const all = readdirSync(d).filter((n) => n.endsWith('.json')); + if (all.length > RECORDS_PER_TEST) { + const byAge = all + .map((n) => ({ n, t: statSync(join(d, n)).mtimeMs })) + .sort((a, b) => a.t - b.t); + for (const { n } of byAge.slice(0, all.length - RECORDS_PER_TEST)) rmSync(join(d, n), { force: true }); + } +} + +export const trippedPath = (dir) => join(dir, 'TRIPPED.json'); + +/** A marker that exists but cannot be parsed still means tripped — the safe reading of a half-write. */ +export function tripped(dir) { + const p = trippedPath(dir); + if (!existsSync(p)) return null; + try { + return JSON.parse(readFileSync(p, 'utf8')); + } catch { + return { at: 'unknown (marker unreadable)', falseSkips: [] }; + } +} + +export function writeTripped(dir, body) { + const p = trippedPath(dir); + const tmp = `${p}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(body, null, 2)); + try { + renameSync(tmp, p); + } catch { + // Renaming over an existing marker can EPERM on Windows. Already tripped is still tripped; + // write in place rather than abort before the offending records are forgotten. + writeFileSync(p, JSON.stringify(body, null, 2)); + rmSync(tmp, { force: true }); + } +} + +/** Every record of a file — used on a false skip, so clearing TRIPPED cannot revive the same skip. */ +export function forget(dir, project, testRel) { + rmSync(join(dir, 'rec', identity(project, testRel)), { recursive: true, force: true }); +} diff --git a/scripts/test-cache/fs-tracker.mjs b/scripts/test-cache/fs-tracker.mjs new file mode 100644 index 0000000000..bf5b1ed6db --- /dev/null +++ b/scripts/test-cache/fs-tracker.mjs @@ -0,0 +1,87 @@ +/** + * Setup file: records every path a test file touches through `fs`, so a test whose inputs are + * files — a fixture, or a whole source tree read as text by a convention guard — can be keyed on + * them. Without it those 224 files (5.9% of modelled worker time) would have to run every time. + * + * Patching the CommonJS `fs` object and then calling `syncBuiltinESMExports` is what makes this + * reach `import { readFileSync } from 'node:fs'` too — measured on a fixture: the ESM named import + * went through the wrapper. Loaded only when the cache is on; it is inert otherwise. + * + * The paths leave the worker as the file task's `meta`, which vitest serialises to the main process + * where the reporter reads it. A per-file Set is correct because the unit projects isolate every + * file in its own process; were that ever turned off, reads would pool across files, which only + * makes keys stricter. + */ + +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { afterAll } from 'vitest'; + +import { mode } from './core.mjs'; + +if (mode() !== 'off') { + const require = createRequire(import.meta.url); + const fs = require('node:fs'); + const reads = new Set(); + + const note = (p) => { + if (typeof p === 'string') reads.add(p); + else if (p instanceof URL) reads.add(p.href); + else if (Buffer.isBuffer(p)) reads.add(p.toString()); + }; + + const wrap = (obj, name) => { + const orig = obj[name]; + if (typeof orig !== 'function') return; + const wrapper = function (p, ...rest) { + note(p); + return orig.call(this, p, ...rest); + }; + // Carry the original's own properties: `fs.realpathSync.native` lives on the function, and + // next's lib/realpath.js calls it off-Windows. Dropping it broke that path with the cache on. + Object.assign(wrapper, orig); + if (typeof orig.native === 'function') { + const native = orig.native; + wrapper.native = function (p, ...rest) { + note(p); + return native.call(this, p, ...rest); + }; + } + obj[name] = wrapper; + }; + + for (const name of [ + 'readFileSync', + 'readFile', + 'readdirSync', + 'readdir', + 'statSync', + 'lstatSync', + 'stat', + 'lstat', + 'existsSync', + 'opendirSync', + 'opendir', + 'createReadStream', + 'accessSync', + 'access', + 'openSync', + 'open', + 'realpathSync', + 'realpath', + 'readlinkSync', + 'readlink', + ]) { + wrap(fs, name); + } + for (const name of ['readFile', 'readdir', 'stat', 'lstat', 'opendir', 'open', 'access', 'realpath', 'readlink']) { + wrap(fs.promises, name); + } + syncBuiltinESMExports(); + + // The first argument must be an object pattern — vitest 4 parses hook signatures as fixtures and + // rejects anything else — and the file's suite arrives second. + // eslint-disable-next-line no-empty-pattern + afterAll(({}, suite) => { + suite.meta.testCacheReads = [...reads]; + }); +} diff --git a/scripts/test-cache/reporter.mjs b/scripts/test-cache/reporter.mjs new file mode 100644 index 0000000000..c81abd2233 --- /dev/null +++ b/scripts/test-cache/reporter.mjs @@ -0,0 +1,250 @@ +/** + * Records, AFTER a run, which test files passed and what they depended on — and checks the cache's + * own work. Pairs with scripts/test-cache/sequencer.mjs, which reads these records before the run. + * + * Dependencies come from vite's module graph, not `diagnostic().importDurations`: measured on a + * fixture, importDurations recorded a static import and MISSED an `await import()` made inside a + * test body, while the graph caught both, cold and warm. File reads come from + * scripts/test-cache/fs-tracker.mjs via the file task's meta. + * + * The check: every file the sequencer found unchanged but ran anyway — all of them in `shadow` + * mode, the random sample in `on` — was predicted to pass. One that fails is a FALSE SKIP: the key + * missed a dependency. The reporter trips the cache, so every later run executes in full until a + * human looks, and deletes that file's records, so clearing the trip cannot revive the same skip. + * + * 🔴 This file must never change a run's outcome. Everything is caught and logged, to stderr. + */ + +import { appendFileSync, mkdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import * as core from './core.mjs'; + +/** Every case ran and passed. A skipped case means part of the file never ran. */ +export function fullyPassed(testModule) { + if (testModule.state() !== 'passed') return false; + for (const test of testModule.children.allTests()) { + if (test.result().state !== 'passed') return false; + } + return true; +} + +const log = (msg) => console.error(msg); + +export default class TestCacheReporter { + onInit(vitest) { + this.vitest = vitest; + this.startedAt = Date.now(); + } + + onTestRunEnd(testModules, unhandledErrors = [], reason) { + try { + this.record(testModules, unhandledErrors, reason); + } catch (err) { + log(`[test-cache] recorder failed, run unaffected: ${err?.stack ?? err}`); + } + } + + record(testModules, unhandledErrors, reason) { + const mode = core.mode(); + if (mode === 'off') return; + const state = globalThis.__civitaiTestCache ?? { + hits: new Set(), + sampled: new Set(), + skipped: [], + startedAt: this.startedAt, + }; + const root = this.vitest.config.root; + const dir = core.cacheDir(root); + mkdirSync(dir, { recursive: true }); + + const rows = testModules.map((m) => { + const d = m.diagnostic(); + const passed = fullyPassed(m); + const wasHit = state.hits.has(`${m.project.name}\0${m.moduleId}`); + return { + m, + file: core.toRel(m.moduleId, root), + project: m.project.name, + ms: (d.prepareDuration ?? 0) + (d.setupDuration ?? 0) + (d.collectDuration ?? 0) + (d.duration ?? 0), + passed, + wasHit, + falseSkip: wasHit && !passed, + why: null, + }; + }); + + // The tripwire FIRST, before anything that can throw: a failure while recording must not be + // what stops a false skip from being reported. + const falseSkips = rows.filter((r) => r.falseSkip); + if (falseSkips.length) { + core.writeTripped(dir, { + at: new Date().toISOString(), + root, + mode, + falseSkips: falseSkips.map((r) => r.file), + }); + for (const r of falseSkips) { + try { + core.forget(dir, r.project, r.file); + } catch (err) { + log(`[test-cache] could not delete records for ${r.file}: ${err?.message ?? err}`); + } + } + log( + `[test-cache] 🔴 FALSE SKIP — the cache predicted these would pass and they did not:\n` + + falseSkips.map((r) => ` ${r.file}`).join('\n') + + `\n Their records are deleted and the cache is TRIPPED: every run executes in full until ` + + `${core.trippedPath(dir)} is removed.` + ); + } + + // An unhandled error fails the run without failing any module, so no module's "passed" can be + // trusted — measured by review: a leaked rejection left its file `passed`, got recorded, and the + // next run skipped it green. Same for an interrupted run. + // Narrowed to the files vitest attributes the errors to (VITEST_TEST_PATH, verified present on + // a leaked rejection), so one flaky leak does not stop the whole suite recording. An error with + // no attribution still blocks everything. + const errorFiles = new Set(); + let unattributed = false; + for (const e of unhandledErrors ?? []) { + if (e?.VITEST_TEST_PATH) errorFiles.add(core.toRel(e.VITEST_TEST_PATH, root)); + else unattributed = true; + } + const runBlock = unattributed + ? 'unattributed unhandled error in the run' + : reason === 'interrupted' + ? 'run interrupted' + : this.vitest.config.testNamePattern + ? 'name-filtered run' + : null; + + const fingerprint = core.makeFingerprinter(root); + const salt = core.globalSalt(root, this.vitest.version, fingerprint); + const opaqueMemo = new Map(); + const opaqueSource = (rel) => { + if (!opaqueMemo.has(rel)) { + let bad = false; + try { + bad = core.alwaysRuns(readFileSync(join(root, rel), 'utf8')); + } catch { + /* unreadable: its fingerprint records that */ + } + opaqueMemo.set(rel, bad); + } + return opaqueMemo.get(rel); + }; + + for (const row of rows) { + try { + const block = + runBlock ?? (errorFiles.has(row.file) ? 'unhandled error attributed to this file' : null); + this.recordOne(row, { root, dir, fingerprint, salt, runBlock: block, since: state.startedAt, opaqueSource }); + } catch (err) { + row.why = `record failed: ${err?.code ?? err?.message ?? err}`; + } + } + + const skipped = state.skipped?.length ?? 0; + const summary = { + at: new Date().toISOString(), + mode, + root, + bailed: state.bailed ?? null, + runBlock, + total: rows.length + skipped, + ran: rows.length, + skipped, + hitsRan: rows.filter((r) => r.wasHit).length, + recorded: rows.filter((r) => r.recorded).length, + ranMs: Math.round(rows.reduce((n, r) => n + r.ms, 0)), + hitMs: Math.round(rows.filter((r) => r.wasHit).reduce((n, r) => n + r.ms, 0)), + falseSkips: falseSkips.map((r) => r.file), + notRecorded: tally(rows.filter((r) => r.why).map((r) => r.why.replace(/:.*$/, ''))), + wallMs: Date.now() - (state.startedAt ?? this.startedAt), + }; + appendFileSync(join(dir, 'ledger.jsonl'), JSON.stringify(summary) + '\n'); + + const verify = + mode === 'shadow' + ? `${summary.hitsRan} unchanged file(s) ran (shadow mode skips nothing)` + : `${summary.hitsRan} unchanged file(s) re-run to verify`; + log( + `[test-cache] ${summary.total} test files: ${summary.ran} ran, ${skipped} skipped as unchanged ` + + `since they last passed. ${verify}; false skips: ${falseSkips.length}.` + ); + } + + recordOne(row, { root, dir, fingerprint, salt, runBlock, since, opaqueSource }) { + const { m, project, file: testRel } = row; + if (!project.startsWith('unit')) return void (row.why = 'not a unit project'); + if (!row.passed) return void (row.why = 'did not fully pass'); + if (runBlock) return void (row.why = runBlock); + if (testRel === null) return void (row.why = 'test file outside the repo'); + const reads = m.meta()?.testCacheReads; + // No tracker, no record: without it, a read made by a helper module is invisible. + if (!Array.isArray(reads)) return void (row.why = 'file reads not tracked'); + + // Whichever environment transformed the file. A `// @vitest-environment happy-dom` test goes + // through the web transform, so its modules are in `client`, not `ssr` — measured: such a file + // was absent from the ssr graph and fell out of the cache entirely. + const graph = Object.values(m.project.vite?.environments ?? {}) + .map((env) => env.moduleGraph) + .find((g) => g?.getModuleById(m.moduleId)); + if (!graph) return void (row.why = 'test file not in module graph'); + + const ids = new Set(core.closureOf(graph, m.moduleId)); + for (const setup of m.project.config.setupFiles ?? []) { + // The fs tracker is instrumentation, not an input: its closure (this cache's own code, which + // imports child_process) is covered by the salt. Walking it marked EVERY test as reaching a + // child process and recorded nothing at all — measured, which made every control vacuous. + // Exactly the tracker, not its directory: the end-to-end fixture's own setup file lives under + // scripts/test-cache/ too, and a directory-wide exclusion silently skipped scanning it. + if (core.toRel(setup, root) === 'scripts/test-cache/fs-tracker.mjs') continue; + const s = core.closureOf(graph, setup); + if (s === null) return void (row.why = 'setup file not in module graph'); + ids.add(setup); + for (const id of s) ids.add(id); + } + const entries = new Set([testRel]); + for (const id of ids) entries.add(core.toRel(id, root)); + for (const p of reads) entries.add(core.toRel(p, root)); + const kept = [...entries].filter((rel) => !core.isCoveredElsewhere(rel)); + + // A module in the graph that is gone now was deleted during the run: the test ran WITH it. + const graphRels = new Set([...ids].map((id) => core.toRel(id, root))); + const vanished = kept.find((rel) => graphRels.has(rel) && fingerprint(rel) === 'missing'); + if (vanished) return void (row.why = `imported module gone: ${vanished}`); + + // A spawned process or worker reads what it likes, and a computed import is invisible to the + // graph — anywhere in the closure, not only in the test file. Measured by review: a helper's + // `import(/* @vite-ignore */ file)` loaded a module the key never saw. + for (const rel of kept) { + if (/\.[cm]?[jt]sx?$/.test(rel) && opaqueSource(rel)) { + return void (row.why = `spawns / computed import / glob in ${rel}`); + } + } + // Key FIRST, then the change check, so an edit landing between the two is caught by the check + // instead of being fingerprinted after a check that already passed. + const key = core.keyFor({ salt, project, testRel, entries: kept, fingerprint }); + // The key is computed from disk at the END of the run. An input modified after the run + // started is not what ran — recording it would certify the edited version as passing. + // Memoised per run: `since` is fixed and every fingerprint is taken before its first check. + // Unmemoised it measured ~19s of synchronous work at the end of a full run. + this.movedMemo ??= new Map(); + const moved = kept.find((rel) => { + if (!this.movedMemo.has(rel)) this.movedMemo.set(rel, core.changedSince(root, rel, since - 2000)); + return this.movedMemo.get(rel); + }); + if (moved) return void (row.why = `changed during the run: ${moved}`); + + core.writeRecord(dir, project, testRel, { key, entries: kept, at: new Date().toISOString(), ms: row.ms }); + row.recorded = true; + } +} + +function tally(values) { + const out = {}; + for (const v of values) out[v] = (out[v] ?? 0) + 1; + return out; +} diff --git a/scripts/test-cache/sequencer.mjs b/scripts/test-cache/sequencer.mjs new file mode 100644 index 0000000000..ba7a7aff0b --- /dev/null +++ b/scripts/test-cache/sequencer.mjs @@ -0,0 +1,114 @@ +/** + * Decides, BEFORE a run starts, which test files the cache lets it skip. Vitest runs exactly the + * list `sort()` returns — measured: a sequencer that dropped a deliberately failing file left the + * run green with that file never executed. + * + * A file is skippable when one of its recorded passes still matches: the record lists every path + * the file depended on last time it passed, and re-fingerprinting those paths now reproduces the + * recorded key. Reusing the OLD list is sound for static imports and tracked file reads, because + * gaining one means editing a file already in the list. It is NOT sound for a computed dynamic + * import or a spawned process; the reporter refuses to record any file whose graph can reach one. + * + * In `on` mode a random sample of the hits runs anyway. If a sampled hit then fails, the key missed + * something; the reporter trips the cache off and says so. The sample is the standing check that + * the cache is still telling the truth — without it a false skip is a green run nobody can see. + */ + +import { BaseSequencer } from 'vitest/node'; + +import * as core from './core.mjs'; + +const parsedRate = Number(process.env.CIVITAI_TEST_CACHE_SAMPLE); +// A malformed rate must not become NaN: `Math.random() < NaN` is never true, which would switch the +// sampling — the tripwire's only input in `on` mode — silently off. +const SAMPLE_RATE = + process.env.CIVITAI_TEST_CACHE_SAMPLE !== undefined && Number.isFinite(parsedRate) ? parsedRate : 0.05; + +export const runKey = (spec) => `${spec.project.name}\0${spec.moduleId}`; + +export default class TestCacheSequencer extends BaseSequencer { + async sort(files) { + const state = { + mode: core.mode(), + // Recorded inputs modified after this instant were not what ran; the reporter refuses them. + startedAt: Date.now(), + total: files.length, + skipped: [], + hits: new Set(), + sampled: new Set(), + tripped: null, + bailed: null, + }; + globalThis.__civitaiTestCache = state; + + if (state.mode === 'off') return super.sort(files); + // A name filter runs part of each file, so a partial run must not be traded for a skip. + if (this.ctx.config.testNamePattern) state.bailed = 'name filter'; + // A run that names files asked for those files. Only a whole-suite run is the cache's to trim. + else if (this.ctx.filenamePattern?.length && !process.env.CIVITAI_TEST_CACHE_ALLOW_FILTERS) + state.bailed = 'file filter'; + // Without the reporter nothing records and nothing checks the sample, so skipping would be + // unobserved. It is loaded by the queue on the command line; a hand-exported env var is not. + else if (!(this.ctx.reporters ?? []).some((r) => r?.constructor?.name === 'TestCacheReporter')) + state.bailed = 'cache reporter not loaded'; + if (state.bailed) { + console.error(`[test-cache] not skipping anything: ${state.bailed}.`); + return super.sort(files); + } + + try { + const root = this.ctx.config.root; + const dir = core.cacheDir(root); + state.tripped = core.tripped(dir); + const fingerprint = core.makeFingerprinter(root); + const salt = core.globalSalt(root, this.ctx.version, fingerprint); + + const keep = []; + for (const spec of files) { + const project = spec.project.name; + const testRel = core.toRel(spec.moduleId, root); + const hit = + project.startsWith('unit') && + testRel !== null && + core + .recordsFor(dir, project, testRel) + .some((rec) => core.keyFor({ salt, project, testRel, entries: rec.entries, fingerprint }) === rec.key); + + if (!hit) { + keep.push(spec); + continue; + } + state.hits.add(runKey(spec)); + if (state.mode === 'shadow' || state.tripped || Math.random() < SAMPLE_RATE) { + if (state.mode === 'on' && !state.tripped) state.sampled.add(runKey(spec)); + keep.push(spec); + continue; + } + state.skipped.push(testRel); + } + + // stderr, not stdout: a caller's own `--reporter=json` with no outputFile writes its JSON to + // stdout, and a line from here would corrupt it. + if (state.mode === 'on') { + console.error( + state.tripped + ? `[test-cache] TRIPPED since ${state.tripped.at} — running everything. ` + + `Delete ${core.trippedPath(dir)} once the cause is understood.` + : `[test-cache] running ${keep.length} of ${files.length} test files: ` + + `${state.skipped.length} skipped as unchanged since they last passed, ` + + `${state.sampled.size} unchanged ones re-run to verify the cache.` + ); + } + // A run the cache skipped ENTIRELY must still pass. Vitest otherwise reports "No test files + // found" and exits 1 — measured on the fixture: a fully cached run exited 1 with nothing + // failing. Set only here, so a run that genuinely matched no files still fails as before. + if (keep.length === 0 && state.skipped.length > 0) this.ctx.config.passWithNoTests = true; + return super.sort(keep); + } catch (err) { + console.error(`[test-cache] could not read the cache, running everything: ${err?.stack ?? err}`); + state.skipped = []; + state.sampled = new Set(); + return super.sort(files); + } + } +} diff --git a/vitest.config.mts b/vitest.config.mts index 4b45259c3c..0731325365 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -2,6 +2,9 @@ import { defineConfig } from 'vitest/config'; import { playwright } from '@vitest/browser-playwright'; import path from 'path'; +import { mode as testCacheMode } from './scripts/test-cache/core.mjs'; +import TestCacheSequencer from './scripts/test-cache/sequencer.mjs'; + // Worker count is UNCAPPED by default — Vitest's own resolution applies untouched (`cpus - 1` in // run mode, `floor(cpus / 2)` in watch; the browser pool sizes itself at `min(12, cpus - 1)`). // @@ -175,7 +178,13 @@ const unitTestConfig = { globals: true, environment: 'node' as const, exclude: ['node_modules', 'tests/**/*'], // Exclude Playwright tests - setupFiles: ['src/__tests__/setup.ts'], + // The fs tracker records which files each test reads, for the result cache. It is loaded only + // when the queue turned the cache on (CIVITAI_TEST_CACHE) and never in CI — see scripts/test-cache. + // Tracker FIRST, so reads made while the main setup file loads are recorded too. + setupFiles: [ + ...(testCacheMode() !== 'off' ? ['scripts/test-cache/fs-tracker.mjs'] : []), + 'src/__tests__/setup.ts', + ], // Several unit tests cold-`await import(...)` a large Next API-page / service // module graph (mocked I/O, but a real ~9–16s TS transform). With the suite's // worker pool saturated, that legitimate cold transform races for CPU and @@ -438,6 +447,9 @@ export default defineConfig({ resolve: { alias }, test: { maxWorkers, + // Root-level because vitest builds ONE sequencer for the whole run. It only ever skips files in + // the unit projects, and only when the cache is on. + ...(testCacheMode() !== 'off' ? { sequence: { sequencer: TestCacheSequencer } } : {}), projects: [ // The `packages/*` suites, referenced by their OWN config files rather than // re-declared here. Until this line existed, nothing in CI invoked them: the `unit`