diff --git a/.claude/agents/civitai-test-review.md b/.claude/agents/civitai-test-review.md index a8325ff0ad..c1259534d3 100644 --- a/.claude/agents/civitai-test-review.md +++ b/.claude/agents/civitai-test-review.md @@ -178,9 +178,9 @@ Worked examples of both fixes: the two retry tests in ### Convention guards -31 live in `src/server/services/__tests__/no-*.test.ts`: +32 live in `src/server/services/__tests__/no-*.test.ts`: `no-agent-ground-truth-write`, `no-coerce-boolean-in-api`, `no-direct-shared-module-mock`, -`no-divergent-can-generate-derivation`, `no-divergent-paid-gate-derivation` (the feed and the search index must derive the paid badge from one helper, never two copies of the query), `no-doubled-free-slot-noun`, `no-hand-typed-redis-key-constants` (the Redis key-constant +`no-divergent-can-generate-derivation`, `no-divergent-paid-gate-derivation` (the feed and the search index must derive the paid badge from one helper, never two copies of the query), `no-divergent-safetensor-rule` (the coverage view and `checkLoadable` state the checkpoint SafeTensor rule twice and nothing executes the SQL, so the two literals and the checkpoint scoping are pinned textually), `no-doubled-free-slot-noun`, `no-hand-typed-redis-key-constants` (the Redis key-constant ratchet — hand-typed `REDIS_KEYS` in an allowlisted mock had drifted 15 times), `no-io-in-transaction`, `no-job-kind-on-remix-mint`, `no-lint-rules-script-drift`, `no-menu-target-tooltip-nesting` (a `Tooltip` INSIDE `Menu.Target` steals the ref the menu needs and @@ -203,7 +203,7 @@ fail only in a full-suite run. Five were missing when this was last audited, on wired in then. If the diff adds a guard, check it was wired into the script, and don't treat a green `test:lint-rules` as "all guards passed". -`test:lint-rules` names 36 files today. +`test:lint-rules` names 37 files today. Both numbers and the list are checked by `no-lint-rules-script-drift`, which reads the two phrasings above literally — edit the numbers, not the shapes. diff --git a/CLAUDE.md b/CLAUDE.md index eda138a851..c95d4fd2b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -211,11 +211,11 @@ Use a top-level `import type * as PromClient` — an inline `typeof import('...' **Before widening a mock, check whether the import edge is needed at all.** A failing suite may be telling you the code pulled in a dependency it doesn't want, not that the mock is too narrow, and widening it would hide that. (Bit us twice in one day, Aug 2026, on two branches; one of those three suites was fixed by extracting the helpers into their own module instead.) #### Convention guards run as tests -Several repo conventions are enforced by tests, not by eslint. 31 live in +Several repo conventions are enforced by tests, not by eslint. 32 live in `src/server/services/__tests__/no-*.test.ts` — `no-agent-ground-truth-write`, `no-coerce-boolean-in-api`, `no-direct-shared-module-mock` (the shared-mock ratchet, see `docs/testing/shared-module-mocks.md`), `no-divergent-can-generate-derivation` (coverage alone is not canGenerate — the ecosystem must also support the model TYPE, and the pair is composed only in `isGenerationEligible`), -`no-divergent-paid-gate-derivation` (the feed and the search index must derive the paid badge from one helper, never two copies of the query), `no-doubled-free-slot-noun`, `no-hand-typed-redis-key-constants` (the Redis key-constant +`no-divergent-paid-gate-derivation` (the feed and the search index must derive the paid badge from one helper, never two copies of the query), `no-divergent-safetensor-rule` (the coverage view and `checkLoadable` state the checkpoint SafeTensor rule twice and nothing executes the SQL, so the two literals and the checkpoint scoping are pinned textually), `no-doubled-free-slot-noun`, `no-hand-typed-redis-key-constants` (the Redis key-constant ratchet — hand-typed `REDIS_KEYS` in an allowlisted mock had drifted 15 times), `no-io-in-transaction`, `no-job-kind-on-remix-mint` (the remix provenance mint must sign `kind: 'mint'` — a `job` token there is spendable on the upload path, which is the free remix-gallery submission), @@ -248,7 +248,7 @@ was last audited, on 2026-08-24, and were wired in then. **Add a new guard to th you write it**, and don't read a green `test:lint-rules` as "all guards passed" without checking the directory against the script. -`test:lint-rules` names 36 files today. +`test:lint-rules` names 37 files today. The count above, the count in the list, and the list itself are what went stale three times, so `no-lint-rules-script-drift` fails when they disagree with the directory or the script. It reads two exact diff --git a/docs/features/paid-model-loading-checklist.md b/docs/features/paid-model-loading-checklist.md index 46ce559d3a..67b3354da3 100644 --- a/docs/features/paid-model-loading-checklist.md +++ b/docs/features/paid-model-loading-checklist.md @@ -146,8 +146,17 @@ every measured number are in [paid-model-loading-coverage.md](paid-model-loading model with a loadable file. The LORA/TI/VAE/LoCon/DoRA/Upscaler branch is unchanged. - [x] drop the `CoveredCheckpoint` conjunct, and allow `Diffusers` while keeping Core ML and ONNX excluded — [the numbers](paid-model-loading-coverage.md#what-changes-in-numbers) + - [x] **checkpoints require a SafeTensor weight file** — + `20260909180000_generation_coverage_next_safetensor_checkpoints`, 2026-09-09. Narrows the + 2026-09-08 view: Diffusers stays loadable for every type *except* checkpoints, and + `CoveredCheckpoint` returns as a disjunct excusing 6 auction-resident versions. + **Written, not yet applied to any environment.** - [ ] 🔴 keep `EcosystemCheckpoints` — 62 of 63 checkpoint defaults depend on it - - [x] diffed against production 2026-09-08 — nothing loses coverage; [the numbers](paid-model-loading-coverage.md#what-changes-in-numbers) + - [x] diffed against production 2026-09-08 — nothing lost coverage *at that point*; [the numbers](paid-model-loading-coverage.md#what-changes-in-numbers) + - [ ] 🔴 **2,242 covered checkpoints lose coverage when the SafeTensor migration is applied** + (33,811 -> 31,569; 834 with generation history, 6.5M lifetime generations). A narrowing, so + there is no safe window — apply it when the readers of `covered` are ready. + *Closes when:* applied to production and the covered-checkpoint count reads 31,569. - [ ] **Set `usageControl = 'ExternalGeneration'` on the 36 mislabelled API versions.** All published, none POI, coverage preserved 36/36. Mod-only to set via the app, so it is a direct DB write. @@ -239,9 +248,11 @@ the platform second. - [ ] **C11 — retire auctions.** ([868ktt5b2](https://app.clickup.com/t/868ktt5b2)) Do not scope until 868gtq1kt (splitting featuring out of auctions) has an answer — auctions do two jobs and paid loading replaces one. ~89 files under `src/`. - - [x] the `CoveredCheckpoint` conflict is resolved by removing it from coverage (Phase 1.6), so - the auction job can no longer un-cover a paid checkpoint. What remains is deciding whether - that job should keep writing rows nothing reads. + - [x] the `CoveredCheckpoint` conflict is resolved by removing it as a coverage *conjunct* + (Phase 1.6), so the auction job can no longer un-cover a paid checkpoint. It survives as a + disjunct covering 6 auction-resident versions that lack a SafeTensor file — those would lose + coverage on the next auction prune, but none is loadable, so none can have been paid for. + What remains is deciding whether that job should keep writing rows nothing else reads. --- diff --git a/docs/features/paid-model-loading-coverage.md b/docs/features/paid-model-loading-coverage.md index 93de3b0396..3d852f9368 100644 --- a/docs/features/paid-model-loading-coverage.md +++ b/docs/features/paid-model-loading-coverage.md @@ -55,7 +55,9 @@ This is the thing an earlier reading of these docs got wrong, and it inverted a | `CoveredCheckpoint` | 514 rows | Auction-won community checkpoints — a **residency proxy**. Written and pruned weekly by `handle-auctions.ts`. **This is what paid loading replaces.** (638 versions are covered *as checkpoints* — the rest come from `EcosystemCheckpoints`.) | | `EcosystemCheckpoints` | 125 | The generator's **default model per ecosystem**. **62 of the 63 checkpoint defaults are covered through it — and zero through `CoveredCheckpoint`.** Not a loophole; the registry that keeps the generator working. | -Dropping `CoveredCheckpoint` is the feature. Dropping `EcosystemCheckpoints` would remove the +Dropping `CoveredCheckpoint` **as a conjunct** is the feature — it no longer gates anything. It +survives as a *disjunct* excusing 6 auction-resident checkpoints from the SafeTensor requirement, and +is deleted when the auction stops writing rows. Dropping `EcosystemCheckpoints` would remove the default model from half the ecosystems the generator supports — see [the defaults audit](#the-defaults-audit). @@ -89,8 +91,23 @@ contributes. Earlier drafts of these docs used 514 for both, which is what made fail to add up. 155 published, licensed, standard checkpoints on supported base models were blocked **only** by file -format: 132 Diffusers, 21 Core ML, 2 ONNX. Diffusers is loadable (Justin, 2026-09-08); Core ML and -ONNX are inference-runtime formats rather than servable weights and stay excluded. +format under `GenerationCoverage`: 132 Diffusers, 21 Core ML, 2 ONNX. + +⚠️ **Diffusers was ruled loadable (Justin, 2026-09-08) and then narrowed back out for CHECKPOINTS on +2026-09-09.** The loader serves SafeTensor only, so the checkpoint branch of `GenerationCoverageNext` +now requires a SafeTensor weight file (migration +`20260909180000_generation_coverage_next_safetensor_checkpoints`) and `checkLoadable` in +`resource-load.service.ts` refuses everything else with `unsupported-format`. Diffusers remains +accepted for **every other model type** — the shared `EXISTS` is unchanged, so the Core ML / ONNX +deny-list still governs LoRA/TI/VAE/LoCon/DoRA/Upscaler. + +🔴 **The SafeTensor narrowing took 2,242 checkpoints back out.** Measured 2026-09-09 against the +production replica: covered checkpoints **33,811 -> 31,569**, total view rows **933,851 -> 931,609** +(every row of the delta a checkpoint; textual inversions unchanged at 6,299, all 514 auction rows +retained). 834 of the 2,242 have any generation history — 6.5M lifetime generations, 0.43% of all +checkpoint generation. Scoping matters: putting the rule on the shared `EXISTS` instead would have +removed 3,287 textual inversions carrying **1.39 billion** generations, more than every SafeTensor TI +combined. ### The loader population, split by bucket diff --git a/docs/features/paid-model-loading-decisions.md b/docs/features/paid-model-loading-decisions.md index b7fd95e190..a91722e138 100644 --- a/docs/features/paid-model-loading-decisions.md +++ b/docs/features/paid-model-loading-decisions.md @@ -169,6 +169,25 @@ is dead code. Removing it widens covered checkpoints by roughly two orders of ma must stay**, because 62 of 63 checkpoint defaults are covered through it and none through `CoveredCheckpoint`. See [coverage](paid-model-loading-coverage.md#the-two-tables-do-opposite-jobs). +⚠️ **Amended 2026-09-09.** `CoveredCheckpoint` is no longer a conjunct — Justin's condition holds — +but it returns as a **disjunct** in `20260909180000_generation_coverage_next_safetensor_checkpoints`, +excusing 6 auction-resident checkpoints from the new SafeTensor requirement. Auction membership no +longer decides coverage; it stands in for residency until C11 retires the job, and is deleted with it. + +### 2.7 ⚠️ Diffusers is loadable — except for checkpoints + +Justin ruled Diffusers loadable on 2026-09-08, and the first `GenerationCoverageNext` migration +removed it from the excluded-format list for every type. The loader serves **SafeTensor only**, so +the checkpoint branch was narrowed back on 2026-09-09 — 174 Diffusers checkpoints sit among the +2,242 versions that lose coverage. Diffusers is untouched for LoRA/TI/VAE/LoCon/DoRA/Upscaler. + +**Open:** whether Justin accepts the narrowing as an implementation constraint, or wants the loader +to grow Diffusers support. *Decides:* Justin. *Closes when:* he answers here, or the migration is +applied to production with the narrowing intact. + +@dev: The loader can only serve SafeTensor today. OK to drop Diffusers/GGUF/PickleTensor checkpoints +from coverage (2,242 versions, 0.43% of checkpoint generation), or should the loader learn them? + ### 2.4 ✅ The C4 webhook — not now **Closed 2026-09-08: no.** Both reasons to build it went away on the same day. diff --git a/docs/features/paid-model-loading.md b/docs/features/paid-model-loading.md index 01fbf3aaa6..d436df1644 100644 --- a/docs/features/paid-model-loading.md +++ b/docs/features/paid-model-loading.md @@ -240,15 +240,18 @@ The premise of the feature is that the generator stops being restricted to a cur 2026-09-08 this is scoped and decided; the full model, the audit and every measured number live in [paid-model-loading-coverage.md](paid-model-loading-coverage.md). In short: -- **`CoveredCheckpoint` goes away.** It is the auction's residency proxy, it has four uses and all - four are generation, and dropping it widens covered checkpoints by roughly two orders of magnitude - ([the numbers](paid-model-loading-coverage.md#what-changes-in-numbers)). +- **`CoveredCheckpoint` stops gating.** It is the auction's residency proxy, it has four uses and all + four are generation, and removing it *as a conjunct* widens covered checkpoints by roughly two + orders of magnitude ([the numbers](paid-model-loading-coverage.md#what-changes-in-numbers)). It + remains as a *disjunct* excusing 6 auction-resident checkpoints from the SafeTensor rule, and is + deleted with the auction. - **`EcosystemCheckpoints` stays.** It is the generator's default model per ecosystem — 62 of the 63 checkpoint defaults are covered through it and none through `CoveredCheckpoint`. Removing it would strip the default model from half the supported ecosystems. - **`GenerationBaseModel` stays as the gate.** It marks the base models where the orchestrator has extended checkpoint/diffuser support, i.e. where community models can run. -- **Diffusers becomes loadable**; Core ML and ONNX stay excluded. +- **Checkpoints must carry a SafeTensor weight file** (2026-09-09); GGUF, PickleTensor, Diffusers, + Core ML, ONNX and unset are all unloadable. Diffusers stays accepted for every other type. - **File-less models never touch the loader**, and "file-less" means *no loadable file*, not *no file row* — 36 API models carry a `Training Data` archive and would otherwise read as loadable. @@ -343,7 +346,9 @@ deciding anything. - **`CoveredCheckpoint` stops gating generation**; `EcosystemCheckpoints` and `GenerationBaseModel` stay. Coverage means *allowed to generate*; residency is the orchestrator's axis. - **Only base models in `GenerationBaseModel` are loadable.** Everything else is out of scope for v1. -- A checkpoint needs a **correct model file** to be loadable; file-less API models never are. +- A checkpoint needs a **SafeTensor** weight file to be loadable (2026-09-09); file-less API models + never are, and a GGUF/PickleTensor/Diffusers checkpoint gets a different refusal — + `UNLOADABLE_MESSAGES` in `resource-load.service.ts` is the single source of both. - A load that never finishes is **refunded**. - The purchase path refuses anything **not in `GenerationCoverageNext`** (composed with ecosystem type support by `isGenerationEligible`), which is how the `RentCivit` rule is enforced without diff --git a/package.json b/package.json index efb0c420d3..e1ef6518c1 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "test:packages:run": "vitest run --project '@civitai/*'", "test:apps": "vitest --project 'app:*'", "test:apps:run": "vitest run --project 'app:*'", - "test:lint-rules": "vitest run --project 'unit*' src/server/notifications/__tests__/notification-settings-polarity.test.ts src/server/schema/__tests__/track.addView.schema.test.ts src/server/services/__tests__/hub-filter-parity.test.ts src/server/services/__tests__/no-agent-ground-truth-write.test.ts src/server/services/__tests__/no-coerce-boolean-in-api.test.ts src/server/services/__tests__/no-direct-shared-module-mock.test.ts src/server/services/__tests__/no-divergent-can-generate-derivation.test.ts src/server/services/__tests__/no-divergent-paid-gate-derivation.test.ts src/server/services/__tests__/no-doubled-free-slot-noun.test.ts src/server/services/__tests__/no-hand-typed-redis-key-constants.test.ts src/server/services/__tests__/no-io-in-transaction.test.ts src/server/services/__tests__/no-job-kind-on-remix-mint.test.ts src/server/services/__tests__/no-lint-rules-script-drift.test.ts src/server/services/__tests__/no-menu-target-tooltip-nesting.test.ts src/server/services/__tests__/no-module-scope-cache.test.ts src/server/services/__tests__/no-pk-addressed-engagement-write.test.ts src/server/services/__tests__/no-server-infra-in-app-graph.test.ts src/server/services/__tests__/no-sharp-outside-native-project.test.ts src/server/services/__tests__/no-stale-moderator-route-probe.test.ts src/server/services/__tests__/no-static-html2canvas-import.test.ts src/server/services/__tests__/no-unbounded-paging-fake.test.ts src/server/services/__tests__/no-unbumped-draft-status-write.test.ts src/server/services/__tests__/no-unguarded-billable-submit.test.ts src/server/services/__tests__/no-unguarded-user-text.test.ts src/server/services/__tests__/no-unloadable-image-fixture.test.ts src/server/services/__tests__/no-unmoderated-blob-retraction.test.ts src/server/services/__tests__/no-unmuteable-comment-processor.test.ts src/server/services/__tests__/no-unpriced-default-model.test.ts src/server/services/__tests__/no-unroled-image-resource-match.test.ts src/server/services/__tests__/no-unscoped-email-verification-exemption.test.ts src/server/services/__tests__/no-untruthy-query-gate.test.ts src/server/services/__tests__/no-unverified-provenance-write.test.ts src/server/services/__tests__/no-unwrapped-knob-rotation.test.ts src/server/services/__tests__/no-wholesale-module-mock.test.ts src/server/services/__tests__/poi-checks-strip-benign-phrases.test.ts src/server/services/__tests__/video-leaderboard-badge-staging.test.ts", + "test:lint-rules": "vitest run --project 'unit*' src/server/notifications/__tests__/notification-settings-polarity.test.ts src/server/schema/__tests__/track.addView.schema.test.ts src/server/services/__tests__/hub-filter-parity.test.ts src/server/services/__tests__/no-agent-ground-truth-write.test.ts src/server/services/__tests__/no-coerce-boolean-in-api.test.ts src/server/services/__tests__/no-direct-shared-module-mock.test.ts src/server/services/__tests__/no-divergent-can-generate-derivation.test.ts src/server/services/__tests__/no-divergent-paid-gate-derivation.test.ts src/server/services/__tests__/no-divergent-safetensor-rule.test.ts src/server/services/__tests__/no-doubled-free-slot-noun.test.ts src/server/services/__tests__/no-hand-typed-redis-key-constants.test.ts src/server/services/__tests__/no-io-in-transaction.test.ts src/server/services/__tests__/no-job-kind-on-remix-mint.test.ts src/server/services/__tests__/no-lint-rules-script-drift.test.ts src/server/services/__tests__/no-menu-target-tooltip-nesting.test.ts src/server/services/__tests__/no-module-scope-cache.test.ts src/server/services/__tests__/no-pk-addressed-engagement-write.test.ts src/server/services/__tests__/no-server-infra-in-app-graph.test.ts src/server/services/__tests__/no-sharp-outside-native-project.test.ts src/server/services/__tests__/no-stale-moderator-route-probe.test.ts src/server/services/__tests__/no-static-html2canvas-import.test.ts src/server/services/__tests__/no-unbounded-paging-fake.test.ts src/server/services/__tests__/no-unbumped-draft-status-write.test.ts src/server/services/__tests__/no-unguarded-billable-submit.test.ts src/server/services/__tests__/no-unguarded-user-text.test.ts src/server/services/__tests__/no-unloadable-image-fixture.test.ts src/server/services/__tests__/no-unmoderated-blob-retraction.test.ts src/server/services/__tests__/no-unmuteable-comment-processor.test.ts src/server/services/__tests__/no-unpriced-default-model.test.ts src/server/services/__tests__/no-unroled-image-resource-match.test.ts src/server/services/__tests__/no-unscoped-email-verification-exemption.test.ts src/server/services/__tests__/no-untruthy-query-gate.test.ts src/server/services/__tests__/no-unverified-provenance-write.test.ts src/server/services/__tests__/no-unwrapped-knob-rotation.test.ts src/server/services/__tests__/no-wholesale-module-mock.test.ts src/server/services/__tests__/poi-checks-strip-benign-phrases.test.ts src/server/services/__tests__/video-leaderboard-badge-staging.test.ts", "test:component": "node scripts/test-component-run.mjs", "test:component:watch": "vitest --project component", "test:geometry": "vitest run --project geometry", diff --git a/packages/civitai-db-schema/prisma/migrations/20260909180000_generation_coverage_next_safetensor_checkpoints/migration.sql b/packages/civitai-db-schema/prisma/migrations/20260909180000_generation_coverage_next_safetensor_checkpoints/migration.sql new file mode 100644 index 0000000000..5791b1f1b5 --- /dev/null +++ b/packages/civitai-db-schema/prisma/migrations/20260909180000_generation_coverage_next_safetensor_checkpoints/migration.sql @@ -0,0 +1,129 @@ +-- GenerationCoverageNext — a checkpoint must carry a SafeTensor weight file. +-- +-- Replaces the body created by 20260908120000_generation_coverage_next. One conjunct changes; the +-- rest is reproduced verbatim so the view has a single readable definition. +-- +-- WHY: paid model loading serves a checkpoint by loading its weights on demand, and the cluster +-- loads SafeTensor only. A checkpoint with no SafeTensor file therefore cannot generate at all — +-- so `covered` must be false for it, not merely "covered but never offered a load". That is what +-- puts this in the view rather than in `hasLoadableFile` alone: the model detail page's Create +-- button reads `canGenerate`, which composes this view. +-- +-- 🔴 SCOPED TO CHECKPOINTS ON PURPOSE. The condition sits on the checkpoint disjunct, NOT on the +-- shared `EXISTS` above it, because loading is a checkpoint-only feature. Putting it in the shared +-- clause would apply it to every type, and measured on 2026-09-09 that removes 3,287 covered +-- textual inversions carrying 1.39 BILLION lifetime generations — more than every SafeTensor TI +-- combined. Embeddings ship as `.pt` (PickleTensor), are not served by the loader, and generate +-- fine. LoRA/LoCon/DoRA/VAE/Upscaler are untouched for the same reason. +-- +-- COST, measured 2026-09-09 against the production replica: +-- * 2,242 covered checkpoint versions lose coverage (33,811 -> 31,569; total view rows 933,851 -> 931,609, +-- every row of the delta a checkpoint; textual inversions unchanged at 6,299 and all 514 +-- auction rows retained) +-- * 834 of them have any generation history; 6.5M lifetime generations, 0.43% of all checkpoint +-- generation (SafeTensor checkpoints: 30,481 versions, 1,491M generations) +-- * by format, and these OVERLAP — a version can carry several files, so they sum above 2,242: +-- ~1,753 PickleTensor, ~938 GGUF, ~174 Diffusers, ~167 Other, ~7 unset, 6 `pt` +-- +-- ⚠️ THIS REVERSES PART OF "CHANGE 2" IN THE PREVIOUS MIGRATION. Diffusers was ruled loadable +-- (Justin, 2026-09-08) and removed from the excluded-format list. "SafeTensor only" excludes it +-- again for CHECKPOINTS — 174 versions. Diffusers remains accepted for every other type, because +-- the shared `EXISTS` is unchanged. Recorded here rather than silently overwritten. +-- +-- An ALLOW-list, not the deny-list the shared clause uses, and the difference is load-bearing: +-- `format` is free text (six rows carry `pt`, which no enum defines) and is frequently unset, so +-- `<> ALL (...)` cannot promise SafeTensor. `= 'SafeTensor'` is null-safe by construction — an +-- unset format fails, which is the intended direction here and the opposite of the clause above. +-- +-- CHANGE 1 IS PARTLY WALKED BACK, DELIBERATELY AND TEMPORARILY. The previous migration dropped +-- `CoveredCheckpoint` — the weekly auction's residency list — entirely. It returns here as a +-- DISJUNCT rather than the conjunct it used to be: auction membership no longer decides coverage, +-- it only excuses a checkpoint from the SafeTensor requirement, because the auction has already +-- made it resident and a resident checkpoint needs no load to generate. +-- +-- Measured 2026-09-09: `CoveredCheckpoint` holds 514 rows, all 514 already covered, all holding +-- RentCivit, all on supported base models — and exactly 6 lacking a SafeTensor file. So this +-- rescues 6 versions and changes nothing else. It is scaffolding for the auction's retirement; +-- delete this disjunct when the auction stops writing that table. +-- +-- Branch 1 (`EcosystemCheckpoints`) still bypasses this, as it bypasses the file check entirely. +-- Those are the generator's own per-ecosystem defaults; leaving them is the existing deliberate +-- choice, not an oversight. +-- +-- 🔴 NARROWING TAKES EFFECT THE MOMENT THIS RUNS. Unlike an additive change there is no safe +-- window: apply it when the readers of `covered` are ready for 2,242 fewer checkpoints. +-- +-- `checkLoadable` in src/server/services/resource-load.service.ts applies the same checkpoint-scoped +-- SafeTensor rule at purchase time with a specific reason — except it has no CoveredCheckpoint +-- escape, so the 6 versions rescued here are still refused there. Correct: they are resident, so +-- they generate without a load. See docs/features/paid-model-loading-coverage.md. + +CREATE OR REPLACE VIEW "GenerationCoverageNext" AS +SELECT + m.id AS "modelId", + mv.id AS "modelVersionId", + true AS covered +FROM "ModelVersion" mv +JOIN "Model" m ON m.id = mv."modelId" +WHERE + -- Branch 1: the generator's per-ecosystem default models. + mv.id IN (SELECT "EcosystemCheckpoints".id FROM "EcosystemCheckpoints") + + -- Branch 2: file-less external/API generation. Covered, never loadable. + OR ( + mv."usageControl" = 'ExternalGeneration'::"ModelUsageControl" + AND mv.status = 'Published'::"ModelStatus" + AND NOT m.poi + ) + + -- Branch 3: the ordinary path — licensed, scanned, on a supported base model. + OR ( + NOT m.poi + AND ( + mv.status = 'Published'::"ModelStatus" + OR m.availability = 'Private'::"Availability" + OR m."uploadType" = 'Trained'::"ModelUploadType" + ) + AND m."allowCommercialUse" && ARRAY['RentCivit'::"CommercialUse"] + AND EXISTS ( + SELECT 1 + FROM "ModelFile" mf + WHERE mf."modelVersionId" = mv.id + AND ( + ( + mf."scannedAt" IS NOT NULL + AND mf.type = ANY (ARRAY['Model'::text, 'Pruned Model'::text, 'Diffusion Model'::text, 'UNet'::text, 'Negative'::text, 'VAE'::text]) + AND COALESCE(mf.metadata ->> 'format'::text, ''::text) <> ALL (ARRAY['Core ML'::text, 'ONNX'::text]) + ) + OR (mf.metadata -> 'trainingResults'::text) IS NOT NULL + ) + ) + AND ( + mv."baseModel" IN (SELECT "GenerationBaseModel"."baseModel" FROM "GenerationBaseModel") + OR m.type = 'Upscaler'::"ModelType" + ) + AND ( + ( + m.type = 'Checkpoint'::"ModelType" + AND mv."baseModelType" = 'Standard' + AND ( + -- The loader serves SafeTensor only, so a checkpoint without one cannot be loaded... + EXISTS ( + SELECT 1 + FROM "ModelFile" mf2 + WHERE mf2."modelVersionId" = mv.id + AND mf2."scannedAt" IS NOT NULL + AND mf2.type = ANY (ARRAY['Model'::text, 'Pruned Model'::text, 'Diffusion Model'::text, 'UNet'::text, 'Negative'::text, 'VAE'::text]) + AND mf2.metadata ->> 'format'::text = 'SafeTensor' + ) + -- ...unless the weekly auction already put it in the cluster, in which case it needs no + -- load to generate. Kept INSIDE the checkpoint disjunct rather than made a fourth + -- top-level branch, so auction membership still cannot bypass the RentCivit licence, the + -- supported-base-model list or the scanned-file check the way branches 1 and 2 do. + OR mv.id IN (SELECT "CoveredCheckpoint".version_id FROM "CoveredCheckpoint") + ) + ) + OR m.type = ANY (ARRAY['LORA'::"ModelType", 'TextualInversion'::"ModelType", 'VAE'::"ModelType", 'LoCon'::"ModelType", 'DoRA'::"ModelType"]) + OR m.type = 'Upscaler'::"ModelType" + ) + ); diff --git a/src/pages/moderator/resource-load.tsx b/src/pages/moderator/resource-load.tsx index fdbafdd367..db63f657a0 100644 --- a/src/pages/moderator/resource-load.tsx +++ b/src/pages/moderator/resource-load.tsx @@ -19,6 +19,8 @@ import { Page } from '~/components/AppLayout/Page'; import { Meta } from '~/components/Meta/Meta'; import { NextLink } from '~/components/NextLink/NextLink'; import { useResourceLoadProgress } from '~/components/ResourceLoad/resource-load.utils'; +import { UNLOADABLE_MESSAGES } from '~/server/services/resource-load.service'; +import type { UnloadableReason } from '~/server/services/resource-load.service'; import { useResourceLoadStore } from '~/store/resource-load.store'; import type { TrackedResourceLoad } from '~/store/resource-load.store'; import type { ResourceLoadProgress } from '~/components/ResourceLoad/resource-load.utils'; @@ -58,11 +60,12 @@ function AvailabilityBadge({ availability }: { availability: ResourceLoadAvailab function loadBlockedReason(state: { eligible: boolean; loadable: boolean; + unloadableReason?: UnloadableReason; availability: ResourceLoadAvailability; }) { if (!state.eligible) return 'Not generatable on the site — coverage or the ecosystem does not support this model type.'; - if (!state.loadable) return 'No weight file — this runs through an external provider.'; + if (!state.loadable) return UNLOADABLE_MESSAGES[state.unloadableReason ?? 'no-weights']; if (state.availability.status === 'unsupported') return 'The cluster cannot host this resource.'; if (state.availability.status === 'unknown') return 'Could not read status from the orchestrator.'; @@ -199,7 +202,11 @@ function SubmitCard({ {state.eligible ? 'generatable' : 'not generatable'} - {state.loadable ? 'has weights' : 'no weight file'} + {state.loadable + ? 'has weights' + : state.unloadableReason === 'unsupported-format' + ? 'unsupported format' + : 'no weight file'} diff --git a/src/server/services/__tests__/no-divergent-safetensor-rule.test.ts b/src/server/services/__tests__/no-divergent-safetensor-rule.test.ts new file mode 100644 index 0000000000..80c2f2300c --- /dev/null +++ b/src/server/services/__tests__/no-divergent-safetensor-rule.test.ts @@ -0,0 +1,123 @@ +import { readFileSync, readdirSync } from 'fs'; +import path from 'path'; +import { describe, expect, it } from 'vitest'; + +/** + * "A checkpoint needs a SafeTensor weight file" is stated twice and cannot be stated once: the + * coverage view decides `canGenerate` in SQL, and `checkLoadable` refuses the purchase in + * TypeScript. Nothing executes the view — the unit suite mocks `$queryRaw` wholesale — so a change + * to one side is invisible until production disagrees with itself: a Create button on a checkpoint + * the loader then refuses, or a load offered for something coverage has dropped. + * + * This is a TEXT guard over a textual property (the same two literals appear on both sides), which + * is the one kind a text guard checks well. It does NOT prove the SQL is correct — that was + * verified by running the view body against the production replica, and the numbers are recorded in + * the migration header. + * + * 🔴 Both halves must also stay CHECKPOINT-SCOPED. Applying the rule to every model type was + * measured on 2026-09-09 as removing 3,287 covered textual inversions carrying 1.39 BILLION + * lifetime generations — embeddings ship as PickleTensor and are not served by the loader. That + * decision is the expensive one here, so it is asserted rather than left to the header comment. + */ + +const repoRoot = path.resolve(__dirname, '../../../..'); +const SERVICE = 'src/server/services/resource-load.service.ts'; +const MIGRATIONS = 'packages/civitai-db-schema/prisma/migrations'; +const VIEW_DDL = 'CREATE OR REPLACE VIEW "GenerationCoverageNext"'; + +/** + * The LAST migration that redefines the view is the one in force — anchoring on a filename would + * leave this reading a superseded body the moment the view is edited again. + */ +function currentViewMigration() { + const dir = path.join(repoRoot, MIGRATIONS); + const matches = readdirSync(dir) + .sort() + .map((name) => ({ name, file: path.join(dir, name, 'migration.sql') })) + .filter(({ file }) => { + try { + return readFileSync(file, 'utf8').includes(VIEW_DDL); + } catch { + return false; + } + }); + return matches.length + ? { + ...matches[matches.length - 1], + sql: readFileSync(matches[matches.length - 1].file, 'utf8'), + } + : null; +} + +const migration = currentViewMigration(); +const service = readFileSync(path.join(repoRoot, SERVICE), 'utf8'); + +/** The `ARRAY[...]` inside the checkpoint disjunct's own EXISTS, keyed on its `mf2` alias. */ +function migrationWeightTypes(sql: string) { + const block = /mf2\.type = ANY \(ARRAY\[([^\]]+)\]\)/.exec(sql); + return block ? [...block[1].matchAll(/'([^']+)'::text/g)].map((m) => m[1]).sort() : null; +} + +function serviceWeightTypes(text: string) { + const decl = /const LOADABLE_FILE_TYPES = \[([^\]]+)\]/.exec(text); + return decl ? [...decl[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() : null; +} + +describe('the SafeTensor rule has one meaning in SQL and TypeScript', () => { + it('a migration defining GenerationCoverageNext exists', () => { + // Without this the whole file passes vacuously once the migration is renamed or removed. + expect(migration, `no migration under ${MIGRATIONS} contains \`${VIEW_DDL}\``).not.toBeNull(); + }); + + it('the service names SafeTensor as the only loadable format', () => { + const declared = /const LOADABLE_FORMAT = '([^']+)'/.exec(service); + expect(declared, `${SERVICE} must declare LOADABLE_FORMAT`).not.toBeNull(); + expect(declared![1]).toBe('SafeTensor'); + }); + + it('the view requires the same format literal the service does', () => { + const declared = /const LOADABLE_FORMAT = '([^']+)'/.exec(service)![1]; + expect( + migration!.sql, + `${migration!.name} must require '${declared}' on the checkpoint branch` + ).toContain(`= '${declared}'`); + }); + + it('both sides accept the same weight file types', () => { + expect( + migrationWeightTypes(migration!.sql), + `${migration!.name}: no mf2.type ARRAY found — the checkpoint EXISTS may have been reshaped` + ).toEqual(serviceWeightTypes(service)); + }); + + it('the service applies the format rule to checkpoints only', () => { + expect( + service, + `${SERVICE}: the format check must be scoped to Checkpoint — see the header of ${ + migration!.name + }` + ).toMatch(/modelType === 'Checkpoint' &&[\s\S]{0,120}LOADABLE_FORMAT/); + }); + + it('the view applies the format rule inside the checkpoint disjunct, not the shared EXISTS', () => { + // The shared clause is keyed on `mf`; the checkpoint-only one on `mf2`. A SafeTensor test + // appearing against `mf` would mean the rule escaped onto every model type. + const shared = /mf\.metadata ->> 'format'::text = 'SafeTensor'/.test(migration!.sql); + expect( + shared, + `${ + migration!.name + }: SafeTensor must not be required in the shared EXISTS — that applies it ` + + `to LoRA/TextualInversion/VAE/LoCon/DoRA and drops 3,287 covered embeddings` + ).toBe(false); + + const checkpointScoped = + /m\.type = 'Checkpoint'::"ModelType"[\s\S]{0,900}mf2\.metadata ->> 'format'::text = 'SafeTensor'/.test( + migration!.sql + ); + expect( + checkpointScoped, + `${migration!.name}: the SafeTensor EXISTS must sit inside the Checkpoint disjunct` + ).toBe(true); + }); +}); diff --git a/src/server/services/__tests__/resource-load.service.test.ts b/src/server/services/__tests__/resource-load.service.test.ts index a46a1357eb..782952f60d 100644 --- a/src/server/services/__tests__/resource-load.service.test.ts +++ b/src/server/services/__tests__/resource-load.service.test.ts @@ -40,6 +40,9 @@ const version = { files: [{ type: 'Model', scannedAt: new Date(), metadata: { format: 'SafeTensor' } }], }; +/** The SafeTensor rule is checkpoint-scoped, so format cases need a checkpoint, not the LoRA above. */ +const checkpointVersion = { ...version, model: { ...version.model, type: 'Checkpoint' } }; + const versionAir = 'urn:air:sdxl:lora:civitai:42@501'; /** @@ -190,6 +193,100 @@ describe('the purchase path refuses before it submits', () => { expect(submitWorkflow).not.toHaveBeenCalled(); }); + // The loader serves SafeTensor only. These are the formats an earlier deny-list + // (`not Core ML, not ONNX`) let through, so each one fails on a revert to that shape. + it.each([ + ['PickleTensor', 'the historical .ckpt/.pt format'], + ['GGUF', 'quantised container'], + ['Diffusers', 'multi-file HF layout'], + ['Other', 'unlabelled'], + ['pt', 'free-text value no enum defines'], + [undefined, 'format never set'], + ])('refuses a %s weight file — %s', async (format) => { + orchestratorReturns({ status: 'unavailable', queuePosition: null }); + dbMock.dbRead.modelVersion.findMany.mockResolvedValue([ + { + ...checkpointVersion, + files: [{ type: 'Model', scannedAt: new Date(), metadata: format ? { format } : {} }], + }, + ]); + + // Not the external-provider copy: these versions HAVE weights, just unservable ones. + // Asserting the specific message is what stops the two reasons collapsing back into one. + await expect( + submitResourceLoad({ modelVersionId: 501, userId: 7, token: 'user-token', currencies: [] }) + ).rejects.toThrow(/only load SafeTensor files/); + expect(submitWorkflow).not.toHaveBeenCalled(); + }); + + it('reports the reason on the state, so a CTA can explain it before the user tries', async () => { + orchestratorReturns({ status: 'unavailable', queuePosition: null }); + dbMock.dbRead.modelVersion.findMany.mockResolvedValue([ + { + ...checkpointVersion, + files: [{ type: 'Model', scannedAt: new Date(), metadata: { format: 'GGUF' } }], + }, + ]); + + const [state] = await getResourceLoadState([501]); + expect(state.loadable).toBe(false); + expect(state.unloadableReason).toBe('unsupported-format'); + }); + + // The scoping the coverage view spends a paragraph on: applying the SafeTensor rule to every + // type would refuse the PickleTensor embeddings and LoRAs the view deliberately keeps covered. + it.each(['LORA', 'TextualInversion'])( + 'does not apply the SafeTensor rule to a %s', + async (type) => { + orchestratorReturns({ status: 'unavailable', queuePosition: null }); + dbMock.dbRead.modelVersion.findMany.mockResolvedValue([ + { + ...version, + model: { ...version.model, type }, + files: [{ type: 'Model', scannedAt: new Date(), metadata: { format: 'PickleTensor' } }], + }, + ]); + + const [state] = await getResourceLoadState([501]); + expect(state.loadable).toBe(true); + expect(state.unloadableReason).toBeUndefined(); + } + ); + + it('distinguishes an API model, which has no weights at all', async () => { + orchestratorReturns({ status: 'unavailable', queuePosition: null }); + dbMock.dbRead.modelVersion.findMany.mockResolvedValue([ + { + ...version, + files: [{ type: 'Training Data', scannedAt: new Date(), metadata: { format: 'Other' } }], + }, + ]); + + const [state] = await getResourceLoadState([501]); + expect(state.unloadableReason).toBe('no-weights'); + }); + + it('accepts a version whose SafeTensor sits alongside unsupported formats', async () => { + orchestratorReturns({ status: 'unavailable', queuePosition: null }); + dbMock.dbRead.modelVersion.findMany.mockResolvedValue([ + { + ...version, + files: [ + { type: 'Model', scannedAt: new Date(), metadata: { format: 'GGUF' } }, + { type: 'Model', scannedAt: new Date(), metadata: { format: 'SafeTensor' } }, + ], + }, + ]); + + await submitResourceLoad({ + modelVersionId: 501, + userId: 7, + token: 'user-token', + currencies: [], + }); + expect(submitWorkflow).toHaveBeenCalled(); + }); + it('refuses a version that does not exist', async () => { dbMock.dbRead.modelVersion.findMany.mockResolvedValue([]); diff --git a/src/server/services/resource-load.service.ts b/src/server/services/resource-load.service.ts index dfda18941b..7faa76af16 100644 --- a/src/server/services/resource-load.service.ts +++ b/src/server/services/resource-load.service.ts @@ -22,20 +22,40 @@ import { BuzzTypes } from '~/shared/constants/buzz.constants'; const PREPARE_STEP_NAME = 'prepare-resource'; /** - * A file the cluster can serve as weights. Mirrors the coverage view's accepted types minus its - * `trainingResults` disjunct — a training archive is not a weight. Core ML and ONNX are - * inference-runtime formats. + * A file the cluster can serve as weights. Types mirror the coverage view minus its + * `trainingResults` disjunct — a training archive is not a weight. + * + * Format is an allow-list because `format` is free text and frequently unset; a deny-list cannot + * promise the loader only ever sees SafeTensor. */ const LOADABLE_FILE_TYPES = ['Model', 'Pruned Model', 'Diffusion Model', 'UNet', 'Negative', 'VAE']; -const UNLOADABLE_FORMATS = ['Core ML', 'ONNX']; +const LOADABLE_FORMAT = 'SafeTensor'; -function hasLoadableFile(files: VersionForAir['files']) { - return files.some( - (f) => - !!f.scannedAt && - LOADABLE_FILE_TYPES.includes(f.type) && - !UNLOADABLE_FORMATS.includes(String(f.metadata?.format ?? '')) - ); +/** + * Why a version cannot be loaded. The two are different things to tell a user: an API model has + * nothing to load and never will, while a GGUF checkpoint has weights the cluster cannot serve. + * Collapsing them is what made the old copy claim every unloadable resource was external. + */ +export type UnloadableReason = 'no-weights' | 'unsupported-format'; + +/** Exported so the load CTA states the reason in the same words the mutation refuses it in. */ +export const UNLOADABLE_MESSAGES: Record = { + 'no-weights': 'This resource has no model file to load — it runs through an external provider.', + 'unsupported-format': + 'The generator can only load SafeTensor files, and this version does not have one.', +}; + +function checkLoadable( + files: VersionForAir['files'], + modelType: ModelType +): { loadable: true } | { loadable: false; unloadableReason: UnloadableReason } { + const weights = files.filter((f) => !!f.scannedAt && LOADABLE_FILE_TYPES.includes(f.type)); + if (!weights.length) return { loadable: false, unloadableReason: 'no-weights' }; + // Scoped to checkpoints to match the coverage view's checkpoint disjunct. Applying it to every + // type would refuse the PickleTensor embeddings and LoRAs the view deliberately keeps covered. + if (modelType === 'Checkpoint' && !weights.some((f) => f.metadata?.format === LOADABLE_FORMAT)) + return { loadable: false, unloadableReason: 'unsupported-format' }; + return { loadable: true }; } /** Each state fetch is an orchestrator grain call, so keep the fan-out bounded. */ @@ -52,8 +72,10 @@ export type ResourceLoadState = { availability: ResourceLoadAvailability; /** Coverage alone over-reports; see docs/features/paid-model-loading-coverage.md. */ eligible: boolean; - /** Whether there is a weight file to download. False for external/API models. */ + /** Whether the cluster can serve this version's weights. */ loadable: boolean; + /** Why not, when `loadable` is false — so a CTA can say so without re-deriving it. */ + unloadableReason?: UnloadableReason; }; type VersionForAir = { @@ -128,7 +150,7 @@ export async function getResourceLoadState( modelType: version.model.type, flags: version.flags, }), - loadable: hasLoadableFile(version.files), + ...checkLoadable(version.files, version.model.type), }; const response = await getModelClient({ token: env.ORCHESTRATOR_ACCESS_TOKEN, air }); @@ -199,9 +221,7 @@ async function resolveLoadable(modelVersionId: number) { 'This resource cannot be generated with on the site, so loading it would buy nothing.' ); if (!state.loadable) - throw throwBadRequestError( - 'This resource has no model file to load — it runs through an external provider.' - ); + throw throwBadRequestError(UNLOADABLE_MESSAGES[state.unloadableReason ?? 'no-weights']); const { status } = state.availability; if (status === 'unsupported')