diff --git a/.env-example b/.env-example index 1ac5835ab6..0a9c1659e8 100644 --- a/.env-example +++ b/.env-example @@ -198,3 +198,15 @@ FLIPT_URL="" FLIPT_FETCHER_SECRET=placeholder IMAGE_SCANNER_NEW=false + +# App Blocks β€” per-app generation spend/velocity ABSOLUTE CEILINGS (incident knobs). +# πŸ”΄ These are UPPER BOUNDS, not the limit an app receives. Each app's actual +# ceilings come from its server-owned `spendTier` (+ any moderator per-app +# override); these clamp the tier table AND any override from above, so setting +# one TIGHTENS every app and can never loosen one. Unset = no extra clamp. +# Formerly BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY / BLOCK_APP_SPEND_VELOCITY_MAX_GENS β€” +# those names are DEPRECATED but still honoured (with a startup warning). +# BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY= +# BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW= +# Window (seconds) the gens-per-window ceiling is measured over. Default 60. +# BLOCK_APP_SPEND_VELOCITY_WINDOW_SECONDS= diff --git a/src/server/metrics/__tests__/app-block-cap-degrade.metrics.test.ts b/src/server/metrics/__tests__/app-block-cap-degrade.metrics.test.ts new file mode 100644 index 0000000000..5f996c9e54 --- /dev/null +++ b/src/server/metrics/__tests__/app-block-cap-degrade.metrics.test.ts @@ -0,0 +1,108 @@ +import client from 'prom-client'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + ensureRegisterAppBlockRuntimeMetrics, + recordAppCapLimitsDegrade, + type AppCapLimitsDegradeReason, +} from '../app-block-runtime.metrics'; + +/** + * The REAL prom-client side of the cap-limit degrade signal. + * + * The service-level test mocks this module, which proves the resolver CALLS the + * emitter β€” it cannot prove the emitter produces a scrapeable series with the + * right name and label. A metric name typo or a label-name mismatch would sail + * through that test and produce an alert rule that silently never fires, which + * is the exact failure class the signal exists to prevent. So this file drives + * the real registry. + */ + +const METRIC = 'civitai_app_block_cap_limits_degraded_total'; + +/** Read one `{reason}` series' current value from the default registry. */ +async function readReason(reason: string): Promise { + const metric = client.register.getSingleMetric(METRIC) as + | { get(): Promise<{ values: Array<{ labels: Record; value: number }> }> } + | undefined; + if (!metric) return Number.NaN; + const { values } = await metric.get(); + return values.find((v) => v.labels.reason === reason)?.value ?? 0; +} + +beforeEach(() => { + client.register.resetMetrics(); +}); + +describe('civitai_app_block_cap_limits_degraded_total', () => { + it('is registered on the default registry that /api/metrics scrapes', () => { + ensureRegisterAppBlockRuntimeMetrics(); + expect(client.register.getSingleMetric(METRIC)).toBeDefined(); + }); + + it.each([['db_error'], ['missing_row']] as Array<[AppCapLimitsDegradeReason]>)( + 'increments the `%s` series', + async (reason) => { + const before = await readReason(reason); + recordAppCapLimitsDegrade(reason); + expect(await readReason(reason)).toBe(before + 1); + } + ); + + it('πŸ”΄ the two reasons are SEPARATE series β€” an operator can alert on infra alone', async () => { + recordAppCapLimitsDegrade('db_error'); + recordAppCapLimitsDegrade('db_error'); + recordAppCapLimitsDegrade('missing_row'); + + expect(await readReason('db_error')).toBe(2); + expect(await readReason('missing_row')).toBe(1); + }); + + it('πŸ”΄ DECLARES exactly one label, `reason` β€” the cardinality bound is structural', async () => { + // `missing_row` fires for ids that are by construction absent from the app + // catalog, so an app_block_id label would be seeded from an unbounded + // population β€” and prom-client retains every distinct label set in the Node + // heap forever (the --max-old-space-size exit-139 OOM class). Attribution + // lives in the log line instead. + // + // πŸ”΄ Asserted against the DECLARED `labelNames`, not against the emitted + // series. prom-client omits a declared-but-never-supplied label from the + // output, so an inspection of `values[].labels` passes happily while the + // metric is declared wide open β€” the next caller to pass an id then blows + // the cardinality budget with nothing having failed. (Verified: mutating + // labelNames to ['reason','app_block_id'] leaves a values-based check green.) + ensureRegisterAppBlockRuntimeMetrics(); + const metric = client.register.getSingleMetric(METRIC) as unknown as { + labelNames: string[]; + }; + expect([...metric.labelNames].sort()).toEqual(['reason']); + + // …and the emitted series carries only that label too. + recordAppCapLimitsDegrade('db_error'); + const emitted = client.register.getSingleMetric(METRIC) as unknown as { + get(): Promise<{ values: Array<{ labels: Record }> }>; + }; + const { values } = await emitted.get(); + expect(values.length).toBeGreaterThan(0); + for (const v of values) { + expect(Object.keys(v.labels)).toEqual(['reason']); + } + }); + + it('is idempotent to register β€” a double module import does not throw', () => { + // prom-client throws on a duplicate metric name; Next.js can import a module + // twice (hot reload / route bundling), so the get-or-create guard is what + // keeps that from taking the process down. + expect(() => { + ensureRegisterAppBlockRuntimeMetrics(); + ensureRegisterAppBlockRuntimeMetrics(); + }).not.toThrow(); + }); + + it('appears in the scrape output with its help text', async () => { + recordAppCapLimitsDegrade('missing_row'); + const scrape = await client.register.metrics(); + expect(scrape).toContain(METRIC); + expect(scrape).toContain(`${METRIC}{reason="missing_row"}`); + }); +}); diff --git a/src/server/metrics/app-block-runtime.metrics.ts b/src/server/metrics/app-block-runtime.metrics.ts index f7fceb586b..926e4f3bab 100644 --- a/src/server/metrics/app-block-runtime.metrics.ts +++ b/src/server/metrics/app-block-runtime.metrics.ts @@ -7,11 +7,13 @@ // (no new infra, no ClickHouse migration) and are scraped by the same // /api/metrics endpoint that exposes every other app metric. // -// Two signals: +// Three signals: // 1. Per-app REST RED β€” emitted from block-scope.middleware for every // block-JWT-authed /api/v1/blocks/* call. // 2. Render-failure signal β€” emitted from the /api/track/block-render beacon // route (ok at BLOCK_READY, error on a host render failure). +// 3. Cap-limit DEGRADE signal β€” emitted from app-cap-limits.service when the +// per-app spend/velocity resolver falls back to the strictest tier. // // prom-client GOTCHA: Next.js can import a module twice (hot reload / route // bundling), and prom-client throws if a metric name is registered twice. Every @@ -58,6 +60,32 @@ export type AppBlockRequestResult = 'success' | 'client_error' | 'server_error' export type AppBlockRenderResult = 'ok' | 'error'; +/** + * Why `resolveAppCapLimits` fell back to `STRICTEST_APP_CAP_LIMITS` instead of + * resolving the app's real ceilings. The two mean DIFFERENT things and an + * operator responds to them differently, which is the whole reason this is a + * label and not one undifferentiated counter: + * + * - `db_error` β€” the `app_blocks` read THREW (DB unreachable, pool + * exhausted, the override columns not yet applied in this + * environment). INFRA trouble; usually fleet-wide and + * correlated with other DB symptoms. Every app degrades at + * once. This is the page-worthy one. + * - `missing_row` β€” the read SUCCEEDED and returned nothing. There is no such + * app: a newly created app racing its first submit, an app + * deleted mid-session, or a synthetic dev id that slipped + * the caller's `claims.dev` exclusion. Scoped to ONE app, + * and a steady non-zero rate here means a real bug in an + * id-minting path, not a database problem. + * + * πŸ”΄ Both resolve to a REAL, enforced ceiling (today's shipped 5,000,000/120) β€” + * never to "uncapped". The signal exists because the degrade is otherwise + * INVISIBLE: an app silently pinned to the strictest tier looks exactly like an + * app that is simply busy, right up until its users start seeing abuse + * rejections it did not earn. + */ +export type AppCapLimitsDegradeReason = 'db_error' | 'missing_row'; + /** Known render slots. Anything else is bucketed to 'other' to bound the label. */ const KNOWN_SLOT_IDS = new Set([ 'app.page', @@ -171,6 +199,7 @@ type Bundle = { rendersTotal: Counter; customComfyActualBuzz: Histogram; customComfyWallclockSeconds: Histogram; + capLimitsDegradedTotal: Counter; }; // ── customComfy per-engine runtime/cost buckets ────────────────────────────── @@ -265,15 +294,61 @@ export function ensureRegisterAppBlockRuntimeMetrics(reg: Registry = client.regi CUSTOMCOMFY_WALLCLOCK_BUCKETS ); + // ── per-app cap-limit DEGRADE ──────────────────────────────────────────────── + // πŸ”΄ NO `app_block_id` LABEL, deliberately. `missing_row` fires precisely for + // ids that are NOT in the app catalog, so the label set would be seeded from + // exactly the unbounded population `known-app-blocks.service.ts` exists to + // clamp β€” and prom-client retains every distinct label set in the Node heap + // forever (the --max-old-space-size exit-139 OOM class). The usual clamp + // (`boundAppBlockIdLabel`) is unusable here twice over: it needs a DB read, + // which is the very thing that is broken on the `db_error` path, and a + // `missing_row` id can never be in the approved set, so it would collapse to + // 'other' in the one case an operator most wants attributed. + // + // So the split is: this counter is the ALERTABLE aggregate (2 series total), + // and the paired `console.warn` in app-cap-limits.service carries the specific + // `appBlockId` for the operator who is already looking. Alert on the metric, + // attribute from the log. + const capLimitsDegradedTotal = getOrCreateCounter( + reg, + 'civitai_app_block_cap_limits_degraded_total', + 'App Block per-app spend/velocity cap-limit resolutions that DEGRADED to the strictest tier, by reason (db_error = the app_blocks read threw, i.e. infra; missing_row = the read succeeded but there is no such app)', + ['reason'] + ); + return { requestsTotal, requestDurationSeconds, rendersTotal, customComfyActualBuzz, customComfyWallclockSeconds, + capLimitsDegradedTotal, }; } +/** + * Fail-soft emit of one per-app cap-limit DEGRADE (the resolver fell back to + * `STRICTEST_APP_CAP_LIMITS`). Called from `app-cap-limits.service`. + * + * πŸ”΄ TOTAL, like the customComfy emitters above. The thing this instruments is a + * fail-closed SAFETY path; a metrics error (registry collision, label mismatch) + * must never propagate into it and turn "degraded but still generating" into + * "generation down". The caller guards too β€” two layers, because the guarantee + * must not depend on either one alone. + * + * COST: one in-heap counter increment on an already-degraded path. The hot path + * (a warm cap-limits cache hit) never reaches here at all, and neither does a + * cache miss that RESOLVES β€” only an actual degrade emits. + */ +export function recordAppCapLimitsDegrade(reason: AppCapLimitsDegradeReason): void { + try { + const { capLimitsDegradedTotal } = ensureRegisterAppBlockRuntimeMetrics(); + capLimitsDegradedTotal.inc({ reason }); + } catch { + /* instrument-only β€” never let a metrics error touch the cap guardrail */ + } +} + /** * Fail-soft emit of the settled GPU-runtime cost (billed `actual` Buzz) for one * customComfy gen. Called from the settle service at terminal. A metrics error diff --git a/src/server/services/blocks/__tests__/app-cap-limits-degrade-signal.test.ts b/src/server/services/blocks/__tests__/app-cap-limits-degrade-signal.test.ts new file mode 100644 index 0000000000..699d37b2d3 --- /dev/null +++ b/src/server/services/blocks/__tests__/app-cap-limits-degrade-signal.test.ts @@ -0,0 +1,437 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +/** + * OBSERVABILITY of the per-app cap-limit DEGRADE path, and the ABSOLUTE-CEILING + * env rename. + * + * ── Why the degrade signal exists ──────────────────────────────────────────── + * `resolveAppCapLimits` falls back to `STRICTEST_APP_CAP_LIMITS` on a DB error + * or a missing `app_blocks` row. That behaviour is correct (never uncapped; + * never a hard deny that would turn a DB blip into a generation outage) β€” but it + * used to be SILENT, and an app pinned to the strictest ceiling is + * indistinguishable from an app that is merely busy. The first symptom would be + * that app's users hitting abuse rejections they did not earn. + * + * Three properties are load-bearing and pinned below: + * 1. The signal FIRES on a degrade, and `db_error` (infra β€” every app degrades + * at once) is DISTINGUISHABLE from `missing_row` (one app; points at an + * id-minting bug, not the database). + * 2. It does NOT fire on the happy path β€” an alert on it must mean something. + * 3. πŸ”΄ The signal is NOT a failure path. A throwing emitter must not break cap + * resolution: it stays total and still degrades to STRICTEST. + * + * ── Why the env rename ─────────────────────────────────────────────────────── + * `BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY` / `BLOCK_APP_SPEND_VELOCITY_MAX_GENS` used + * to BE the ceilings. They are now ABSOLUTE bounds that clamp the tier table AND + * any per-app override, so the old names mislead an operator mid-incident. The + * legacy names keep working (silently ignoring a set spend-guardrail var is + * unacceptable) β€” loudly, and only when the new name does not supply a value. + */ + +const { mockFindUnique, mockRecordDegrade } = vi.hoisted(() => ({ + mockFindUnique: vi.fn(async (..._a: unknown[]): Promise => null), + mockRecordDegrade: vi.fn((_reason: string): void => undefined), +})); + +vi.mock('~/server/db/client', () => ({ + dbRead: { appBlock: { findUnique: mockFindUnique } }, + dbWrite: { appBlock: { findUnique: mockFindUnique } }, +})); + +// The service dynamic-imports the metrics module, so this mock applies. Only the +// degrade emitter is stubbed; the real module is exercised separately in +// src/server/metrics/__tests__/app-block-cap-degrade.metrics.test.ts. +vi.mock('~/server/metrics/app-block-runtime.metrics', () => ({ + recordAppCapLimitsDegrade: mockRecordDegrade, +})); + +import { STRICTEST_APP_CAP_LIMITS } from '../app-cap-limits.constants'; +import { __resetAppCapLimitsCacheForTests, resolveAppCapLimits } from '../app-cap-limits.service'; + +const APP = 'apb_degrade_test'; + +function row(over: Record = {}) { + return { + spendTier: 'standard', + spendCapBuzzPerDay: null, + spendVelocityMaxGens: null, + ...over, + }; +} + +/** Reasons passed to the degrade emitter, in call order. */ +function reasons(): string[] { + return mockRecordDegrade.mock.calls.map((c) => c[0] as string); +} + +let warnSpy: ReturnType; + +beforeEach(() => { + __resetAppCapLimitsCacheForTests(); + mockFindUnique.mockReset(); + mockFindUnique.mockResolvedValue(row()); + mockRecordDegrade.mockReset(); + mockRecordDegrade.mockImplementation(() => undefined); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); +}); + +afterEach(() => { + warnSpy.mockRestore(); +}); + +describe('cap-limit degrade signal β€” it fires, and the two causes are distinguishable', () => { + it('a DB ERROR emits `db_error` (and still degrades to STRICTEST)', async () => { + mockFindUnique.mockRejectedValue(new Error('connection terminated')); + + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + + expect(reasons()).toEqual(['db_error']); + }); + + it('a MISSING ROW emits `missing_row` (and still degrades to STRICTEST)', async () => { + mockFindUnique.mockResolvedValue(null); + + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + + expect(reasons()).toEqual(['missing_row']); + }); + + it('πŸ”΄ the two reasons are DISTINCT β€” infra trouble never reads as a new app', async () => { + // The discrimination is the whole point: `db_error` is fleet-wide and + // page-worthy; `missing_row` is one app and points at an id-minting bug. A + // single undifferentiated "degraded" counter could not tell an operator + // which incident they are in. + mockFindUnique.mockRejectedValue(new Error('db down')); + await resolveAppCapLimits('apb_a'); + + __resetAppCapLimitsCacheForTests(); + mockFindUnique.mockReset(); + mockFindUnique.mockResolvedValue(null); + await resolveAppCapLimits('apb_b'); + + expect(reasons()).toEqual(['db_error', 'missing_row']); + expect(new Set(reasons()).size).toBe(2); + }); + + it('the LOG carries the specific appBlockId + reason (the metric deliberately does not)', async () => { + // The prom counter has no `app_block_id` label β€” `missing_row` fires for ids + // that are by construction NOT in the app catalog, which is exactly the + // unbounded label population that grows prom-client's heap forever. So the + // attribution an operator needs has to be in the log line. + mockFindUnique.mockRejectedValue(new Error('pool exhausted')); + await resolveAppCapLimits(APP); + + const line = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(line).toContain(APP); + expect(line).toContain('db_error'); + expect(line).toContain('pool exhausted'); + }); + + it('a MISSING ROW is logged too β€” it used to degrade with no log at all', async () => { + mockFindUnique.mockResolvedValue(null); + await resolveAppCapLimits(APP); + + const line = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(line).toContain(APP); + expect(line).toContain('missing_row'); + }); +}); + +describe('cap-limit degrade signal β€” it does NOT fire when nothing degraded', () => { + it('a resolved row emits NOTHING (an alert on this must mean something)', async () => { + mockFindUnique.mockResolvedValue(row({ spendTier: 'platform' })); + + await resolveAppCapLimits(APP); + + expect(mockRecordDegrade).not.toHaveBeenCalled(); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('an UNKNOWN TIER does not emit β€” the row resolved, the tier table just clamped it', async () => { + // Deliberate: an unrecognised tier string still resolves to STRICTEST, but + // it is NOT a lookup degradation β€” the read worked and the row exists. Only + // "we could not learn this app's limits" is signalled, so the counter stays + // a clean infra/id-minting signal rather than a mixed bag that also fires on + // data the resolver read successfully. + mockFindUnique.mockResolvedValue(row({ spendTier: 'platinum' })); + + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + + expect(mockRecordDegrade).not.toHaveBeenCalled(); + }); + + it('is bounded by the CACHE, not by submit rate β€” a burst emits ONCE', async () => { + // The chattiness bound. A degrade is cached for the 5s fallback TTL and + // concurrent misses single-flight, so the emit ceiling is + // `active_apps / fallback_TTL` per pod regardless of traffic β€” NOT one per + // submit. This is what makes a per-degrade signal affordable on a hot path. + mockFindUnique.mockRejectedValue(new Error('db down')); + + await Promise.all(Array.from({ length: 40 }, () => resolveAppCapLimits(APP))); + for (let i = 0; i < 40; i++) await resolveAppCapLimits(APP); + + expect(mockRecordDegrade).toHaveBeenCalledTimes(1); + expect(warnSpy).toHaveBeenCalledTimes(1); + }); +}); + +describe('πŸ”΄ the signal is NOT a failure path', () => { + it('a THROWING metric emitter does not break cap resolution (still STRICTEST, no throw)', async () => { + mockRecordDegrade.mockImplementation(() => { + throw new Error('prom registry exploded'); + }); + mockFindUnique.mockRejectedValue(new Error('db down')); + + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + }); + + it('a throwing emitter does not suppress the LOG (the emitters are independently guarded)', async () => { + mockRecordDegrade.mockImplementation(() => { + throw new Error('prom registry exploded'); + }); + mockFindUnique.mockResolvedValue(null); + + await resolveAppCapLimits(APP); + + expect(warnSpy).toHaveBeenCalled(); + expect(String(warnSpy.mock.calls[0]?.[0])).toContain('missing_row'); + }); + + it('a throwing CONSOLE does not suppress the METRIC, nor break resolution', async () => { + warnSpy.mockImplementation(() => { + throw new Error('stdout gone'); + }); + mockFindUnique.mockRejectedValue(new Error('db down')); + + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + expect(reasons()).toEqual(['db_error']); + }); + + it('πŸ”΄ a throwing CONSOLE does not defeat the FALLBACK CACHE (no stampede on a sick DB)', async () => { + // The subtle one, and the reason the log needs its own guard rather than + // relying on `resolveAppCapLimits`'s outer `.catch` belt. If the log throws + // unguarded, the rejection escapes `loadAppCapLimits`, the belt still + // returns STRICTEST β€” so the RETURN VALUE looks perfect β€” but `setCacheEntry` + // never ran. Every subsequent submit then re-queries, turning a DB outage + // into a query stampede against the already-sick database: precisely what + // CAP_LIMITS_FALLBACK_TTL_MS exists to prevent. The observable difference is + // the QUERY COUNT, not the limits. + warnSpy.mockImplementation(() => { + throw new Error('stdout gone'); + }); + mockFindUnique.mockRejectedValue(new Error('db down')); + + for (let i = 0; i < 20; i++) { + await expect(resolveAppCapLimits(APP)).resolves.toEqual(STRICTEST_APP_CAP_LIMITS); + } + + expect(mockFindUnique).toHaveBeenCalledTimes(1); + }); + + it('a throwing emitter on the MISSING-ROW path also stays total', async () => { + mockRecordDegrade.mockImplementation(() => { + throw new Error('boom'); + }); + mockFindUnique.mockResolvedValue(null); + + const limits = await resolveAppCapLimits(APP); + expect(limits).toEqual(STRICTEST_APP_CAP_LIMITS); + expect(limits.dailyBuzz).toBeGreaterThan(0); + expect(limits.velocityMaxGens).toBeGreaterThan(0); + }); +}); + +/** + * The absolute-ceiling env rename. The constants are module-level, so each case + * needs a fresh module registry β€” `vi.resetModules()` + a dynamic import. + */ +describe('env rename β€” BLOCK_APP_SPEND_ABSOLUTE_MAX_* (legacy names still honoured)', () => { + const NEW_DAILY = 'BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY'; + const OLD_DAILY = 'BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY'; + const NEW_GENS = 'BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW'; + const OLD_GENS = 'BLOCK_APP_SPEND_VELOCITY_MAX_GENS'; + const ENV_KEYS = [NEW_DAILY, OLD_DAILY, NEW_GENS, OLD_GENS] as const; + + const HARD_DEFAULT_DAILY = 1_000_000_000; + const HARD_DEFAULT_GENS = 100_000; + + const saved: Record = {}; + + beforeEach(() => { + for (const k of ENV_KEYS) { + saved[k] = process.env[k]; + delete process.env[k]; + } + vi.resetModules(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + vi.resetModules(); + }); + + async function loadConstants() { + return import('../app-cap-limits.constants'); + } + + it('NEITHER set β†’ the hard built-in default (no extra clamp, no warning)', async () => { + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY).toBe(HARD_DEFAULT_DAILY); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(HARD_DEFAULT_GENS); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('the NEW name is honoured', async () => { + process.env[NEW_DAILY] = '4200'; + process.env[NEW_GENS] = '77'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY).toBe(4_200); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(77); + }); + + it('the new name alone emits NO deprecation warning', async () => { + process.env[NEW_DAILY] = '4200'; + process.env[NEW_GENS] = '77'; + await loadConstants(); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('πŸ”΄ the OLD name is still honoured β€” a set spend guardrail is never silently ignored', async () => { + process.env[OLD_DAILY] = '1234'; + process.env[OLD_GENS] = '56'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY).toBe(1_234); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(56); + }); + + it('using the OLD name logs a DEPRECATION notice naming both the old and new var', async () => { + process.env[OLD_GENS] = '56'; + await loadConstants(); + const text = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(text).toContain('DEPRECATED'); + expect(text).toContain(OLD_GENS); + expect(text).toContain(NEW_GENS); + }); + + it('the deprecation notice also states that the MEANING changed to an absolute ceiling', async () => { + // The rename exists because the name misdescribes the semantics. A notice + // that only said "renamed" would leave the operator with the wrong model. + process.env[OLD_GENS] = '56'; + await loadConstants(); + const text = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(text).toContain('ABSOLUTE CEILING'); + }); + + it('πŸ”΄ the NEW name WINS when both are set', async () => { + process.env[NEW_DAILY] = '111'; + process.env[OLD_DAILY] = '999'; + process.env[NEW_GENS] = '11'; + process.env[OLD_GENS] = '99'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY).toBe(111); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(11); + }); + + it('…and says so β€” an IGNORED guardrail var must not be silent either', async () => { + process.env[NEW_GENS] = '11'; + process.env[OLD_GENS] = '99'; + await loadConstants(); + const text = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(text).toContain('IGNORED'); + expect(text).toContain(OLD_GENS); + }); + + it('an UNUSABLE new value falls through to a usable legacy value (intent beats a typo)', async () => { + process.env[NEW_GENS] = 'not-a-number'; + process.env[OLD_GENS] = '42'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(42); + }); + + it.each([ + ['UNPARSEABLE', 'not-a-number'], + ['an empty string', ''], + ['ZERO', '0'], + ['NEGATIVE', '-500'], + ['Infinity', 'Infinity'], + ['whitespace', ' '], + ])( + '%s under EITHER name falls back to the hard default (a typo can never disable a cap)', + async (_label, raw) => { + process.env[NEW_DAILY] = raw; + process.env[OLD_DAILY] = raw; + process.env[NEW_GENS] = raw; + process.env[OLD_GENS] = raw; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY).toBe(HARD_DEFAULT_DAILY); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(HARD_DEFAULT_GENS); + } + ); + + it('a legacy name set to an UNUSABLE value warns that no clamp is in force', async () => { + // The silent-typo case: the operator believes they clamped, and nothing did. + process.env[OLD_GENS] = 'oops'; + await loadConstants(); + const text = warnSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(text).toContain('NO deploy-time clamp is in force'); + }); + + it('a legacy value is FLOORED, exactly like the new name', async () => { + process.env[OLD_GENS] = '250.7'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW).toBe(250); + }); + + it('the DEPRECATED export aliases still resolve to the new values', async () => { + // Kept so pre-rename importers (and their tests) keep compiling. + process.env[NEW_DAILY] = '4200'; + process.env[OLD_GENS] = '56'; + const m = await loadConstants(); + expect(m.BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY).toBe(m.BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY); + expect(m.BLOCK_APP_SPEND_VELOCITY_MAX_GENS).toBe( + m.BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW + ); + }); + + it('πŸ”΄ a legacy value still clamps EVERY tier β€” the compat path is a real ceiling', async () => { + // The backcompat is worthless if the honoured value does not actually bind. + process.env[OLD_GENS] = '50'; + const m = await loadConstants(); + for (const tier of m.APP_SPEND_TIERS) { + expect(m.APP_SPEND_TIER_CAP_LIMITS[tier].velocityMaxGens).toBe(50); + } + expect(m.STRICTEST_APP_CAP_LIMITS.velocityMaxGens).toBe(50); + }); + + it('πŸ”΄ a legacy value also clamps a per-app OVERRIDE, not just the tier', async () => { + process.env[OLD_GENS] = '50'; + process.env[OLD_DAILY] = '1000'; + const m = await loadConstants(); + expect( + m.resolveLimitsFromRow({ + spendTier: 'platform', + spendCapBuzzPerDay: 900_000_000, + spendVelocityMaxGens: 90_000, + }) + ).toEqual({ dailyBuzz: 1_000, velocityMaxGens: 50 }); + }); + + it('a NEW-name value clamps every tier and every override identically', async () => { + process.env[NEW_GENS] = '50'; + process.env[NEW_DAILY] = '1000'; + const m = await loadConstants(); + for (const tier of m.APP_SPEND_TIERS) { + expect(m.APP_SPEND_TIER_CAP_LIMITS[tier].velocityMaxGens).toBe(50); + } + expect( + m.resolveLimitsFromRow({ + spendTier: 'platform', + spendCapBuzzPerDay: 900_000_000, + spendVelocityMaxGens: 90_000, + }) + ).toEqual({ dailyBuzz: 1_000, velocityMaxGens: 50 }); + }); +}); diff --git a/src/server/services/blocks/app-cap-limits.constants.ts b/src/server/services/blocks/app-cap-limits.constants.ts index 372308424d..8d009639a7 100644 --- a/src/server/services/blocks/app-cap-limits.constants.ts +++ b/src/server/services/blocks/app-cap-limits.constants.ts @@ -100,6 +100,16 @@ export type AppCapLimits = { velocityMaxGens: number; }; +/** + * Parse one env var as a positive integer, or `undefined` when it is unset / + * unparseable / non-finite / zero / negative β€” so a typo in a deploy env can + * never disable a cap; it falls through to a sane positive ceiling instead. + */ +function parsePositiveInt(raw: string | undefined): number | undefined { + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined; +} + /** * Parse a positive-integer env override, fail-safe. Unset / unparseable / * non-finite / zero / negative β†’ the built-in default, so a sane positive @@ -108,14 +118,72 @@ export type AppCapLimits = { * `BLOCK_APP_SPEND_*` env vars keep working exactly as documented.) */ function envPositiveInt(name: string, fallback: number): number { - const fromEnv = Number(process.env[name]); - return Number.isFinite(fromEnv) && fromEnv > 0 ? Math.floor(fromEnv) : fallback; + return parsePositiveInt(process.env[name]) ?? fallback; +} + +/** + * As `envPositiveInt`, but ALSO honours a DEPRECATED legacy env name. + * + * πŸ”΄ WHY A COMPAT PATH AT ALL. The two absolute-ceiling knobs were renamed + * because their old names lied about what they now do (see the ceiling constants + * below). `civitai-dp-prod` sets neither β€” verified 2026-07-31 β€” but these are + * SPEND GUARDRAILS, and other environments are not enumerable from here. + * Silently ignoring a value an operator deliberately set on a spend guardrail is + * the worst possible outcome of a rename: they would believe a clamp is in force + * that is not. So the legacy name keeps working, loudly. + * + * PRECEDENCE (a valid new value always wins): + * 1. `name` parses to a positive int β†’ use it. If `legacyName` is ALSO set, + * warn that it is being ignored β€” a set-but-overridden guardrail var must + * not be silent either. + * 2. else `legacyName` parses β†’ use it, and warn that it is + * deprecated. (Reached when `name` is unset OR set to an unusable value: + * an unusable value carries no operator intent, so it must not shadow one + * that does.) + * 3. else β†’ the built-in default. If + * `legacyName` was set but unusable, warn β€” otherwise a typo'd clamp is + * indistinguishable from no clamp. + */ +function envPositiveIntWithLegacy(name: string, legacyName: string, fallback: number): number { + const fromNew = parsePositiveInt(process.env[name]); + const legacyRaw = process.env[legacyName]; + const legacyPresent = legacyRaw !== undefined; + const fromLegacy = parsePositiveInt(legacyRaw); + + if (fromNew !== undefined) { + if (legacyPresent) { + // eslint-disable-next-line no-console + console.warn( + `[app-cap-limits] ${legacyName} is DEPRECATED and IGNORED here: ${name} is also set and takes precedence (using ${fromNew}). Remove ${legacyName}.` + ); + } + return fromNew; + } + + if (fromLegacy !== undefined) { + // eslint-disable-next-line no-console + console.warn( + `[app-cap-limits] ${legacyName} is DEPRECATED β€” rename it to ${name}. Honouring the legacy value ${fromLegacy} for now. πŸ”΄ Its MEANING changed: it is an ABSOLUTE CEILING that clamps every spend tier AND every per-app override, not the limit an app receives.` + ); + return fromLegacy; + } + + if (legacyPresent) { + // eslint-disable-next-line no-console + console.warn( + `[app-cap-limits] ${legacyName} (DEPRECATED) is set to an unusable value ${JSON.stringify( + legacyRaw + )} and ${name} is unset β€” falling back to the built-in default ${fallback}. NO deploy-time clamp is in force.` + ); + } + return fallback; } /** * The ceilings that were IN FORCE IN PRODUCTION before this feature: one global * pair applied to every app. Verified 2026-07-31 β€” `civitai-dp-prod` sets - * neither `BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY` nor `BLOCK_APP_SPEND_VELOCITY_MAX_GENS`, + * neither of the two absolute-ceiling env knobs below (under their new + * `BLOCK_APP_SPEND_ABSOLUTE_MAX_*` names or their deprecated legacy ones), * so the shipped defaults below were the live numbers. * * πŸ”΄ These are a COMPATIBILITY PIN, not a tuning target. They are the @@ -175,9 +243,12 @@ export const APP_CAP_OVERRIDE_MAX_VELOCITY_GENS = 100_000; * aggregate must be a generous multiple β€” the 5,000,000 `standard` tier is β‰ˆ * the aggregate spend of ~100 fully-maxed legitimate users through one app. * - * Override at deploy time with `BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY`. + * Override at deploy time with `BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY` + * (legacy name `BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY` still honoured β€” see + * `envPositiveIntWithLegacy` and the deprecated alias below). */ -export const BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY: number = envPositiveInt( +export const BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY: number = envPositiveIntWithLegacy( + 'BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY', 'BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY', APP_CAP_OVERRIDE_MAX_DAILY_BUZZ ); @@ -207,13 +278,40 @@ export const BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY: number = envPositiveInt( * knob does NOT loosen any existing app; only a moderator promoting an app's * `spendTier` does. Lowering it tightens everything. * - * Override with `BLOCK_APP_SPEND_VELOCITY_MAX_GENS`. + * Override with `BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW` (legacy name + * `BLOCK_APP_SPEND_VELOCITY_MAX_GENS` still honoured). The window is + * `BLOCK_APP_SPEND_VELOCITY_WINDOW_SECONDS` below. */ -export const BLOCK_APP_SPEND_VELOCITY_MAX_GENS: number = envPositiveInt( +export const BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW: number = envPositiveIntWithLegacy( + 'BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW', 'BLOCK_APP_SPEND_VELOCITY_MAX_GENS', APP_CAP_OVERRIDE_MAX_VELOCITY_GENS ); +/** + * ───────────────────────────────────────────────────────────────────────────── + * DEPRECATED ALIASES β€” the pre-rename export names. + * ───────────────────────────────────────────────────────────────────────────── + * Same values, kept so existing importers (and their tests) keep compiling while + * the rename settles. πŸ”΄ Prefer the `..._ABSOLUTE_MAX_...` names: they are what + * the ENV VARS are now called, so a grep for the variable an operator typed + * lands on the code that reads it β€” which is the whole point of the rename. + * + * WHY THE NAMES WERE WRONG. Before the per-app tier work these two WERE the + * ceilings β€” one global pair applied to every app, so `..._VELOCITY_MAX_GENS` + * genuinely was "the max gens". They are now an ABSOLUTE bound that clamps the + * tier table AND any per-app moderator override from above. An operator reaching + * for `..._VELOCITY_MAX_GENS` mid-incident would reasonably read it as "set the + * limit to X" when it actually means "nothing may exceed X" β€” the same string, + * two different mental models, one of which quietly does nothing if the app's + * tier is already below X. + */ +/** @deprecated Use {@link BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY}. */ +export const BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY: number = BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY; +/** @deprecated Use {@link BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW}. */ +export const BLOCK_APP_SPEND_VELOCITY_MAX_GENS: number = + BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW; + /** * The window (seconds) the velocity ceiling is measured over. Fixed bucket: * `floor(now/window)` β€” the standard fixed-window limiter. GLOBAL by design (it @@ -287,8 +385,11 @@ export const APP_SPEND_TIER_CAP_LIMITS: Readonly { + try { + const { recordAppCapLimitsDegrade } = await import( + '~/server/metrics/app-block-runtime.metrics' + ); + recordAppCapLimitsDegrade(reason); + } catch { + /* observability must never break the guardrail it observes */ + } + try { + // eslint-disable-next-line no-console + console.warn( + `[app-cap-limits] DEGRADED to the strictest tier for ${appBlockId} (reason=${reason}): ${detail}` + ); + } catch { + /* observability must never break the guardrail it observes */ + } +} + async function loadAppCapLimits( appBlockId: string ): Promise<{ limits: AppCapLimits; ttlMs: number }> { @@ -90,7 +153,11 @@ async function loadAppCapLimits( }); if (!row) { // No such app (revoked mid-session, a synthetic dev id that slipped the - // caller's `claims.dev` exclusion, …) β†’ strictest. Never uncapped. + // caller's `claims.dev` exclusion, a brand-new app racing its first + // submit, …) β†’ strictest. Never uncapped β€” but NOT silent: this is a + // one-app, read-succeeded condition, which points at an id-minting bug + // rather than at the database, so it gets its own reason. + await signalCapLimitsDegrade(appBlockId, 'missing_row', 'no app_blocks row for this id'); return { limits: STRICTEST_APP_CAP_LIMITS, ttlMs: CAP_LIMITS_FALLBACK_TTL_MS }; } return { limits: resolveLimitsFromRow(row), ttlMs: CAP_LIMITS_TTL_MS }; @@ -98,12 +165,13 @@ async function loadAppCapLimits( // DB unreachable, or the override columns not yet applied to this // environment (this DB does NOT auto-apply migrations). Either way: fall // back to the strictest tier β€” which is the ceiling that was in force - // before this feature β€” and log so ops can see it. - // eslint-disable-next-line no-console - console.warn( - `[app-cap-limits] limit lookup failed for ${appBlockId}; falling back to the strictest tier: ${ - err instanceof Error ? err.message : String(err) - }` + // before this feature β€” and signal so ops can see it. Distinct reason from + // the missing-row case above: this one is INFRA and degrades every app at + // once, so it is the alert an operator should be paged on. + await signalCapLimitsDegrade( + appBlockId, + 'db_error', + err instanceof Error ? err.message : String(err) ); return { limits: STRICTEST_APP_CAP_LIMITS, ttlMs: CAP_LIMITS_FALLBACK_TTL_MS }; } diff --git a/src/server/services/blocks/app-spend-cap.service.ts b/src/server/services/blocks/app-spend-cap.service.ts index 1900eac4ec..1a7c885c3f 100644 --- a/src/server/services/blocks/app-spend-cap.service.ts +++ b/src/server/services/blocks/app-spend-cap.service.ts @@ -87,9 +87,18 @@ import { resolveAppCapLimits } from '~/server/services/blocks/app-cap-limits.ser * any particular app gets. The per-app default (the `standard` spend tier, and * the DB default for every row) is still 5,000,000 / 120, exactly as before. * The effective per-app value is whatever `resolveAppCapLimits` returns. + * + * The `..._ABSOLUTE_MAX_...` names say that; the older `..._CAP_BUZZ_PER_DAY` / + * `..._VELOCITY_MAX_GENS` names read as "the limit" and are DEPRECATED (both the + * exports and the env vars β€” the legacy env names are still honoured, with a + * deprecation warning; see `app-cap-limits.constants.ts`). */ export { + BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY, + BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW, + /** @deprecated Use `BLOCK_APP_SPEND_ABSOLUTE_MAX_BUZZ_PER_DAY`. */ BLOCK_APP_SPEND_CAP_BUZZ_PER_DAY, + /** @deprecated Use `BLOCK_APP_SPEND_ABSOLUTE_MAX_GENS_PER_WINDOW`. */ BLOCK_APP_SPEND_VELOCITY_MAX_GENS, BLOCK_APP_SPEND_VELOCITY_WINDOW_SECONDS, } from '~/server/services/blocks/app-cap-limits.constants';