From cde95673f296cd37c16f39d916f829cfd676e39f Mon Sep 17 00:00:00 2001 From: briant Date: Wed, 2 Sep 2026 17:26:55 -0600 Subject: [PATCH] feat(form-graph): stage the cutover behind flags and migrate v1 stored settings Server: validateInput shadow-compares the hub parse behind form-graph-shadow-parse (outcome counter + keys-only Axiom logging, never field values) and serves it behind form-graph-parse; v1 keeps running for substitution metrics and reverse compare. computedKeys on the serve path comes from the parse result's own wire-named computeds. Client: formGraphGenerator feature flag (mod) swaps GenerationTabs between GenerationFormV2 and the new FormGraphGenerator shell. On first mount the form runs a one-time partial localStorage migration (prompt/negativePrompt, outputFormat, priority, quantity, workflow, per-output ecosystem, per-family model + resources); v1 records are read, never touched. Co-Authored-By: Claude Fable 5 --- docs/form-graph-port-plan.md | 44 ++++--- .../ImageGeneration/GenerationTabs.tsx | 8 +- .../generation/BaseGenerationForm.tsx | 6 +- .../generation/FormGraphGenerator.tsx | 36 ++++++ .../__tests__/migrate-v1-storage.test.ts | 86 ++++++++++++++ .../generation/migrate-v1-storage.ts | 93 +++++++++++++++ src/server/flipt/client.ts | 4 + src/server/prom/form-graph.metrics.ts | 16 +++ src/server/services/feature-flags.service.ts | 4 + .../form-graph/__tests__/shadow-parse.test.ts | 78 ++++++++++++ .../orchestrator/form-graph/shadow-parse.ts | 112 ++++++++++++++++++ .../orchestrator/orchestration-new.service.ts | 26 +++- 12 files changed, 490 insertions(+), 23 deletions(-) create mode 100644 src/components/form-graph/generation/FormGraphGenerator.tsx create mode 100644 src/components/form-graph/generation/__tests__/migrate-v1-storage.test.ts create mode 100644 src/components/form-graph/generation/migrate-v1-storage.ts create mode 100644 src/server/prom/form-graph.metrics.ts create mode 100644 src/server/services/orchestrator/form-graph/__tests__/shadow-parse.test.ts create mode 100644 src/server/services/orchestrator/form-graph/shadow-parse.ts diff --git a/docs/form-graph-port-plan.md b/docs/form-graph-port-plan.md index 1a4377d723..5317dad337 100644 --- a/docs/form-graph-port-plan.md +++ b/docs/form-graph-port-plan.md @@ -287,26 +287,30 @@ cfgScale/steps `perModelScope` — per model version, v1's TURBO_VARIANT_ECOSYSTEMS. One localStorage record under `form-graph:generation` (`persistedStorage`, debounced, flushed on pagehide) attached in BaseGenerationForm. Layout pinned by `__tests__/persistence.test.ts`. -Not decided (Briant): migrating v1's stored values (different keys AND record -shape — v1 splits across many localStorage keys) vs a one-time reset; and v1's -"preferences survive reset" semantics (outputFormat/priority), which belongs to -the reset affordance when one exists. Store-path gotcha this surfaced: STORE +Decided (Briant, 2026-09-02): partial one-time migration, implemented in +`src/components/form-graph/generation/migrate-v1-storage.ts` — carries +prompt/negativePrompt, outputFormat, priority, quantity, workflow, per-output +ecosystem, and per-family model + resources; everything else resets. v1's +records are only read, never deleted. Store-path gotcha this surfaced: STORE state holds raw inputs (a bare-number model), so mode picks and scopes read ids via `modelIdOf` rather than `.id`. -### Phase 4 — server swap (small, high-stakes) +### Phase 4 — server swap (small, high-stakes) — BUILT, staged behind Flipt flags (2026-09-02) -1. Write one adapter in `src/server/services/orchestrator/`: a function with the exact - shape the three call sites expect from `generationGraph.safeParse`, implemented over - the ported root's `.parse` (map `notes` → whatever the substitution metrics read — - study `generation-model-substitution.metrics.ts` first; its tests must stay green). -2. Swap the call sites (`orchestration-new.service.ts`, `legacy-metadata-mapper.ts`) - behind that adapter. -3. Run the covering suites for those services, then the FULL unit suite once. - -**Closing condition:** full unit suite green (compare failing files against `main` via -stash before blaming the port); substitution-metrics tests green; committed. **Ask -Briant before this phase begins** — it changes the production submit path. +No adapter needed. `validateInput` in `orchestration-new.service.ts` is staged behind +two Flipt flags (both default off; see +`src/server/services/orchestrator/form-graph/shadow-parse.ts`): +`form-graph-shadow-parse` runs the hub parse alongside v1 and compares — outcomes +counted in `form_graph_shadow_parse_total`, divergence logged with diff KEYS only — +and `form-graph-parse` serves the hub result. The v1 parse always runs (substitution +metrics + reverse compare); dropping it belongs to Phase 6. `computedKeys` on the +serve path comes from the parse result's own `computedKeys` (wire-named computeds — +the v1 node-partition equivalent), and the substitution metrics need no mapping +because the port's `checkpoint.ts` records into `ext.modelSubstitutions` directly. +`legacy-metadata-mapper.ts`'s `getGenerationDisplayKeys` deliberately stays on the v1 +graph: its input/computed partition differs in the hub (workflow/ecosystem are +fields, not computeds), so it moves in Phase 6 with a behavior decision, not +mechanically. Flip criterion: a sustained zero on diverged/error outcomes. ### Phase 5 — client swap (large, UI) @@ -314,9 +318,11 @@ Replace `DataGraphProvider`/`useDataGraph` usage with `form-graph/react` (`useForm(rootGraph, { ext, storage: persistedStorage(...) })`, `useTypedField`, `createTypedController`). `GenerationFormProvider.tsx` is the hub; port it first, then walk the ~119 consumer files (most only consume via the provider's context and need -import/type updates, not logic changes). Storage: the adapter layout is built (see the Persistence note under the Phase 4/5 -groundwork above); the v1 stored-value migration-vs-reset decision recorded there is -the remaining Phase 5 item. +import/type updates, not logic changes). Storage: the adapter layout is built and the +v1 stored-value migration is implemented (see the Persistence note above). The swap +itself is staged: GenerationTabs mounts `FormGraphGenerator` behind the +`formGraphGenerator` feature flag (mod availability, Flipt key +`form-graph-generator`), falling back to `GenerationFormV2` when off. **Closing condition:** the generation form works end to end in the dev server (use the `/dev-server` skill; verify with `probe`), typecheck green, full suite green. This phase diff --git a/src/components/ImageGeneration/GenerationTabs.tsx b/src/components/ImageGeneration/GenerationTabs.tsx index c980ecde23..7e5a3aa678 100644 --- a/src/components/ImageGeneration/GenerationTabs.tsx +++ b/src/components/ImageGeneration/GenerationTabs.tsx @@ -24,6 +24,7 @@ import { import { SignalStatusNotification } from '~/components/Signals/SignalsProvider'; import { ScrollArea } from '~/components/ScrollArea/ScrollArea'; import { GenerationFormV2 } from '~/components/generation_v2'; +import { FormGraphGenerator } from '~/components/form-graph/generation/FormGraphGenerator'; import { ChallengeIndicator } from '~/components/Challenges/ChallengeIndicator'; import { PresetHeaderButton } from '~/components/generation_v2/preset/PresetHeaderButton'; import { useIsClient } from '~/providers/IsClientProvider'; @@ -89,7 +90,7 @@ function GenerationTabsContent({ fullScreen }: { fullScreen?: boolean }) { // Perf experiment: defer the generation-tab-switch remount to fix mobile INP. // Switching tabs swaps `View` to a DIFFERENT component, so React synchronously - // unmounts the whole GenerationFormV2 tree and mounts Queue/Feed inside the tap's + // unmounts the whole generation-form tree and mounts Queue/Feed inside the tap's // onChange handler (~1s of processing_duration counted against INP). `useDeferredValue` // moves that heavy remount off the urgent path; the SegmentedControl highlight stays on // the live `view` for instant tap feedback. startTransition does NOT work here — zustand @@ -100,7 +101,10 @@ function GenerationTabsContent({ fullScreen }: { fullScreen?: boolean }) { const deferredView = useDeferredValue(view); const contentView = deferGenTabView ? deferredView : view; - const GenerationFormComponent = GenerationFormV2; + // form-graph cutover: the new lane behind its flag; OFF is byte-identical + const GenerationFormComponent = features.formGraphGenerator + ? FormGraphGenerator + : GenerationFormV2; const tabs = useMemo( () => ({ diff --git a/src/components/form-graph/generation/BaseGenerationForm.tsx b/src/components/form-graph/generation/BaseGenerationForm.tsx index 167f8256ac..1b0619029b 100644 --- a/src/components/form-graph/generation/BaseGenerationForm.tsx +++ b/src/components/form-graph/generation/BaseGenerationForm.tsx @@ -22,6 +22,7 @@ import { AudioGenerationForm } from './AudioGenerationForm'; import { Model3dGenerationForm } from './Model3dGenerationForm'; import { FormFooter } from './FormFooter'; import { WhatIfProvider } from './WhatIfProvider'; +import { migrateV1GenerationStorage } from './migrate-v1-storage'; import { useOutputType, type GenerationStore } from './store'; /** @@ -74,7 +75,10 @@ export function BaseGenerationForm() { ] ); - const storage = useMemo(() => persistedStorage(STORAGE_KEY), []); + const storage = useMemo(() => { + migrateV1GenerationStorage(STORAGE_KEY); + return persistedStorage(STORAGE_KEY); + }, []); useEffect(() => () => storage?.dispose(), [storage]); const store = useForm(generationHub, { ext, storage }) as GenerationStore; diff --git a/src/components/form-graph/generation/FormGraphGenerator.tsx b/src/components/form-graph/generation/FormGraphGenerator.tsx new file mode 100644 index 0000000000..740e1152de --- /dev/null +++ b/src/components/form-graph/generation/FormGraphGenerator.tsx @@ -0,0 +1,36 @@ +import { useIsClient } from '~/providers/IsClientProvider'; +import { ScrollArea } from '~/components/ScrollArea/ScrollArea'; +import { GenerationProvider } from '~/components/ImageGeneration/GenerationProvider'; +import { Announcements } from '~/components/Announcements/Announcements'; +import { ResourceDataProvider } from '~/components/generation_v2/inputs/ResourceDataProvider'; + +import { BaseGenerationForm } from './BaseGenerationForm'; + +/** + * The form-graph lane's counterpart of `GenerationFormV2` — the shell + * GenerationTabs mounts when the `formGraphGenerator` flag is on. Same + * provider stack (queue state, resource data, announcements, scroll + * restore); only the form inside differs. + */ +export function FormGraphGenerator() { + const isClient = useIsClient(); + + if (!isClient) return null; + + return ( + +
+ + + + + + +
+ + ); +} diff --git a/src/components/form-graph/generation/__tests__/migrate-v1-storage.test.ts b/src/components/form-graph/generation/__tests__/migrate-v1-storage.test.ts new file mode 100644 index 0000000000..8c94e5d4b6 --- /dev/null +++ b/src/components/form-graph/generation/__tests__/migrate-v1-storage.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from 'vitest'; +import { generationHub } from '~/shared/form-graph/generation/hub.graph'; +import type { GenerationCtx } from '~/shared/data-graph/generation/context'; +import { buildV1MigrationIntent } from '../migrate-v1-storage'; + +const EXT: GenerationCtx = { + limits: { maxQuantity: 10, maxResources: 9, vidQuantity: 4 }, + user: { isMember: true, tier: 'gold' }, + flags: {}, + gateRules: [], +}; + +const V1_FIXTURE: Record = { + 'generation-graph': JSON.stringify({ + workflow: 'txt2img', + prompt: 'a fox in the snow', + negativePrompt: 'blurry', + quantity: 3, + seed: 1234, + snippets: { mode: 'random' }, + }), + 'generation-graph.preferences': JSON.stringify({ outputFormat: 'png', priority: 'high' }), + 'generation-graph.workflow.txt2img:draft': JSON.stringify({ quantity: 8 }), + 'generation-graph.output.image': JSON.stringify({ ecosystem: 'SDXL' }), + 'generation-graph.output.video': JSON.stringify({ ecosystem: 'WanVideo25T2V' }), + 'generation-graph.ecosystem.SDXL': JSON.stringify({ + model: { id: 128713, model: { type: 'Checkpoint' } }, + resources: [{ id: 555, model: { type: 'LORA' } }], + cfgScale: 7, + steps: 25, + }), +}; + +const read = (fixture: Record) => (key: string) => fixture[key] ?? null; + +describe('buildV1MigrationIntent', () => { + it('carries exactly the preserved fields, at the addresses the hub reads', () => { + const intent = buildV1MigrationIntent(read(V1_FIXTURE)); + expect(intent).toEqual({ + workflow: 'txt2img', + prompt: 'a fox in the snow', + negativePrompt: 'blurry', + quantity: 3, + outputFormat: 'png', + priority: 'high', + 'quantity@txt2img:draft': 8, + 'ecosystem@image': 'SDXL', + 'ecosystem@video': 'WanVideo25T2V', + 'model@SDXL': { id: 128713, model: { type: 'Checkpoint' } }, + 'resources@SDXL': [{ id: 555, model: { type: 'LORA' } }], + }); + }); + + it('returns undefined when v1 stored nothing', () => { + expect(buildV1MigrationIntent(() => null)).toBeUndefined(); + expect(buildV1MigrationIntent(read({ 'generation-graph': 'not json{' }))).toBeUndefined(); + }); + + it('a grouped ecosystem migrates under its group id', () => { + const intent = buildV1MigrationIntent( + read({ + 'generation-graph.ecosystem.WanVideo': JSON.stringify({ + model: { id: 999, model: { type: 'Checkpoint' } }, + }), + }) + ); + expect(intent).toEqual({ 'model@WanVideo': { id: 999, model: { type: 'Checkpoint' } } }); + }); + + it('the hub store hydrates the migrated record', () => { + const intent = buildV1MigrationIntent(read(V1_FIXTURE)); + if (!intent) throw new Error('expected an intent record'); + const store = generationHub.createStore({ + ext: EXT, + storage: { load: () => intent, save: () => undefined }, + }); + const state = store.getSnapshot().state as Record; + expect(state.workflow).toBe('txt2img'); + expect(state.prompt).toBe('a fox in the snow'); + expect(state.negativePrompt).toBe('blurry'); + expect(state.quantity).toBe(3); + expect(state.outputFormat).toBe('png'); + expect(state.priority).toBe('high'); + expect(state.ecosystem).toBe('SDXL'); + }); +}); diff --git a/src/components/form-graph/generation/migrate-v1-storage.ts b/src/components/form-graph/generation/migrate-v1-storage.ts new file mode 100644 index 0000000000..8d64c66231 --- /dev/null +++ b/src/components/form-graph/generation/migrate-v1-storage.ts @@ -0,0 +1,93 @@ +import { scopedAddress } from 'form-graph'; +import { ecosystemByKey, ecosystemGroups } from '~/shared/constants/basemodel.constants'; + +/** + * One-time carry-over of a user's v1 generation-form state into the + * form-graph store. Deliberately partial: only the fields worth preserving + * (prompt/negativePrompt, outputFormat, priority, quantity, workflow, + * per-output ecosystem, per-family model + resources) — everything else + * starts fresh. Values are copied raw; the hub's input schemas validate them + * on first parse, so a stale or malformed v1 value degrades to the default + * rather than breaking the form. + * + * v1's records are left untouched: while the flag rolls out, sessions without + * it still run GenerationFormV2 against them. + */ + +const V1_KEY = 'generation-graph'; +const OUTPUT_TYPES = ['image', 'video', 'audio', 'model3d'] as const; + +function readRecord( + read: (key: string) => string | null, + key: string +): Record | undefined { + const raw = read(key); + if (!raw) return undefined; + try { + const parsed = JSON.parse(raw); + return typeof parsed === 'object' && parsed !== null + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + +/** The intent record to seed the form-graph store with, or undefined if v1 holds nothing. */ +export function buildV1MigrationIntent( + read: (key: string) => string | null +): Record | undefined { + const intent: Record = {}; + const take = (source: Record | undefined, key: string, address = key) => { + const value = source?.[key]; + if (value !== undefined) intent[address] = value; + }; + + const global = readRecord(read, V1_KEY); + take(global, 'workflow'); + take(global, 'prompt'); + take(global, 'negativePrompt'); + take(global, 'quantity'); + + const preferences = readRecord(read, `${V1_KEY}.preferences`); + take(preferences, 'outputFormat'); + take(preferences, 'priority'); + + const draft = readRecord(read, `${V1_KEY}.workflow.txt2img:draft`); + take(draft, 'quantity', scopedAddress('quantity', 'txt2img:draft')); + + for (const output of OUTPUT_TYPES) { + const record = readRecord(read, `${V1_KEY}.output.${output}`); + take(record, 'ecosystem', scopedAddress('ecosystem', output)); + } + + // Same bucket keys on both sides: grouped ecosystems store under the group + // id, standalone ones under their own key (v1's adapter groups; the port's + // familyScope). + const familyKeys = new Set([ + ...ecosystemGroups.map((group) => group.id), + ...ecosystemByKey.keys(), + ]); + for (const familyKey of familyKeys) { + const record = readRecord(read, `${V1_KEY}.ecosystem.${familyKey}`); + take(record, 'model', scopedAddress('model', familyKey)); + take(record, 'resources', scopedAddress('resources', familyKey)); + } + + return Object.keys(intent).length ? intent : undefined; +} + +/** + * Runs the migration against localStorage, once: a no-op whenever the + * form-graph record already exists (including from a previous migration). + */ +export function migrateV1GenerationStorage(targetKey: string) { + if (typeof localStorage === 'undefined') return; + try { + if (localStorage.getItem(targetKey) !== null) return; + const intent = buildV1MigrationIntent((key) => localStorage.getItem(key)); + if (intent) localStorage.setItem(targetKey, JSON.stringify(intent)); + } catch { + // Quota/privacy-mode failures: start fresh instead. + } +} diff --git a/src/server/flipt/client.ts b/src/server/flipt/client.ts index 9e14fe9dbd..85b6e54a12 100644 --- a/src/server/flipt/client.ts +++ b/src/server/flipt/client.ts @@ -43,6 +43,10 @@ export enum FLIPT_FEATURE_FLAGS { GENERATION_EXPERIMENTAL = 'generation-experimental', AI_TOOLKIT_DEFAULT_SD = 'ai-toolkit-default-sd', WAN22_MULTI_STEP = 'wan22-multi-step', + // form-graph cutover, staged: shadow-compare first, then serve. See + // src/server/services/orchestrator/form-graph/shadow-parse.ts. + FORM_GRAPH_SHADOW_PARSE = 'form-graph-shadow-parse', + FORM_GRAPH_PARSE = 'form-graph-parse', ENHANCED_COMPATIBILITY_SDCPP = 'enhanced-compatibility-sdcpp', IMAGE_INDEX_FEED = 'image-index-feed', // Routes ImageResourceNew reads to the writer (primary) instead of the read diff --git a/src/server/prom/form-graph.metrics.ts b/src/server/prom/form-graph.metrics.ts new file mode 100644 index 0000000000..377279bcf9 --- /dev/null +++ b/src/server/prom/form-graph.metrics.ts @@ -0,0 +1,16 @@ +import { registerCounterWithLabels } from '@civitai/telemetry/client'; + +/** + * Shadow-parse comparison outcomes for the form-graph cutover: while the + * `form-graph-shadow-parse` flag is on, every server-side generation parse + * runs through BOTH graphs and the results are compared. `match` should be + * the only outcome; a sustained zero on the others is the flip criterion for + * `form-graph-parse`. + * + * outcome: match | diverged (results differ) | error (the hub parse threw) + */ +export const formGraphShadowParseCounter = registerCounterWithLabels({ + name: 'form_graph_shadow_parse_total', + help: 'Form-graph shadow parse comparisons by outcome (match/diverged/error) and workflow', + labelNames: ['outcome', 'workflow'] as const, +}); diff --git a/src/server/services/feature-flags.service.ts b/src/server/services/feature-flags.service.ts index 95e353cf13..df63075758 100644 --- a/src/server/services/feature-flags.service.ts +++ b/src/server/services/feature-flags.service.ts @@ -507,6 +507,10 @@ const featureFlags = createFeatureFlags({ // kill lever. Off ⇒ v2.0 is dropped from the picker and a submitted v2.0 // version id falls back to the ecosystem default (see grok-graph.ts). grokImagine2: { availability: ['mod'], fliptKey: 'grok-imagine-2' }, + // form-graph cutover: swaps GenerationTabs' form for the form-graph lane + // (FormGraphGenerator). Server parsing is staged separately via the + // form-graph-shadow-parse / form-graph-parse Flipt flags. + formGraphGenerator: { availability: ['mod'], fliptKey: 'form-graph-generator' }, // Retool privileged endpoints — `granted` means the moderator must carry the // matching permission key in user.permissions. Endpoints lookup the key // directly from `RetoolAction.privileged`, so the permission name MUST stay diff --git a/src/server/services/orchestrator/form-graph/__tests__/shadow-parse.test.ts b/src/server/services/orchestrator/form-graph/__tests__/shadow-parse.test.ts new file mode 100644 index 0000000000..426f0171e4 --- /dev/null +++ b/src/server/services/orchestrator/form-graph/__tests__/shadow-parse.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from 'vitest'; +import type * as PromMetrics from '~/server/prom/form-graph.metrics'; +import { loggingMock } from '~/__tests__/mocks/logging.mock'; + +const { inc } = vi.hoisted(() => ({ inc: vi.fn() })); +vi.mock('~/server/prom/form-graph.metrics', async (importOriginal) => ({ + ...(await importOriginal()), + formGraphShadowParseCounter: { inc }, +})); + +import { recordShadowComparison, runHubParse } from '../shadow-parse'; +import type { GenerationCtx } from '~/shared/data-graph/generation/context'; + +const logToAxiom = loggingMock.logToAxiom; + +const EXT: GenerationCtx = { + limits: { maxQuantity: 4, maxResources: 9, vidQuantity: 4 }, + user: { isMember: true, tier: 'gold' }, + flags: {}, + gateRules: [], +}; + +describe('shadow-parse comparison', () => { + it('a real parse compared against itself is a match', () => { + inc.mockClear(); + logToAxiom.mockClear(); + const hub = runHubParse({ workflow: 'txt2img', ecosystem: 'SDXL', prompt: 'a cat' }, EXT); + if (hub.ok !== true) throw new Error('hub parse failed'); + recordShadowComparison({ success: true, data: hub.data }, hub, 'txt2img'); + expect(inc).toHaveBeenCalledWith({ outcome: 'match', workflow: 'txt2img' }); + expect(logToAxiom).not.toHaveBeenCalled(); + }); + + it('a differing value diverges, logging the KEY only — never the value', () => { + inc.mockClear(); + logToAxiom.mockClear(); + const hub = runHubParse({ workflow: 'txt2img', ecosystem: 'SDXL', prompt: 'a cat' }, EXT); + if (hub.ok !== true) throw new Error('hub parse failed'); + const v1Data = { ...hub.data, prompt: 'SECRET USER PROMPT' }; + recordShadowComparison({ success: true, data: v1Data }, hub, 'txt2img'); + expect(inc).toHaveBeenCalledWith({ outcome: 'diverged', workflow: 'txt2img' }); + const logged = JSON.stringify(logToAxiom.mock.calls); + expect(logged).toContain('"prompt"'); + expect(logged).not.toContain('SECRET USER PROMPT'); + }); + + it('success/failure disagreement diverges with the losing side error keys', () => { + inc.mockClear(); + logToAxiom.mockClear(); + recordShadowComparison( + { success: false, errors: { prompt: { message: 'Prompt is required' } } }, + { ok: true, data: {}, computedKeys: [] }, + 'txt2img' + ); + expect(inc).toHaveBeenCalledWith({ outcome: 'diverged', workflow: 'txt2img' }); + expect(JSON.stringify(logToAxiom.mock.calls)).toContain('success-disagreement'); + }); + + it('both failing with the same error keys is a match', () => { + inc.mockClear(); + recordShadowComparison( + { success: false, errors: { prompt: { message: 'v1 message' } } }, + { ok: false, errors: { prompt: { message: 'different port message' } } }, + 'txt2img' + ); + expect(inc).toHaveBeenCalledWith({ outcome: 'match', workflow: 'txt2img' }); + }); + + it('a hub throw is the error outcome, not a crash', () => { + inc.mockClear(); + recordShadowComparison( + { success: true, data: {} }, + { ok: null, error: new Error('boom') }, + 'txt2img' + ); + expect(inc).toHaveBeenCalledWith({ outcome: 'error', workflow: 'txt2img' }); + }); +}); diff --git a/src/server/services/orchestrator/form-graph/shadow-parse.ts b/src/server/services/orchestrator/form-graph/shadow-parse.ts new file mode 100644 index 0000000000..9e80743c75 --- /dev/null +++ b/src/server/services/orchestrator/form-graph/shadow-parse.ts @@ -0,0 +1,112 @@ +import { isEqual } from 'lodash-es'; +import { FLIPT_FEATURE_FLAGS, isFliptSync } from '~/server/flipt/client'; +import { logToAxiom } from '~/server/logging/client'; +import { formGraphShadowParseCounter } from '~/server/prom/form-graph.metrics'; +import type { GenerationCtx } from '~/shared/data-graph/generation/context'; +import { generationHub } from '~/shared/form-graph/generation/hub.graph'; +import { reconcileSelectors } from '~/shared/form-graph/generation/reconcile'; + +/** + * The form-graph cutover's server side, staged behind two Flipt flags: + * + * 1. `form-graph-shadow-parse` — every generation parse ALSO runs through + * `generationHub`; results are compared and divergence is counted + * (`form_graph_shadow_parse_total`) and logged with diff KEYS only — no + * field values, so no prompts or user content reach the log. + * 2. `form-graph-parse` — the hub result is SERVED. The v1 parse still runs + * (it feeds the substitution metrics and the reverse shadow-compare); + * dropping it entirely belongs to the delete-data-graph change, which + * ports the metrics tap onto the hub's correction notes. + * + * Both flags default off; with neither set this module costs one sync flag + * check per parse. + */ + +export type HubParse = + | { + ok: true; + data: Record; + /** Wire-named computed keys, straight from the parse result. */ + computedKeys: readonly string[]; + } + | { ok: false; errors: Record }; + +export function shadowFlags() { + const serve = isFliptSync(FLIPT_FEATURE_FLAGS.FORM_GRAPH_PARSE) === true; + const shadow = serve || isFliptSync(FLIPT_FEATURE_FLAGS.FORM_GRAPH_SHADOW_PARSE) === true; + return { serve, shadow }; +} + +/** The hub parse, never throwing — a throw is a divergence class of its own. */ +export function runHubParse( + input: Record, + externalCtx: GenerationCtx +): HubParse | { ok: null; error: unknown } { + try { + const result = generationHub.parse(reconcileSelectors(input).raw, externalCtx); + return result.success + ? { + ok: true, + data: result.data as Record, + computedKeys: result.computedKeys ?? [], + } + : { ok: false, errors: result.errors }; + } catch (error) { + return { ok: null, error }; + } +} + +/** + * Compare the two parses and record the outcome. Only key-level information + * leaves this function: which top-level keys differ, never their values. + */ +export function recordShadowComparison( + v1: { success: boolean; data?: Record; errors?: Record }, + hub: ReturnType, + workflow: string +) { + const emit = (outcome: 'match' | 'diverged' | 'error', detail?: Record) => { + formGraphShadowParseCounter.inc({ outcome, workflow }); + if (outcome !== 'match') { + logToAxiom({ + name: 'form-graph-shadow-parse', + type: outcome, + workflow, + ...detail, + }).catch(() => undefined); + } + }; + + if (hub.ok === null) { + emit('error', { message: hub.error instanceof Error ? hub.error.message : String(hub.error) }); + return; + } + + if (v1.success !== hub.ok) { + emit('diverged', { + kind: 'success-disagreement', + v1Success: v1.success, + hubSuccess: hub.ok, + errorKeys: Object.keys((v1.success ? (hub as { errors?: object }).errors : v1.errors) ?? {}), + }); + return; + } + + if (!v1.success && hub.ok === false) { + const v1Keys = Object.keys(v1.errors ?? {}).sort(); + const hubKeys = Object.keys(hub.errors).sort(); + if (isEqual(v1Keys, hubKeys)) emit('match'); + else emit('diverged', { kind: 'error-keys', v1Keys, hubKeys }); + return; + } + + const v1Data = v1.data ?? {}; + const hubData = hub.ok === true ? hub.data : {}; + const keys = new Set([...Object.keys(v1Data), ...Object.keys(hubData)]); + const differing: string[] = []; + for (const key of keys) { + if (!isEqual(v1Data[key], hubData[key])) differing.push(key); + } + if (differing.length === 0) emit('match'); + else emit('diverged', { kind: 'data-keys', keys: differing.sort() }); +} diff --git a/src/server/services/orchestrator/orchestration-new.service.ts b/src/server/services/orchestrator/orchestration-new.service.ts index 49783b59d4..ff3dc77881 100644 --- a/src/server/services/orchestrator/orchestration-new.service.ts +++ b/src/server/services/orchestrator/orchestration-new.service.ts @@ -104,6 +104,7 @@ import { parsePromptSnippetReferences } from '~/utils/prompt-helpers'; // Ecosystem handlers - unified router import { createEcosystemStepInput } from './ecosystems'; +import { recordShadowComparison, runHubParse, shadowFlags } from './form-graph/shadow-parse'; import { createComfyInput, resourcesToImageMetadataResources } from './ecosystems/comfy-input'; import { extractStepErrors, sanitizeProviderError } from './provider-errors'; import { resolveSourceImageIds, signProvenance } from './remix-provenance'; @@ -584,7 +585,17 @@ function normalizeInput(input: Record): Record * (computed values like `triggerWords` are derived, not user input). */ function validateInput(input: Record, externalCtx: GenerationCtx) { - const result = generationGraph.safeParse(normalizeInput(input), externalCtx); + const normalized = normalizeInput(input); + const result = generationGraph.safeParse(normalized, externalCtx); + + // form-graph cutover: shadow-compare (and optionally serve) the hub parse. + // The v1 parse above always runs — it feeds the substitution metrics and, + // while serving the hub, the reverse comparison. + const cutover = shadowFlags(); + const hubResult = cutover.shadow ? runHubParse(normalized, externalCtx) : undefined; + if (hubResult) { + recordShadowComparison(result, hubResult, String(normalized.workflow ?? 'unknown')); + } // Issue #3520 — count silent checkpoint substitutions. This is the single // choke point every SERVER-side graph validation passes through (submit, @@ -598,6 +609,19 @@ function validateInput(input: Record, externalCtx: GenerationCt // awaited: this function is synchronous and on the submit path. void emitModelSubstitutions(externalCtx.modelSubstitutions); + if (cutover.serve && hubResult && hubResult.ok !== null) { + if (!hubResult.ok) { + const errorMessages = Object.entries(hubResult.errors) + .map(([key, error]) => `${key}: ${error.message}`) + .join(', '); + throw throwBadRequestError(`Validation failed: ${errorMessages}`); + } + return { + data: hubResult.data as GenerationGraphOutput, + computedKeys: new Set(hubResult.computedKeys), + }; + } + if (!result.success) { const errorMessages = Object.entries(result.errors) .map(([key, error]) => `${key}: ${error.message}`)