Merge pull request #3622 from civitai/ltxv-enhancements

@
This commit is contained in:
Briant Diehl
2026-08-04 10:28:49 -06:00
committed by GitHub
9 changed files with 174 additions and 39 deletions
+1 -1
View File
@@ -118,7 +118,7 @@
"@civitai/app-sdk": "^0.14.0",
"@civitai/auth": "workspace:*",
"@civitai/buzz": "workspace:*",
"@civitai/client": "0.2.0-beta.83",
"@civitai/client": "0.2.0-beta.84",
"@civitai/cybertipline-tools": "^0.1.0",
"@civitai/db-queries": "workspace:*",
"@civitai/db-schema": "workspace:*",
+5 -5
View File
@@ -47,8 +47,8 @@ importers:
specifier: workspace:*
version: link:packages/civitai-buzz
'@civitai/client':
specifier: 0.2.0-beta.83
version: 0.2.0-beta.83
specifier: 0.2.0-beta.84
version: 0.2.0-beta.84
'@civitai/cybertipline-tools':
specifier: ^0.1.0
version: 0.1.0
@@ -2044,8 +2044,8 @@ packages:
resolution: {integrity: sha512-dSXkI8hVoozzlPS9DymzoyncWz8eapH3wkSDzDVGWjjqamXn8Dbcq1hWu5xUZkZmqCaR76F7BS9BQNqvOJ++Sw==}
engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
'@civitai/client@0.2.0-beta.83':
resolution: {integrity: sha512-67kVYSlmaApnpOIDhDUKI3tyw6YfLR8n0ZPch+4216eiQqb/QczistRRBuV6xMclSTy1UuuYeqGHM13lTs7h3g==}
'@civitai/client@0.2.0-beta.84':
resolution: {integrity: sha512-4YwvmdGwbD0gWocDSoSSz+y0VehAriNZhfCRygi0SojvEjbjFFRtgqqcTStxGkDG4wifaENFXGaN8C4u/2AV4Q==}
engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
'@civitai/cybertipline-tools@0.1.0':
@@ -13767,7 +13767,7 @@ snapshots:
rfc6902: 5.2.0
tslib: 2.8.1
'@civitai/client@0.2.0-beta.83':
'@civitai/client@0.2.0-beta.84':
dependencies:
'@hey-api/client-fetch': 0.1.14
rfc6902: 5.2.0
@@ -4,14 +4,19 @@
* Handles xAI Grok Imagine workflows for both image and video generation.
* Image workflows use imageGen step type, video workflows use videoGen step type.
*
* Image operations:
* Image operations (version-less on the API — v1.5 is video-only):
* - createImage: Text to image (GrokCreateImageGenInput)
* - editImage: Edit with source images (GrokEditImageGenInput)
*
* Video operations:
* Video operations, v1.0:
* - text-to-video: Text to video (GrokTextToVideoInput)
* - image-to-video: Image to video (GrokImageToVideoInput)
* - edit-video: Edit video with AI (GrokEditVideoInput)
*
* Video operations, v1.5:
* - textToVideo: Text to video, up to 1080p (GrokV15TextToVideoInput)
* - imageToVideo: Single source image, ratio follows the image (GrokV15ImageToVideoInput)
* - referenceToVideo: 1-7 reference images (GrokV15ReferenceToVideoInput)
*/
import type {
@@ -20,13 +25,19 @@ import type {
GrokTextToVideoInput,
GrokImageToVideoInput,
GrokEditVideoInput,
GrokV15TextToVideoInput,
GrokV15ImageToVideoInput,
GrokV15ReferenceToVideoInput,
ImageGenStepTemplate,
VideoGenStepTemplate,
} from '@civitai/client';
import { removeEmpty } from '~/utils/object-helpers';
import { findClosestAspectRatio } from '~/utils/aspect-ratio-helpers';
import type { GenerationGraphTypes } from '~/shared/data-graph/generation/generation-graph';
import { grokVideoAspectRatiosByResolution } from '~/shared/data-graph/generation/grok-graph';
import {
grokVideoAspectRatiosByResolution,
isGrokV15,
} from '~/shared/data-graph/generation/grok-graph';
import { defineHandler } from './handler-factory';
// Types derived from generation graph
@@ -73,7 +84,8 @@ export const createGrokImageInput = defineHandler<GrokCtx, [ImageGenStepTemplate
/**
* Creates videoGen input for Grok video workflows.
* Handles text-to-video, image-to-video, and edit-video operations.
* The selected model version id selects the engine version, which in turn
* determines the available operations and their naming.
*/
export const createGrokVideoInput = defineHandler<GrokCtx, [VideoGenStepTemplate]>((data) => {
const hasImages = !!data.images?.length;
@@ -87,6 +99,53 @@ export const createGrokVideoInput = defineHandler<GrokCtx, [VideoGenStepTemplate
resolution,
};
if (isGrokV15(data.model?.id)) {
const v15Base = { ...baseData, version: 'v1.5' as const };
if (data.workflow === 'img2vid:ref2vid') {
const images = data.images?.map((x) => x.url) ?? [];
if (!images.length)
throw new Error('At least one reference image is required for img2vid:ref2vid');
return [
{
$type: 'videoGen',
input: removeEmpty({
...v15Base,
operation: 'referenceToVideo',
images,
aspectRatio: data.aspectRatio?.value as GrokV15ReferenceToVideoInput['aspectRatio'],
}) as GrokV15ReferenceToVideoInput,
},
];
}
if (hasImages) {
return [
{
$type: 'videoGen',
input: removeEmpty({
...v15Base,
operation: 'imageToVideo',
images: [data.images![0].url] as [string],
}) as GrokV15ImageToVideoInput,
},
];
}
return [
{
$type: 'videoGen',
input: removeEmpty({
...v15Base,
operation: 'textToVideo',
aspectRatio: data.aspectRatio?.value as GrokV15TextToVideoInput['aspectRatio'],
}) as GrokV15TextToVideoInput,
},
];
}
const v1Base = { ...baseData, version: 'v1.0' as const };
// Edit video (vid2vid:edit)
if (hasVideo) {
const video = data.video as { url: string; metadata?: { duration?: number } };
@@ -94,7 +153,7 @@ export const createGrokVideoInput = defineHandler<GrokCtx, [VideoGenStepTemplate
{
$type: 'videoGen',
input: removeEmpty({
...baseData,
...v1Base,
operation: 'edit-video',
videoUrl: video.url,
analyzedDuration: video.metadata?.duration,
@@ -112,7 +171,7 @@ export const createGrokVideoInput = defineHandler<GrokCtx, [VideoGenStepTemplate
{
$type: 'videoGen',
input: removeEmpty({
...baseData,
...v1Base,
operation: 'image-to-video',
aspectRatio: aspectRatio?.value as GrokImageToVideoInput['aspectRatio'],
images: [data.images![0].url] as [string],
@@ -126,7 +185,7 @@ export const createGrokVideoInput = defineHandler<GrokCtx, [VideoGenStepTemplate
{
$type: 'videoGen',
input: removeEmpty({
...baseData,
...v1Base,
operation: 'text-to-video',
aspectRatio: data.aspectRatio?.value as GrokTextToVideoInput['aspectRatio'],
}) as GrokTextToVideoInput,
@@ -80,7 +80,8 @@ function resolveImageDimensions(
/**
* Creates videoGen input for LTX (v2 and v2.3) ecosystems.
* When `enablePromptEnhancer` is on, prepends a promptEnhancement step and
* wires its `output.enhancedPrompt` into the videoGen step's `prompt` via $ref.
* wires its `output.enhancedPrompt` / `output.enhancedNegativePrompt` into the
* videoGen step via $ref.
* Reference images (img2vid / ref2vid) are passed to the enhancer so the
* vision-capable LLM can ground the rewrite in the input frames.
*/
@@ -89,6 +90,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
const steps: StepInput[] = [];
let prompt: string = data.prompt;
let negativePrompt: string | undefined = data.negativePrompt || undefined;
if (data.enablePromptEnhancer) {
// Pull image URLs off `data.images` when present (img2vid + ref2vid carry
// them; vid2vid uses `data.video` and has no images).
@@ -102,10 +104,15 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
? "Audio generation is enabled. Preserve any audio descriptions the user already wrote in the prompt (music, voices, dialogue, sound effects, ambient sounds) — do not remove, replace, or contradict them. If the user's prompt has little or no audio detail, add appropriate audio cues that fit the scene."
: undefined;
const { step, prompt: promptRef } = createChainedPromptEnhancementStep(
const {
step,
prompt: promptRef,
negativePrompt: negativePromptRef,
} = createChainedPromptEnhancementStep(
{
ecosystem: data.ecosystem.toLowerCase(),
prompt: data.prompt,
negativePrompt,
preserveTriggerWords: data.triggerWords,
images: enhancerImages?.length ? enhancerImages : undefined,
instruction,
@@ -114,6 +121,9 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
);
steps.push(step);
prompt = promptRef;
// Only follow the enhanced-negative ref when the user actually wrote one —
// otherwise the step has nothing to enhance and the ref resolves to null.
if (negativePrompt) negativePrompt = negativePromptRef;
}
if (data.ltxVersion === 'v23') {
@@ -146,6 +156,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2.3',
operation: 'firstLastFrameToVideo',
prompt,
negativePrompt,
width,
height,
model,
@@ -172,6 +183,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2.3',
operation: 'editVideo',
prompt,
negativePrompt,
width: 'video' in data ? data.video?.metadata?.width : undefined,
height: 'video' in data ? data.video?.metadata?.height : undefined,
model,
@@ -199,6 +211,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2.3',
operation: 'extendVideo',
prompt,
negativePrompt,
width: data.video?.metadata?.width,
height: data.video?.metadata?.height,
model,
@@ -224,6 +237,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2.3',
operation: 'createVideo',
prompt,
negativePrompt,
width: data.aspectRatio?.width,
height: data.aspectRatio?.height,
model,
@@ -264,6 +278,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2',
operation: 'firstLastFrameToVideo',
prompt,
negativePrompt,
width,
height,
guidanceScale,
@@ -284,6 +299,7 @@ export const createLTXInput = defineHandler<LTXCtx, StepInput[]>((data, ctx) =>
engine: 'ltx2',
operation: 'createVideo',
prompt,
negativePrompt,
width: data.aspectRatio?.width,
height: data.aspectRatio?.height,
guidanceScale,
@@ -440,6 +440,7 @@ export async function createXGuardModerationRequest(args: XGuardModerationArgs)
labels,
labelOverrides,
storeFullResponse: false,
includeReasoning: false,
}
: {
mode: 'prompt' as const,
@@ -449,6 +450,7 @@ export async function createXGuardModerationRequest(args: XGuardModerationArgs)
labels,
labelOverrides,
storeFullResponse: false,
includeReasoning: false,
};
// The orchestrator submit can either return `{ data: null, error }` for a
@@ -23,6 +23,7 @@ import type { OutputType } from './types';
// graphs import helpers from this file, so importing them back here would form a
// graph <-> config/workflows cycle (the cause of "X is undefined" at module-eval).
import {
grokVersionIds,
happyHorseVersionIds,
klingVersionIds,
viduVersionIds,
@@ -165,6 +166,8 @@ export const workflowConfigs: WorkflowConfigs = {
description: 'Generate an AI image from text',
category: 'image',
ecosystemIds: TXT2IMG_IDS,
// Grok image generation is version-less on the API — v1.5 is video-only.
excludeModelVersionIds: [grokVersionIds['v1.5']],
},
'txt2img:draft': {
@@ -202,6 +205,7 @@ export const workflowConfigs: WorkflowConfigs = {
description: 'Generate or edit using reference images',
category: 'image',
ecosystemIds: EDIT_IMG_IDS,
excludeModelVersionIds: [grokVersionIds['v1.5']],
},
'img2img:face-fix': {
@@ -309,8 +313,10 @@ export const workflowConfigs: WorkflowConfigs = {
ECO.WanVideo27,
ECO.HappyHorse,
ECO.MiniMaxH3,
ECO.Grok,
],
excludeModelVersionIds: [viduVersionIds.q3],
// Grok referenceToVideo is a v1.5-only operation.
excludeModelVersionIds: [viduVersionIds.q3, grokVersionIds['v1.0']],
},
// ===========================================================================
@@ -339,7 +345,8 @@ export const workflowConfigs: WorkflowConfigs = {
category: 'video',
ecosystemIds: [ECO.Grok, ECO.WanVideo27, ECO.HappyHorse],
// HappyHorse v1.1 has no videoEdit operation — v1.0 only.
excludeModelVersionIds: [happyHorseVersionIds['v1.1']],
// Grok edit-video is likewise v1.0-only.
excludeModelVersionIds: [happyHorseVersionIds['v1.1'], grokVersionIds['v1.5']],
},
// Disabled — LTXV23 extendVideo is producing poor results. Re-enable once
+64 -20
View File
@@ -4,14 +4,19 @@
* Controls for Grok ecosystem (xAI Grok Imagine).
* Supports both image and video generation workflows.
*
* Image workflows:
* Version-selectable (v1.0 / v1.5) via the model picker. Image generation is
* version-less on the API, so the image workflows are v1.0-only and excluded for
* v1.5 in config/workflows.ts.
*
* Image workflows (v1.0 only):
* - txt2img: Create image from text (GrokCreateImageGenInput)
* - img2img:edit: Edit image with AI (GrokEditImageGenInput)
*
* Video workflows:
* - txt2vid: Text to video (GrokTextToVideoInput)
* - img2vid: Image to video (GrokImageToVideoInput)
* - vid2vid:edit: Edit video with AI (GrokEditVideoInput)
* - txt2vid 'text-to-video' (v1.0) / 'textToVideo' (v1.5)
* - img2vid 'image-to-video' (v1.0) / 'imageToVideo' (v1.5)
* - img2vid:ref2vid 'referenceToVideo' (v1.5 only, 1-7 reference images)
* - vid2vid:edit 'edit-video' (v1.0 only)
*
* Uses a discriminator on the parent's `output` node ('image' | 'video')
* to split into image-specific and video-specific subgraphs.
@@ -24,6 +29,7 @@ import {
seedNode,
aspectRatioNode,
createTextEditorGraph,
enumNode,
imagesNode,
snippetsGraph,
triggerWordsGraph,
@@ -34,6 +40,7 @@ import {
getAspectRatioOptions,
type GenerationAspectRatio,
} from '~/shared/constants/generation.constants';
import { grokVersionIds } from './version-ids';
// =============================================================================
// Constants
@@ -73,8 +80,20 @@ const grokVideoAspectRatiosByResolution: Record<
const grokResolutions = [
{ label: '480p', value: '480p' },
{ label: '720p', value: '720p' },
] as const;
/** v1.5 text-to-video and image-to-video additionally accept 1080p */
const grokV15Resolutions = [...grokResolutions, { label: '1080p', value: '1080p' }] as const;
/** Options for the Grok version selector (using version IDs as values) */
const grokVersionOptions = [
{ label: 'v1.0', value: grokVersionIds['v1.0'] },
{ label: 'v1.5', value: grokVersionIds['v1.5'] },
];
/** True when the selected model version is Grok Imagine v1.5 */
const isGrokV15 = (modelId?: number) => modelId === grokVersionIds['v1.5'];
// =============================================================================
// Image Subgraph
// =============================================================================
@@ -114,6 +133,7 @@ type GrokVideoCtx = {
ecosystem: string;
workflow: string;
output: 'video';
model?: { id: number };
images?: ImageEntry[];
video?: { url: string; metadata?: { duration?: number } };
};
@@ -129,18 +149,27 @@ const grokVideoGraph = new DataGraph<GrokVideoCtx, GenerationCtx>()
)
.node(
'images',
(ctx) => ({
...imagesNode({ max: 1, warnOnMissingAiMetadata: true }),
when: ctx.workflow === 'img2vid',
}),
(ctx) => {
if (ctx.workflow === 'img2vid:ref2vid')
return { ...imagesNode({ max: 7, warnOnMissingAiMetadata: true }), when: true };
return {
...imagesNode({ max: 1, warnOnMissingAiMetadata: true }),
when: ctx.workflow === 'img2vid',
};
},
['workflow']
)
.node('resolution', {
input: z.enum(['480p', '720p']).optional(),
output: z.enum(['480p', '720p']),
defaultValue: '720p' as const,
meta: { options: grokResolutions },
})
.node(
'resolution',
(ctx) => {
const supports1080p = isGrokV15(ctx.model?.id) && ctx.workflow !== 'img2vid:ref2vid';
return enumNode({
options: supports1080p ? grokV15Resolutions : grokResolutions,
defaultValue: '720p',
});
},
['workflow', 'model']
)
.node('duration', {
input: z.coerce.number().min(6).max(15).optional(),
output: z.number().min(6).max(15),
@@ -156,12 +185,15 @@ const grokVideoGraph = new DataGraph<GrokVideoCtx, GenerationCtx>()
);
const hasImages = Array.isArray(ctx.images) && ctx.images.length > 0;
const hasVideo = !!ctx.video?.url;
// ref2vid takes an explicit ratio even though it has images; the other
// image/video-driven workflows derive it from the input instead.
const isRef2Vid = ctx.workflow === 'img2vid:ref2vid';
return {
...aspectRatioNode({ options, defaultValue: '16:9' }),
when: !hasImages && !hasVideo,
when: isRef2Vid || (!hasImages && !hasVideo),
};
},
['images', 'video', 'resolution']
['workflow', 'images', 'video', 'resolution']
);
// =============================================================================
@@ -169,7 +201,12 @@ const grokVideoGraph = new DataGraph<GrokVideoCtx, GenerationCtx>()
// =============================================================================
/** Context shape for grok graph */
type GrokCtx = { ecosystem: string; workflow: string; output: 'image' | 'video' };
type GrokCtx = {
ecosystem: string;
workflow: string;
output: 'image' | 'video';
model?: { id: number };
};
/**
* Grok generation controls.
@@ -178,8 +215,9 @@ type GrokCtx = { ecosystem: string; workflow: string; output: 'image' | 'video'
* Uses a discriminator on the parent's `output` node to split into image/video subgraphs.
*/
export const grokGraph = new DataGraph<GrokCtx, GenerationCtx>()
// Merge checkpoint graph (default model set via ecosystemSettings)
.merge(() => createCheckpointGraph(), [])
// Version-locked model (v1.0 / v1.5) — swap button hidden via modelLocked in
// ecosystemSettings; the default model id comes from ecosystemSettings.
.merge(() => createCheckpointGraph({ versions: { options: grokVersionOptions } }), [])
// Seed node
.node('seed', seedNode())
@@ -197,4 +235,10 @@ export const grokGraph = new DataGraph<GrokCtx, GenerationCtx>()
.merge(createTextEditorGraph({ name: 'prompt', required: true }));
// Export constants for use in components
export { grokImageAspectRatios, grokVideoAspectRatiosByResolution, grokResolutions };
export {
grokImageAspectRatios,
grokVideoAspectRatiosByResolution,
grokResolutions,
grokV15Resolutions,
isGrokV15,
};
@@ -29,6 +29,7 @@ import type { AspectRatioOption, VersionGroup } from './common';
import {
seedNode,
aspectRatioNode,
negativePromptGraph,
promptGraph,
sliderNode,
snippetsGraph,
@@ -484,10 +485,11 @@ export const ltxGraph = new DataGraph<LTXCtx, GenerationCtx>()
[]
)
// Prompt + triggerWords are common to all LTX versions (no negativePrompt for LTX).
// Prompt + triggerWords are common to all LTX versions.
.merge(triggerWordsGraph)
.merge(snippetsGraph)
.merge(promptGraph);
.merge(promptGraph)
.merge(negativePromptGraph);
// =============================================================================
// Exports
@@ -31,3 +31,8 @@ export const happyHorseVersionIds = {
'v1.0': 2902378,
'v1.1': 3063263,
} as const;
export const grokVersionIds = {
'v1.0': 2738377,
'v1.5': 3197990,
} as const;