From 7c5dfdd31aaad53bf34ce6eac717d313181ce89b Mon Sep 17 00:00:00 2001 From: xucailiang <74602715+xucailiang@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:56:40 +0800 Subject: [PATCH] Add official Kling API providers --- .agents/skills/ai-video-gen/SKILL.md | 9 +- .agents/skills/kling-official/SKILL.md | 200 +++ .agents/skills/video_toolkit/SKILL.md | 456 +++++++ .env.example | 5 + README.md | 17 +- diagram.png | Bin 5915 -> 5336 bytes docs/ARCHITECTURE.md | 14 + docs/PROVIDERS.md | 66 +- docs/kling-official-integration-plan.md | 1133 +++++++++++++++++ docs/kling-official-phase-1-core.md | 784 ++++++++++++ .../kling-official-phase-2-omni-operations.md | 464 +++++++ ...g-official-phase-3-media-avatar-effects.md | 561 ++++++++ .../kling_official_animated_explainer_e2e.py | 570 +++++++++ skills/INDEX.md | 1 + skills/creative/video-gen-prompting.md | 1 + tests/contracts/test_kling_avatar.py | 120 ++ tests/contracts/test_kling_lip_sync.py | 233 ++++ tests/contracts/test_kling_official_client.py | 179 +++ tests/contracts/test_kling_official_docs.py | 88 ++ .../test_kling_official_e2e_script.py | 54 + .../contracts/test_kling_official_helpers.py | 166 +++ tests/contracts/test_kling_official_image.py | 299 +++++ tests/contracts/test_kling_official_video.py | 422 ++++++ tests/contracts/test_kling_tts.py | 177 +++ tests/contracts/test_phase3_contracts.py | 2 +- .../kling_official/schema_snapshot.json | 226 ++++ tools/_kling/__init__.py | 6 + tools/_kling/account.py | 121 ++ tools/_kling/callbacks.py | 17 + tools/_kling/client.py | 216 ++++ tools/_kling/elements.py | 86 ++ tools/_kling/errors.py | 46 + tools/_kling/media.py | 111 ++ tools/_kling/omni.py | 40 + tools/_kling/schemas.py | 121 ++ tools/audio/kling_tts.py | 340 +++++ tools/audio/tts_selector.py | 11 + tools/avatar/kling_avatar.py | 332 +++++ tools/avatar/kling_lip_sync.py | 551 ++++++++ tools/graphics/image_selector.py | 60 + tools/graphics/kling_official_image.py | 437 +++++++ tools/video/kling_official_video.py | 684 ++++++++++ tools/video/video_selector.py | 60 + 43 files changed, 9470 insertions(+), 16 deletions(-) create mode 100644 .agents/skills/kling-official/SKILL.md create mode 100644 .agents/skills/video_toolkit/SKILL.md create mode 100644 docs/kling-official-integration-plan.md create mode 100644 docs/kling-official-phase-1-core.md create mode 100644 docs/kling-official-phase-2-omni-operations.md create mode 100644 docs/kling-official-phase-3-media-avatar-effects.md create mode 100644 scripts/kling_official_animated_explainer_e2e.py create mode 100644 tests/contracts/test_kling_avatar.py create mode 100644 tests/contracts/test_kling_lip_sync.py create mode 100644 tests/contracts/test_kling_official_client.py create mode 100644 tests/contracts/test_kling_official_docs.py create mode 100644 tests/contracts/test_kling_official_e2e_script.py create mode 100644 tests/contracts/test_kling_official_helpers.py create mode 100644 tests/contracts/test_kling_official_image.py create mode 100644 tests/contracts/test_kling_official_video.py create mode 100644 tests/contracts/test_kling_tts.py create mode 100644 tests/fixtures/kling_official/schema_snapshot.json create mode 100644 tools/_kling/__init__.py create mode 100644 tools/_kling/account.py create mode 100644 tools/_kling/callbacks.py create mode 100644 tools/_kling/client.py create mode 100644 tools/_kling/elements.py create mode 100644 tools/_kling/errors.py create mode 100644 tools/_kling/media.py create mode 100644 tools/_kling/omni.py create mode 100644 tools/_kling/schemas.py create mode 100644 tools/audio/kling_tts.py create mode 100644 tools/avatar/kling_avatar.py create mode 100644 tools/avatar/kling_lip_sync.py create mode 100644 tools/graphics/kling_official_image.py create mode 100644 tools/video/kling_official_video.py diff --git a/.agents/skills/ai-video-gen/SKILL.md b/.agents/skills/ai-video-gen/SKILL.md index feb155ac..da1f6e9d 100644 --- a/.agents/skills/ai-video-gen/SKILL.md +++ b/.agents/skills/ai-video-gen/SKILL.md @@ -1,7 +1,7 @@ --- name: ai-video-gen description: | - Generate AI videos from text prompts using multiple provider gateways. Use when: (1) Generating videos from text descriptions, (2) Creating AI-generated video clips for content production, (3) Image-to-video generation with a reference image, (4) Choosing between video generation providers (VEO, Kling, Sora, Runway, Seedance, MiniMax). Supports two gateways: HeyGen API and fal.ai API. + Generate AI videos from text prompts using multiple provider gateways. Use when: (1) Generating videos from text descriptions, (2) Creating AI-generated video clips for content production, (3) Image-to-video generation with a reference image, (4) Choosing between video generation providers (VEO, Kling, Sora, Runway, Seedance, MiniMax). Supports HeyGen API, fal.ai API, and Kling official direct API. allowed-tools: mcp__heygen__* metadata: openclaw: @@ -9,16 +9,18 @@ metadata: env_any: - HEYGEN_API_KEY - FAL_KEY + - KLING_API_KEY --- # Video Generation (Multi-Gateway) -Generate AI videos from text prompts. Supports multiple providers via two API gateways: +Generate AI videos from text prompts. Supports multiple providers via three API paths: | Gateway | Env Variable | Providers | Tool | |---------|-------------|-----------|------| | **fal.ai** | `FAL_KEY` | **Seedance 2.0** (standard + fast), Kling v3/v2.1, MiniMax, VEO | `seedance_video`, `kling_video`, `minimax_video`, `veo_video` | | **HeyGen** | `HEYGEN_API_KEY` | VEO 3.1, Kling Pro, Sora v2, Runway Gen-4, Seedance Pro / Lite (1.x) | `heygen_video` | +| **Kling Official** | `KLING_API_KEY` | Kling official Classic, Turbo, and basic Omni video | `kling_official_video` | **Preferred premium default — Seedance 2.0.** When any premium gateway is configured (`FAL_KEY` → `seedance_video`, or HeyGen's Video Agent / Avatar Shots path), Seedance 2.0 is the preferred default for cinematic, trailer, and high-fidelity clip work. It is the only model in the fleet with **single-pass native synchronized audio, multi-shot generation, director-level camera control, and lip-sync from quoted dialogue**, and it ranks #1 on Artificial Analysis Elo as of early 2026. Switch off it only when the user has a specific reason (budget, provider preference, stylistic fit like VEO for photoreal landscape or Kling for specific anime look). See Layer 3 `seedance-2-0` for the authoritative prompting and parameter guide. @@ -30,9 +32,12 @@ Use whichever configured gateway best matches the user's available providers and - **HeyGen:** Set `HEYGEN_API_KEY` to access the multi-model gateway. - **fal.ai:** Set `FAL_KEY` to access Kling, MiniMax, and Veo through fal.ai. +- **Kling Official:** Set `KLING_API_KEY` to access Kling's official direct API via `provider="kling_official"`. Do not describe either gateway as the default or top choice without checking the registry and current task fit first. +fal.ai Kling (`kling_video`, `provider="kling"`) and Kling Official (`kling_official_video`, `provider="kling_official"`) are different paths. Do not reuse fal.ai queue URLs, `FAL_KEY`, or image upload behavior when the official provider is selected. + ```bash curl -X POST "https://api.heygen.com/v1/workflows/executions" \ -H "X-Api-Key: $HEYGEN_API_KEY" \ diff --git a/.agents/skills/kling-official/SKILL.md b/.agents/skills/kling-official/SKILL.md new file mode 100644 index 00000000..f2ffd86e --- /dev/null +++ b/.agents/skills/kling-official/SKILL.md @@ -0,0 +1,200 @@ +--- +name: kling-official +description: Official Kling direct API guidance for OpenMontage providers. Use before calling `kling_official_video`, `kling_official_image`, `kling_tts`, `kling_avatar`, or `kling_lip_sync`. +metadata: + openclaw: + requires: + env_any: + - KLING_API_KEY +--- + +# Kling Official Direct API + +Use this skill for OpenMontage tools with `provider="kling_official"`. This is not the fal.ai Kling gateway. Official Kling uses `KLING_API_KEY`, optional `KLING_API_BASE_URL`, and `Authorization: Bearer `. + +## Provider Split + +- `kling_video` uses fal.ai, `FAL_KEY`, fal.ai queue URLs, and `provider="kling"`. +- `kling_official_video` uses Kling official API, `KLING_API_KEY`, official task protocols, and `provider="kling_official"`. +- `kling_official_image` uses the same official auth and task protocol for image generation. +- `kling_tts` uses the official audio TTS endpoint and stays in the existing `tts` capability. +- `kling_avatar` and `kling_lip_sync` use official avatar/lip-sync endpoints and stay in the existing `avatar` capability. They do not replace local `talking_head` or `lip_sync`. + +Never silently switch between these paths. If the selected provider is unavailable, surface the blocker and ask before substituting. + +## Auth And Endpoint + +Default base URL: + +```text +https://api-singapore.klingai.com +``` + +Users may override it with `KLING_API_BASE_URL`, for example for a regional endpoint. All requests send JSON and: + +```text +Authorization: Bearer +``` + +## Task Protocols + +Classic APIs: + +- Create ID path: `data.task_id` +- Statuses: `submitted`, `processing`, `succeed`, `failed` +- Result paths: `data.task_result.videos[]`, `data.task_result.images[]`, `data.task_result.audios[]` + +Turbo APIs: + +- Create ID path: `data.id` +- Poll path: `GET /tasks?task_ids=` +- Statuses: `submitted`, `processing`, `succeeded`, `failed` +- Result path: `data[0].outputs[]` + +Keep the parsers separate. Do not write a fuzzy parser that guesses between `task_id` and `id` or between `succeed` and `succeeded`. + +## Omni References + +Video Omni and Image Omni stay inside the existing provider tools through `api_family="omni"`. +Do not create selector-level Omni operations. + +Video Omni accepts official reference structures: + +- `image_list[]` with `image_url` and optional `type` such as `first_frame` or `end_frame`. +- `video_list[]` with `video_url`, official `refer_type` values such as `feature` or `base`, and optional `keep_original_sound`. +- `element_list[]` with official `element_id` values. +- Structured `multi_prompt[]`; do not split natural language into shots automatically. + +Local image references may be normalized through `tools/_kling/media.py`. Local video paths must not be silently uploaded through fal.ai; ask for or require a reachable URL. + +Image Omni accepts `image_list[]` with official `image` values. Prompt placeholders such as `<<>>` must map stably to the provided image order. If the prompt already contains placeholders, validate that the referenced images exist and do not insert duplicates. + +## Phase 3 Capability Boundaries + +TTS, avatar, and lip sync are provider additions to existing OpenMontage capabilities. Audio effects and video effects are official Kling endpoints, but they are not registered as default OpenMontage tools until a pipeline has a stable capability slot for them. + +- Do not add `sound_effects` or `video_effects` capabilities from inside a provider implementation. +- Do not let video effects enter the ordinary `video_generation` selector path. +- Do not disguise short sound effects as long background music unless a pipeline explicitly consumes that shape and the tool's `best_for` / `not_good_for` says so. + +## Video Parameters + +Use `operation` for OpenMontage semantics: + +- `text_to_video` +- `image_to_video` +- `reference_to_video` + +Use `api_family` for official protocol choice: + +- `classic` +- `turbo` +- `omni` + +Important constraints: + +- Official video provider input schema must not expose top-level `image_url`; use `reference_image_url` or `reference_image_path`. +- Classic image-to-video accepts `reference_image_url` or a local path converted to raw base64 in official field `image`. +- Turbo image-to-video requires a URL first frame. Do not upload local files through fal.ai as a fallback. +- Send `aspect_ratio` only where the current schema supports it: Classic text-to-video, Turbo text-to-video, and Video Omni. +- Default paid path should avoid `4k`, native sound, or batch behavior unless explicitly selected. + +## Image Parameters + +Use `api_family="generation"` for `/v1/images/generations` and `api_family="omni"` for `/v1/images/omni-image`. + +Generation/edit path: + +- `prompt` is required and should stay under the official 2500 character limit. +- `image_url` passes through as official `image`. +- `image_path` is converted to raw base64 and sent as official `image`. +- `image_reference` can be `subject` or `face`. + +Omni path: + +- Put references in `image_list[]` using official `image` values. +- Use prompt placeholders such as `<<>>` only when the prompt needs to bind a specific reference image. + +## TTS Parameters + +`kling_tts` uses: + +- `text` +- `voice_id` +- `voice_language`, currently `zh` or `en` +- `voice_speed` + +Require an explicit `voice_id` unless an official account-specific default has been verified. Do not hard-code a made-up voice. Download every returned audio item, set `data.output_path` to the first local file, and include `voice_id`, `voice_language`, `voice_speed`, `task_id`, and non-zero `cost_usd`. + +## Avatar Parameters + +`kling_avatar` uses `/v1/videos/avatar/image2video` and accepts: + +- avatar image via URL or local path converted to raw base64 +- `audio_id` or `sound_file` +- optional `prompt` +- `mode`, such as `std` or `pro` + +Keep it separate from local `talking_head`. Pipelines that want Kling avatar output must list and choose it explicitly. + +## Lip Sync Parameters + +`kling_lip_sync` has two steps: + +1. `POST /v1/videos/identify-face` with `video_id` or `video_url` +2. `POST /v1/videos/advanced-lip-sync` with `session_id`, `face_choose[]`, and `audio_id` or `sound_file` + +Local video paths must not be silently uploaded through fal.ai or any other provider. If multiple faces are returned and the user did not pass `face_id` or `face_choose`, stop and return the face list for confirmation unless `auto_select_face=True` was explicitly set. If auto-selecting, record the selection reason and selected face in the result/artifact. + +## Audio Effects And Video Effects + +Official Kling audio effects (`/v1/audio/text-to-audio`, `/v1/audio/video-to-audio`) and video effects (`/v1/videos/effects`) are intentionally not default OpenMontage selector tools in Phase 3. Record the non-mapping reason in docs/tests instead of registering tools that current pipelines might misuse. + +## Elements Helper + +Elements are an internal Kling Official helper, not a new OpenMontage tool capability. +Use `tools/_kling/elements.py` to normalize `element_list[].element_id`, optionally query read-only element endpoints, and record element metadata when queried. Do not create or delete elements from the default provider path. + +## Account Usage Helper + +Account Usage is diagnostic only. Use `tools/_kling/account.py` for low-frequency `/account/costs` checks, with local cache and throttle protection. Do not call it before every generation and do not put it in selectors or production pipeline stages. + +For `1101` or `1102`, surface that the account or resource pack is exhausted and include an account-usage diagnostic hint. + +## Callback Notes + +Providers may accept `callback_url`, but polling remains the default execution mode. + +- Classic and Omni paths pass `callback_url` at the top level. +- Turbo paths pass it as `options.callback_url`. +- Successful results should record `callback_requested=true`, `polling_used=true`, the `callback_url`, and `task_id`. +- Validate callback URLs before sending; only absolute `http` or `https` URLs should pass. + +## Error Handling + +Surface official `code`, `message`, and `request_id` whenever available. + +Do not retry: + +- Auth failures: `1000`-`1004` +- Balance/resource-pack exhaustion: `1101`, `1102` +- Permission/model access: `1103` +- Parameter errors: `1200`, `1201` +- Safety policy: `1301` + +Limited retry is acceptable for: + +- `1302` request too fast +- `1303` concurrency/resource-pack slot limit +- `5000`, `5001`, `5002` server/maintenance/backlog errors + +For `1303`, explain that the account hit a concurrency or resource-pack slot limit. + +## Cost Governance + +Official Kling is a paid remote API. Provider tools must return non-zero conservative estimates from `estimate_cost()` and include `cost_usd` on successful paid results. Treat estimates as low-confidence until account usage reconciliation is implemented. +High-cost Omni inputs such as multiple references, element IDs, `result_type="series"`, `mode="4k"`, and `sound="on"` must increase or flag the cost estimate. + +## Prompt Notes + +For video, start from the universal OpenMontage video prompt skeleton: subject, subject motion, scene, spatial framing, and camera. Kling tends to respond well to clear temporal action order, camera movement verbs, and concise negative prompts. For reference workflows, state what should stay consistent from the reference and what should change. diff --git a/.agents/skills/video_toolkit/SKILL.md b/.agents/skills/video_toolkit/SKILL.md new file mode 100644 index 00000000..88d337f0 --- /dev/null +++ b/.agents/skills/video_toolkit/SKILL.md @@ -0,0 +1,456 @@ +--- +name: video-toolkit +description: Create professional videos autonomously using Codex-video-toolkit — AI voiceovers, image generation, music, talking heads, and Remotion rendering. +metadata: + openclaw: + emoji: "🎬" + skillKey: "video-toolkit" + os: ["darwin", "linux"] + requires: + bins: ["node", "python3", "ffmpeg", "npm"] +--- + +# Video Toolkit + +Create professional explainer videos from a text brief. The toolkit uses open-source AI models on cloud GPUs (Modal or RunPod) for voiceover, image generation, music, and talking head animation. Remotion (React) handles composition and rendering. + +## CRITICAL: Toolkit Path + +The toolkit lives at a fixed path. **ALWAYS `cd` here before running any tool command.** + +```bash +TOOLKIT=~/.openclaw/workspace/Codex-video-toolkit +cd $TOOLKIT +``` + +**NEVER run tool commands from inside a project directory.** Tools resolve paths relative to the toolkit root. + +## Setup + +### Step 1: Check Current State + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/verify_setup.py +``` + +If everything shows `[x]`, skip to "Quick Test" below. Otherwise continue setup. + +### Step 2: Install Python Dependencies + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +pip3 install --break-system-packages -r tools/requirements.txt +``` + +Note: `--break-system-packages` is needed on Debian/Ubuntu with managed Python (PEP 668). Safe inside containers. + +### Step 3: Configure Cloud GPU Endpoints + +The toolkit needs cloud GPU endpoint URLs in `.env`. Check if `.env` exists and has Modal endpoints: + +```bash +cat ~/.openclaw/workspace/Codex-video-toolkit/.env | grep MODAL +``` + +If Modal endpoints are configured, you're ready. If not, **ask the user to provide Modal endpoint URLs** or set up Modal: + +```bash +pip3 install --break-system-packages modal +python3 -m modal setup # Opens browser for authentication + +# Deploy each tool — capture the endpoint URL from output +cd ~/.openclaw/workspace/Codex-video-toolkit +modal deploy docker/modal-qwen3-tts/app.py +modal deploy docker/modal-flux2/app.py +modal deploy docker/modal-music-gen/app.py +modal deploy docker/modal-sadtalker/app.py +modal deploy docker/modal-image-edit/app.py +modal deploy docker/modal-upscale/app.py +modal deploy docker/modal-propainter/app.py +modal deploy docker/modal-ltx2/app.py # Requires: modal secret create huggingface-token HF_TOKEN=hf_... +``` + +**LTX-2 prerequisite:** Before deploying LTX-2, create a HuggingFace secret and accept the [Gemma 3 license](https://huggingface.co/google/gemma-3-12b-it-qat-q4_0-unquantized): +```bash +modal secret create huggingface-token HF_TOKEN=hf_your_read_access_token +``` + +Add each URL to `.env`: +``` +MODAL_QWEN3_TTS_ENDPOINT_URL=https://...modal.run +MODAL_FLUX2_ENDPOINT_URL=https://...modal.run +MODAL_MUSIC_GEN_ENDPOINT_URL=https://...modal.run +MODAL_SADTALKER_ENDPOINT_URL=https://...modal.run +MODAL_IMAGE_EDIT_ENDPOINT_URL=https://...modal.run +MODAL_UPSCALE_ENDPOINT_URL=https://...modal.run +MODAL_DEWATERMARK_ENDPOINT_URL=https://...modal.run +MODAL_LTX2_ENDPOINT_URL=https://...modal.run +``` + +Optional but recommended — Cloudflare R2 for reliable file transfer: +``` +R2_ACCOUNT_ID=... +R2_ACCESS_KEY_ID=... +R2_SECRET_ACCESS_KEY=... +R2_BUCKET_NAME=video-toolkit +``` + +### Step 4: Verify and Quick Test + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/verify_setup.py +``` + +All tools should show `[x]`. Then run a quick test to confirm the GPU pipeline works: + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/qwen3_tts.py --text "Hello, this is a test." --speaker Ryan --tone warm --output /tmp/video-toolkit-test.mp3 --cloud modal +``` + +If you get a valid .mp3 file, setup is complete. If it fails, check: +- `.env` has the correct `MODAL_QWEN3_TTS_ENDPOINT_URL` +- Run `python3 tools/verify_setup.py --json` and check `modal_tools` for which endpoints are missing + +**Cost:** Modal includes $30/month free compute. A typical 60s video costs $1-3. + +--- + +## Creating a Video + +### Step 1: Create Project + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +cp -r templates/product-demo projects/PROJECT_NAME +cd projects/PROJECT_NAME +npm install +``` + +Templates: `product-demo` (marketing/explainer), `sprint-review`, `sprint-review-v2` (composable scenes). + +### Step 2: Write Config + +Edit `projects/PROJECT_NAME/src/config/demo-config.ts`: + +```typescript +export const demoConfig: ProductDemoConfig = { + product: { + name: 'My Product', + tagline: 'What it does in one line', + website: 'example.com', + }, + scenes: [ + { type: 'title', durationSeconds: 9, content: { headline: '...', subheadline: '...' } }, + { type: 'problem', durationSeconds: 14, content: { headline: '...', problems: ['...', '...'] } }, + { type: 'solution', durationSeconds: 13, content: { headline: '...', highlights: ['...', '...'] } }, + { type: 'stats', durationSeconds: 12, content: { stats: [{value: '99%', label: '...'}, ...] } }, + { type: 'cta', durationSeconds: 10, content: { headline: '...', links: ['...'] } }, + ], + audio: { + backgroundMusicFile: 'audio/bg-music.mp3', + backgroundMusicVolume: 0.12, + }, +}; +``` + +Scene types: `title`, `problem`, `solution`, `demo`, `feature`, `stats`, `cta`. + +**Duration rule:** Estimate `durationSeconds` as `ceil(word_count / 2.5) + 2`. You will adjust this after generating audio in Step 4. + +### Step 3: Write Voiceover Script + +Create `projects/PROJECT_NAME/VOICEOVER-SCRIPT.md`: + +```markdown +## Scene 1: Title (9s, ~17 words) +Build videos with AI. The product name toolkit makes it easy. + +## Scene 2: Problem (14s, ~30 words) +The problem statement goes here. Keep it punchy and relatable. +``` + +**Word budget per scene:** `(durationSeconds - 2) * 2.5` words. The -2 accounts for 1s audio delay + 1s padding. + +### Step 4: Generate Assets + +**CRITICAL: All commands below MUST be run from the toolkit root, not the project directory.** + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +``` + +#### 4a. Background Music + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/music_gen.py \ + --preset corporate-bg \ + --duration 90 \ + --output projects/PROJECT_NAME/public/audio/bg-music.mp3 \ + --cloud modal +``` + +Presets: `corporate-bg`, `upbeat-tech`, `ambient`, `dramatic`, `tension`, `hopeful`, `cta`, `lofi`. + +#### 4b. Voiceover (per-scene) + +Generate ONE .mp3 file PER SCENE. Do NOT generate a single voiceover file. + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit + +# Scene 01 +python3 tools/qwen3_tts.py \ + --text "The voiceover text for scene one." \ + --speaker Ryan --tone warm \ + --output projects/PROJECT_NAME/public/audio/scenes/01.mp3 \ + --cloud modal + +# Scene 02 +python3 tools/qwen3_tts.py \ + --text "The voiceover text for scene two." \ + --speaker Ryan --tone warm \ + --output projects/PROJECT_NAME/public/audio/scenes/02.mp3 \ + --cloud modal + +# ... repeat for each scene +``` + +**Speakers:** `Ryan`, `Aiden`, `Vivian`, `Serena`, `Uncle_Fu`, `Dylan`, `Eric`, `Ono_Anna`, `Sohee` +**Tones:** `neutral`, `warm`, `professional`, `excited`, `calm`, `serious`, `storyteller`, `tutorial` + +For voice cloning (needs a reference recording): +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/qwen3_tts.py \ + --text "Text to speak" \ + --ref-audio assets/voices/reference.m4a \ + --ref-text "Exact transcript of the reference audio" \ + --output projects/PROJECT_NAME/public/audio/scenes/01.mp3 \ + --cloud modal +``` + +#### 4c. Scene Images + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/flux2.py \ + --prompt "Dark tech background with blue geometric grid, cinematic lighting" \ + --width 1920 --height 1080 \ + --output projects/PROJECT_NAME/public/images/title-bg.png \ + --cloud modal +``` + +Image presets (use `--preset` instead of `--prompt --width --height`): +`title-bg`, `problem`, `solution`, `demo-bg`, `stats-bg`, `cta`, `thumbnail`, `portrait-bg` + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/flux2.py \ + --preset title-bg \ + --output projects/PROJECT_NAME/public/images/title-bg.png \ + --cloud modal +``` + +#### 4d. Video Clips — B-Roll & Animated Backgrounds (optional) + +Generate AI video clips for b-roll cutaways, animated slide backgrounds, or intro/outro sequences: + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit + +# B-roll clip from text +python3 tools/ltx2.py \ + --prompt "Aerial drone shot over a European city at golden hour, cinematic wide angle" \ + --output projects/PROJECT_NAME/public/videos/broll-europe.mp4 \ + --cloud modal + +# Animate a slide/screenshot (image-to-video) +python3 tools/ltx2.py \ + --prompt "Gentle particle effects, soft ambient light shifts, very slight camera drift" \ + --input projects/PROJECT_NAME/public/images/title-bg.png \ + --output projects/PROJECT_NAME/public/videos/animated-title.mp4 \ + --cloud modal + +# Abstract intro/outro background +python3 tools/ltx2.py \ + --prompt "Dark moody abstract background with flowing blue light streaks, bokeh particles, cinematic" \ + --output projects/PROJECT_NAME/public/videos/intro-bg.mp4 \ + --cloud modal +``` + +Use in Remotion compositions with ``: +```tsx + +``` + +**LTX-2 rules:** +- Max ~8 seconds per clip (193 frames at 24fps). Default is ~5s (121 frames). +- Width/height must be divisible by 64. Default: 768x512. +- ~$0.20-0.25 per clip, ~2.5 min generation time. +- Cold start ~60-90s. Subsequent clips on warm GPU are faster. +- Generated audio is ambient only — use voiceover/music tools for speech and music. +- ~30% of generations may have training data artifacts (logos/text). Re-run with `--seed` to vary. + +#### 4e. Talking Head Narrator (optional) + +Generate a presenter portrait, then animate per-scene clips: + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit + +# 1. Generate portrait +python3 tools/flux2.py \ + --prompt "Professional presenter portrait, clean style, dark background, facing camera, upper body" \ + --width 1024 --height 576 \ + --output projects/PROJECT_NAME/public/images/presenter.png \ + --cloud modal + +# 2. Generate per-scene narrator clips (one per scene, NOT one long video) +python3 tools/sadtalker.py \ + --image projects/PROJECT_NAME/public/images/presenter.png \ + --audio projects/PROJECT_NAME/public/audio/scenes/01.mp3 \ + --preprocess full --still --expression-scale 0.8 \ + --output projects/PROJECT_NAME/public/narrator-01.mp4 \ + --cloud modal + +# Repeat for each scene that needs a narrator +``` + +**SadTalker rules — follow these exactly:** +- **ALWAYS** use `--preprocess full` (default `crop` outputs a square, wrong aspect ratio) +- **ALWAYS** use `--still` (reduces head movement, looks professional) +- **ALWAYS** generate per-scene clips (6-15s each), NEVER one long video +- Processing: ~3-4 min per 10s of audio on Modal A10G +- `--expression-scale 0.8` keeps expressions subtle (range 0.0-1.5) + +#### 4e. Image Editing (optional) + +Create scene variants from existing images: + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/image_edit.py \ + --input projects/PROJECT_NAME/public/images/title-bg.png \ + --prompt "Make it darker with red tones, more ominous" \ + --output projects/PROJECT_NAME/public/images/problem-bg.png \ + --cloud modal +``` + +#### 4f. Upscaling (optional) + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +python3 tools/upscale.py \ + --input projects/PROJECT_NAME/public/images/some-image.png \ + --output projects/PROJECT_NAME/public/images/some-image-4x.png \ + --scale 4 --cloud modal +``` + +### Step 5: Sync Timing + +**ALWAYS do this after generating voiceover.** Audio duration differs from estimates. + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit +for f in projects/PROJECT_NAME/public/audio/scenes/*.mp3; do + echo "$(basename $f): $(ffprobe -v error -show_entries format=duration -of csv=p=0 "$f")s" +done +``` + +Update each scene's `durationSeconds` in `demo-config.ts` to: `ceil(actual_audio_duration + 2)`. + +Example: if `01.mp3` is 6.8s, set scene 1 `durationSeconds` to `9` (ceil(6.8 + 2) = 9). + +### Step 6: Review Still Frames + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit/projects/PROJECT_NAME +npx remotion still src/index.ts ProductDemo --frame=100 --output=/tmp/review-scene1.png +npx remotion still src/index.ts ProductDemo --frame=400 --output=/tmp/review-scene2.png +``` + +Check: text truncation, animation timing, narrator PiP positioning, background contrast. + +### Step 7: Render + +```bash +cd ~/.openclaw/workspace/Codex-video-toolkit/projects/PROJECT_NAME +npm run render +``` + +**Output:** `out/ProductDemo.mp4` + +--- + +## Composition Patterns + +### Per-Scene Audio + +Use per-scene audio with a 1-second delay (`from={30}` = 30 frames = 1s at 30fps): + +```tsx + + +``` + +### Per-Scene Narrator PiP + +```tsx + + + +``` + +**ALWAYS use ``, NEVER `