From 16791a3a808cb4d48ba3245449d516dfc2644716 Mon Sep 17 00:00:00 2001 From: calesthio Date: Fri, 17 Apr 2026 21:51:54 -0700 Subject: [PATCH] video-gen: adopt Seedance 2.0 as preferred premium default Seedance 2.0 is now routed as the top-ranked video generator whenever a premium gateway is available. Touches the tool layer, scoring engine, cinematic pipeline, and both skill layers so discovery works from every entry point. - tools/video/seedance_video: BETA stability, quality_score=0.95, add reference_to_video operation plus 9 img + 3 vid + 3 audio ceilings, fix pre-existing upload_image_fal import - tools/base_tool: surface optional quality_score / success_rate / latency fields in get_info so the scorer can read them - lib/scoring: fix reliability enum-vs-string bug that was pinning every available tool to 0.0, switch to overlap coefficient so rich best_for descriptions aren't penalized, add premium-cinematic feature bonus - pipeline_defs/cinematic + cinematic asset-director: add pixabay_music and freesound_music, restore pixabay-first music default - cinematic compose-director: mandatory Remotion preflight at stage entry - New Layer 3 .agents/skills/seedance-2-0/SKILL.md (8-part prompt structure, multi-shot, lip-sync, reference-to-video, provider landscape) - New Layer 2 skills/creative/prompting/seedance-prompting.md - Update ai-video-gen, video-gen-prompting, AGENT_GUIDE, INDEX to flag Seedance 2.0 as the preferred premium default and make the skill discoverable from every routing path --- .agents/skills/ai-video-gen/SKILL.md | 10 +- .agents/skills/seedance-2-0/SKILL.md | 210 ++++++++++++++++++ .claude/skills/ai-video-gen/SKILL.md | 10 +- AGENT_GUIDE.md | 2 +- lib/scoring.py | 58 ++++- pipeline_defs/cinematic.yaml | 4 + skills/INDEX.md | 2 + .../creative/prompting/seedance-prompting.md | 138 ++++++++++++ skills/creative/video-gen-prompting.md | 9 +- skills/pipelines/cinematic/asset-director.md | 4 +- .../pipelines/cinematic/compose-director.md | 14 ++ tools/base_tool.py | 11 + tools/video/seedance_video.py | 85 ++++++- 13 files changed, 524 insertions(+), 33 deletions(-) create mode 100644 .agents/skills/seedance-2-0/SKILL.md create mode 100644 skills/creative/prompting/seedance-prompting.md diff --git a/.agents/skills/ai-video-gen/SKILL.md b/.agents/skills/ai-video-gen/SKILL.md index 031c8e02..feb155ac 100644 --- a/.agents/skills/ai-video-gen/SKILL.md +++ b/.agents/skills/ai-video-gen/SKILL.md @@ -17,10 +17,12 @@ Generate AI videos from text prompts. Supports multiple providers via two API ga | Gateway | Env Variable | Providers | Tool | |---------|-------------|-----------|------| -| **fal.ai** | `FAL_KEY` | Kling v3/v2.1, MiniMax, VEO | `kling_video`, `minimax_video`, `veo_video` | -| **HeyGen** | `HEYGEN_API_KEY` | VEO 3.1, Kling Pro, Sora v2, Runway Gen-4, Seedance | `heygen_video` | +| **fal.ai** | `FAL_KEY` | **Seedance 2.0** (standard + fast), Kling v3/v2.1, MiniMax, VEO | `seedance_video`, `kling_video`, `minimax_video`, `veo_video` | +| **HeyGen** | `HEYGEN_API_KEY` | VEO 3.1, Kling Pro, Sora v2, Runway Gen-4, Seedance Pro / Lite (1.x) | `heygen_video` | -**IMPORTANT:** Always use `video_selector` instead of calling provider tools directly. The selector handles availability checks, cost comparison, and automatic fallback. +**Preferred premium default — Seedance 2.0.** When any premium gateway is configured (`FAL_KEY` → `seedance_video`, or HeyGen's Video Agent / Avatar Shots path), Seedance 2.0 is the preferred default for cinematic, trailer, and high-fidelity clip work. It is the only model in the fleet with **single-pass native synchronized audio, multi-shot generation, director-level camera control, and lip-sync from quoted dialogue**, and it ranks #1 on Artificial Analysis Elo as of early 2026. Switch off it only when the user has a specific reason (budget, provider preference, stylistic fit like VEO for photoreal landscape or Kling for specific anime look). See Layer 3 `seedance-2-0` for the authoritative prompting and parameter guide. + +**IMPORTANT:** Always use `video_selector` instead of calling provider tools directly. The selector handles availability checks, cost comparison, and automatic fallback, and its scoring engine already biases toward Seedance 2.0 for cinematic intent. ## Authentication @@ -312,7 +314,7 @@ curl -X POST "https://api.heygen.com/v1/workflows/executions" \ ## Best Practices 1. **Be descriptive in prompts** — include camera movement, lighting, style, and mood details -2. **Default to VEO 3.1** for highest quality; use `ltx_distilled` or `veo3_fast` when speed matters +2. **Default to Seedance 2.0 (via `seedance_video`) for cinematic and motion-led work** when `FAL_KEY` is set — single-pass synced audio, multi-shot, lip-sync, director-level camera. Use VEO 3.1 / Sora V2 Pro when the user specifically wants Google or OpenAI motion character; use `ltx_distilled` or `veo3_fast` only when speed is the hard constraint 3. **Use reference images** for image-to-video generation — great for animating product photos or still images 4. **Video generation is the slowest workflow** — allow up to 5 minutes, poll every 10 seconds 5. **Aspect ratio matters** — use `9:16` for social media stories/reels, `16:9` for landscape, `1:1` for square diff --git a/.agents/skills/seedance-2-0/SKILL.md b/.agents/skills/seedance-2-0/SKILL.md new file mode 100644 index 00000000..f8526111 --- /dev/null +++ b/.agents/skills/seedance-2-0/SKILL.md @@ -0,0 +1,210 @@ +--- +name: seedance-2-0 +description: | + Generate cinematic clips with ByteDance Seedance 2.0 — the preferred premium video model in OpenMontage when a paid gateway is configured. Use when: (1) producing trailers, teasers, hype edits, or premium cinematic clips, (2) needing native synchronized audio (speech, SFX, ambience) in a single pass, (3) needing multi-shot cuts inside one generation, (4) needing director-level camera control, (5) needing lip-sync from quoted dialogue in the prompt, (6) needing reference-conditioned generation with up to 9 images + 3 video clips + 3 audio clips, (7) wanting consistent character identity across shots. Accessible via fal.ai (`seedance_video` tool), HeyGen (Video Agent / Avatar Shots), Replicate, Runway (Enterprise, non-US), Freepik, BytePlus ModelArk, Higgsfield, Pollo, and other aggregators. +allowed-tools: Bash, Read, Write +metadata: + openclaw: + requires: + env_any: + - FAL_KEY + - HEYGEN_API_KEY + - REPLICATE_API_TOKEN +--- + +# Seedance 2.0 (ByteDance) + +Seedance 2.0 is the ByteDance Seed team's unified multimodal video+audio model (released Feb 2026, globally available via partner APIs April 2026). It is currently the **preferred premium default** for cinematic, trailer, teaser, and motion-led work inside OpenMontage when a paid gateway is configured. + +## Why it is the OpenMontage premium default + +| Capability | Seedance 2.0 | Notes | +|---|---|---| +| Single-pass native synced audio | Yes | Speech + SFX + ambience generated jointly, not post-sync | +| Multi-shot inside one generation | Yes | Multiple cuts/shots in a single prompt | +| Director-level camera control | Yes | Camera language (dolly, tilt, arc, crane, handheld) honored | +| Lip-sync from quoted dialogue | Yes | `Character says: "..."` matches mouth shapes | +| Reference conditioning | Up to 9 images + 3 video clips + 3 audio clips | 12-asset multimodal | +| Character identity consistency | Yes | Face/subject stable across shots | +| Max shot duration | 15 s | auto / 4–15 s | +| Resolution ceiling | 1080p on some endpoints (720p default on fal.ai) | Provider-dependent | +| Elo (Artificial Analysis) | 1269 (#1 as of Feb 2026) | Beat Veo 3, Sora 2, Runway Gen-4.5 | + +Switch away only for a specific reason: strict budget (use the `fast` variant or LTX), user-preferred provider (VEO/Sora/Kling), or a stylistic fit that favors another model. + +## Provider surfaces + +| Surface | Env | OpenMontage tool | Notes | +|---|---|---|---| +| **fal.ai** (primary) | `FAL_KEY` | `seedance_video` | Model IDs below. Supports T2V, I2V, reference-to-video; `standard` and `fast` variants. Default in OpenMontage. | +| **HeyGen** | `HEYGEN_API_KEY` | `heygen_video` (Video Agent / Avatar Shots) | Only platform with consent-verified real-face use via Avatar Shots. The `seedance_pro` / `seedance_lite` legacy provider strings on HeyGen map to Seedance 1.x — 2.0 access flows through the Video Agent / Avatar Shots endpoints rather than a stable provider parameter. | +| **Replicate** | `REPLICATE_API_TOKEN` | not wrapped | `bytedance/seedance-2.0`, `bytedance/seedance-2.0-fast` | +| **Runway** | Runway credentials | not wrapped | Third-party model inside Runway, **Unlimited/Enterprise plans, non-US only** | +| **BytePlus ModelArk / Volcengine** | BytePlus token | not wrapped | Direct from ByteDance. Pro ~$0.15 / 5 s, Lite ~$0.010/s. Token-based. | +| **Freepik** | Freepik token | not wrapped | `POST /v1/ai/image-to-video/seedance-pro-1080p` for 1080p I2V | +| **Higgsfield** | Higgsfield plan | not wrapped | Emphasis on character identity + long-form chaining | +| **Pollo / PiAPI / Atlas Cloud / AIMLAPI** | various | not wrapped | Aggregators resell fal.ai or ByteDance endpoints | + +### fal.ai model IDs (used by `seedance_video`) + +``` +bytedance/seedance-2.0/text-to-video +bytedance/seedance-2.0/image-to-video +bytedance/seedance-2.0/reference-to-video # 9 img + 3 vid + 3 audio +bytedance/seedance-2.0/fast/text-to-video +bytedance/seedance-2.0/fast/image-to-video +bytedance/seedance-2.0/fast/reference-to-video +``` + +Pricing (fal.ai, 720p): standard $0.3034 / s (T2V), $0.3024 / s (I2V). Fast $0.2419 / s across endpoints. +The `fast` variant trades some camera/motion fidelity for latency and cost — do **not** route slow-mo, multi-shot, or dolly-heavy prompts to `fast` on the first try. + +## Calling Seedance 2.0 inside OpenMontage + +Always go through `video_selector` with `preferred_provider="seedance"` (or let the scoring engine pick it): + +```python +from tools.tool_registry import registry +registry.ensure_discovered() +selector = registry.get("video_selector") +result = selector.execute({ + "prompt": PROMPT, + "preferred_provider": "seedance", + "operation": "text_to_video", # or image_to_video / reference_to_video + "aspect_ratio": "21:9", # 21:9 / 16:9 / 9:16 / 4:3 / 1:1 / 3:4 + "duration": "10", # auto / 4..15 + "resolution": "720p", # 480p / 720p + "output_path": "projects//assets/video/clip_01.mp4", +}) +``` + +Direct call to the provider tool (only when you must bypass the selector): + +```python +seedance = registry.get("seedance_video") +seedance.execute({ + "prompt": PROMPT, + "model_variant": "standard", # "standard" or "fast" + "operation": "text_to_video", + "aspect_ratio": "21:9", + "duration": "10", + "resolution": "720p", + "generate_audio": True, + "seed": 12345, # optional, for reproducible variations + "output_path": "...", +}) +``` + +## Prompt structure + +Seedance 2.0 is unusually literal about camera language, multi-shot cuts, and quoted dialogue. Use this 8-part template: + +``` +[Shot / framing] + [Camera movement] + +[Subject description — physical detail that must persist across shots] + +[Action beat 1] → [optional cut] → [Action beat 2] + +[Setting / environment] + [Lighting / palette] + +[Style / grade / era] + [Audio — ambient, diegetic, music, dialogue] +``` + +### Multi-shot inside one generation + +Seedance honors explicit shot lists inside a prompt. Format each shot: + +``` +Shot 1 (wide establishing, slow aerial push-in): ... +Shot 2 (medium close-up, handheld): ... +Shot 3 (extreme close-up, rack focus): ... +``` + +Keep subject description consistent across shots for identity stability. + +### Lip-sync from quoted dialogue + +``` +Aang stands on the cliff edge, staff raised, wind in his cloak. +Aang says: "I won't run anymore." +Sokka, half a step behind, replies: "Then we fight." +``` + +Use `Character says: "..."` / `Character replies: "..."` exactly — mouth shapes key off quoted strings. Keep each line under ~6 words; longer lines risk drift on fast clips. + +### Audio cues that work + +Ambient: `distant thunder rolling over mountains`, `wind through reeds`, `crackling campfire` +Diegetic: `boots crunching snow`, `staff planting on stone`, `wingbeats overhead` +Music direction (light touch only): `low orchestral swell building`, `taiko drums entering on Shot 3` +Do **not** request complex multi-instrument scores — keep music language textural. + +### Reference-to-video + +When you have character / product / wardrobe references, use the reference-to-video endpoint and name each asset in the prompt: + +``` +Reference 1: hero character (Aang) — bald, blue arrow tattoo, orange robes. +Reference 2: environment plate — snowy Air Temple courtyard at dawn. +Shot 1: Aang (from reference 1) walks across the courtyard (reference 2), +wind lifting his robes. Low-angle tracking shot, slow push-in. +``` + +## Parameter guidance + +| Parameter | Guidance | +|---|---| +| `duration` | `5`–`8` for hero shots, `10`–`12` for full scenes with multi-shot cuts, `4` for quick inserts. `auto` when unsure. | +| `aspect_ratio` | `21:9` for cinematic trailers, `16:9` for broadcast / YouTube, `9:16` for Reels/Shorts/TikTok | +| `resolution` | `720p` default. Drop to `480p` for cost-capped batch previews, not for finals | +| `generate_audio` | Keep **on** unless you have a specific reason to mute — Seedance's moat is synced audio. Strip audio downstream in compose if needed. | +| `model_variant` | `standard` for hero/cinematic shots; `fast` only for b-roll, previews, or when latency is the hard constraint | +| `seed` | Set a seed before iterating variants of a chosen shot — everything else held constant | + +## What to avoid + +| Don't | Why | +|---|---| +| Cram four-plus simultaneous character actions into one shot | Motion coherence breaks; split into multi-shot | +| Request readable text / logos inside the clip | Text rendering is unreliable — handle text in Remotion overlay | +| Mix conflicting lighting ("bright noon" + "neon night") | Model picks one and ignores the other | +| Write dialogue longer than ~6 words on fast-cut shots | Lip-sync drift | +| Use `fast` variant for slow-mo, multi-shot, or complex camera moves | Routinely misses on first try — route to `standard` | +| Generate music through Seedance audio | Texture-only is fine; for real scoring use `music` / `pixabay_music` / `elevenlabs` and mix in compose | +| Bypass `video_selector` without a reason | Loses cost/availability/fallback handling and scoring context | + +## Iteration strategy + +1. **Block out shape** with a single `duration=5` `fast` T2V pass at the intended framing. Confirm the composition works. +2. **Lock the seed** once the composition reads. +3. **Upgrade to `standard`** with the same seed, tighten camera and lighting language. +4. **Extend and add shots** — move to multi-shot or longer duration only after a single-shot version is clean. +5. **Keep a per-clip README** with prompt + seed + variant for every shot that makes the cut, so the compose stage can re-render consistent retakes. + +## Integration notes for OpenMontage pipelines + +- **Cinematic pipeline:** Seedance 2.0 is the default video model. Use 21:9 for hero, multi-shot for montage beats, reference-to-video when the brief has a visual bible. +- **Animated explainer:** Use Seedance 2.0 for the establishing / mood clips only; most shots should stay in Remotion. Don't replace Remotion motion graphics with Seedance — different tool, different job. +- **Screen demo / podcast / clip factory:** Seedance is not the right default — these are footage-led. Only use for stylized cold-opens. +- **Cost discipline:** `standard` at 10 s ≈ $3.03 per clip. Budget accordingly in the proposal stage. `fast` at 5 s ≈ $1.21 for previews. + +## Verification checklist for every Seedance shot + +- [ ] Motion reads coherently at the chosen shot length +- [ ] Audio is actually synced (check dialogue + foot/impact hits) +- [ ] Character identity matches reference / prior shots +- [ ] Camera direction matches the prompt (no auto-dolly when you asked for static) +- [ ] No readable text the model tried to render +- [ ] Grade matches the approved style playbook +- [ ] Output duration matches what you requested (some endpoints round) + +## Sources + +- fal.ai Seedance 2.0: https://fal.ai/seedance-2.0 +- fal.ai how-to-use: https://fal.ai/learn/tools/how-to-use-seedance-2-0 +- Replicate bytedance collection: https://replicate.com/bytedance +- HeyGen Seedance 2.0: https://www.heygen.com/blog/introducing-seedance-2-and-heygen +- Runway Seedance: https://runwayml.com/product/seedance +- BytePlus Dreamina Seedance 2.0: https://www.byteplus.com/en/product/seedance +- Freepik Seedance 2.0: https://www.freepik.com/seedance-2 +- Higgsfield Seedance 2.0: https://higgsfield.ai/seedance/2.0 +- Pollo Seedance 2.0: https://pollo.ai/m/seedance/seedance-2-0 +- ByteDance Seed official: https://seed.bytedance.com/en/seedance2_0 +- Seedance 2.0 Wikipedia: https://en.wikipedia.org/wiki/Seedance_2.0 diff --git a/.claude/skills/ai-video-gen/SKILL.md b/.claude/skills/ai-video-gen/SKILL.md index 031c8e02..feb155ac 100644 --- a/.claude/skills/ai-video-gen/SKILL.md +++ b/.claude/skills/ai-video-gen/SKILL.md @@ -17,10 +17,12 @@ Generate AI videos from text prompts. Supports multiple providers via two API ga | Gateway | Env Variable | Providers | Tool | |---------|-------------|-----------|------| -| **fal.ai** | `FAL_KEY` | Kling v3/v2.1, MiniMax, VEO | `kling_video`, `minimax_video`, `veo_video` | -| **HeyGen** | `HEYGEN_API_KEY` | VEO 3.1, Kling Pro, Sora v2, Runway Gen-4, Seedance | `heygen_video` | +| **fal.ai** | `FAL_KEY` | **Seedance 2.0** (standard + fast), Kling v3/v2.1, MiniMax, VEO | `seedance_video`, `kling_video`, `minimax_video`, `veo_video` | +| **HeyGen** | `HEYGEN_API_KEY` | VEO 3.1, Kling Pro, Sora v2, Runway Gen-4, Seedance Pro / Lite (1.x) | `heygen_video` | -**IMPORTANT:** Always use `video_selector` instead of calling provider tools directly. The selector handles availability checks, cost comparison, and automatic fallback. +**Preferred premium default — Seedance 2.0.** When any premium gateway is configured (`FAL_KEY` → `seedance_video`, or HeyGen's Video Agent / Avatar Shots path), Seedance 2.0 is the preferred default for cinematic, trailer, and high-fidelity clip work. It is the only model in the fleet with **single-pass native synchronized audio, multi-shot generation, director-level camera control, and lip-sync from quoted dialogue**, and it ranks #1 on Artificial Analysis Elo as of early 2026. Switch off it only when the user has a specific reason (budget, provider preference, stylistic fit like VEO for photoreal landscape or Kling for specific anime look). See Layer 3 `seedance-2-0` for the authoritative prompting and parameter guide. + +**IMPORTANT:** Always use `video_selector` instead of calling provider tools directly. The selector handles availability checks, cost comparison, and automatic fallback, and its scoring engine already biases toward Seedance 2.0 for cinematic intent. ## Authentication @@ -312,7 +314,7 @@ curl -X POST "https://api.heygen.com/v1/workflows/executions" \ ## Best Practices 1. **Be descriptive in prompts** — include camera movement, lighting, style, and mood details -2. **Default to VEO 3.1** for highest quality; use `ltx_distilled` or `veo3_fast` when speed matters +2. **Default to Seedance 2.0 (via `seedance_video`) for cinematic and motion-led work** when `FAL_KEY` is set — single-pass synced audio, multi-shot, lip-sync, director-level camera. Use VEO 3.1 / Sora V2 Pro when the user specifically wants Google or OpenAI motion character; use `ltx_distilled` or `veo3_fast` only when speed is the hard constraint 3. **Use reference images** for image-to-video generation — great for animating product photos or still images 4. **Video generation is the slowest workflow** — allow up to 5 minutes, poll every 10 seconds 5. **Aspect ratio matters** — use `9:16` for social media stories/reels, `16:9` for landscape, `1:1` for square diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index beb9031c..75e461d9 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -604,7 +604,7 @@ The `.agents/skills/` directory is large. When you're not coming in through a to | **Composition runtime** | `remotion`, `remotion-best-practices`, `synthetic-screen-recording` (fake terminal/UI demos via Remotion TerminalScene) | | **Animation knowledge (generic)** | `gsap-core`, `gsap-timeline`, `gsap-plugins` (SplitText / MorphSVG / DrawSVG / MotionPath / Flip / CustomEase), `gsap-utils`, `gsap-react`, `gsap-performance`, `gsap-scrolltrigger`, `gsap-frameworks`, `framer-motion` (Disney 12 principles), `lottie-bodymovin` (Lottie export) | | **Image generation** | `bfl-api`, `flux-best-practices` | -| **Video generation** | `ai-video-gen`, `ltx2` | +| **Video generation** | `seedance-2-0` (preferred premium default — cinematic, trailer, multi-shot, synced audio, lip-sync), `ai-video-gen`, `ltx2` | | **Audio** | `elevenlabs`, `music`, `sound-effects`, `acestep`, `text-to-speech`, `setup-api-key` | | **Avatar / lip-sync** | `avatar-video`, `heygen`, `create-video`, `faceswap`, `video-translate`, `speech-to-text`, `agents` | | **Capture** | `playwright-recording` (browser flows), `ffmpeg` (post) | diff --git a/lib/scoring.py b/lib/scoring.py index ed586890..618c6bb2 100644 --- a/lib/scoring.py +++ b/lib/scoring.py @@ -112,14 +112,23 @@ class ProductionPathScore: # --------------------------------------------------------------------------- def _keyword_overlap(set_a: set[str], set_b: set[str]) -> float: - """Jaccard-like overlap score between two keyword sets.""" + """Overlap coefficient between two keyword sets. + + Uses |A ∩ B| / min(|A|, |B|) rather than Jaccard. Jaccard over-penalizes + tools whose best_for describes many strengths — a premium provider with + seven rich bullets ends up with a smaller Jaccard than a narrowly-scoped + provider with one bullet, even when the premium provider fully covers the + intent. Overlap coefficient answers the relevant question: "is the intent + a subset of what this tool advertises?" which is what we actually care + about for provider scoring. + """ if not set_a or not set_b: return 0.0 a = {s.lower().strip() for s in set_a} b = {s.lower().strip() for s in set_b} intersection = len(a & b) - union = len(a | b) - return intersection / union if union > 0 else 0.0 + smaller = min(len(a), len(b)) + return intersection / smaller if smaller > 0 else 0.0 # Semantic synonym clusters: when intent says "cinematic" and tool says @@ -200,16 +209,18 @@ def _compute_task_fit( ) -> float: """Score how well a tool's best_for matches the task intent and style. - Uses synonym expansion so that semantic near-misses (e.g. "cinematic" - vs "film") still score well, not just literal keyword overlap. + Uses synonym expansion and a real tokenizer so that semantic near-misses + (e.g. "cinematic" vs "film") and punctuation-adjacent tokens (e.g. + "trailers," vs "trailer") still score well, not just literal whitespace + splits. """ if not best_for: return 0.3 # Unknown capability — modest default - intent_words = _expand_synonyms(set(intent.lower().split())) - best_for_words = set() + intent_words = _expand_synonyms(set(_tokenize_text(intent))) + best_for_words: set[str] = set() for desc in best_for: - best_for_words.update(desc.lower().split()) + best_for_words.update(_tokenize_text(desc)) best_for_words = _expand_synonyms(best_for_words) intent_score = _keyword_overlap(intent_words, best_for_words) @@ -374,7 +385,11 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore: """ task_context = normalize_task_context(task_context) info = tool.get_info() - status = str(tool.get_status()) + # .value on the ToolStatus enum returns "available" / "degraded" / "unavailable". + # str() on the enum returns "ToolStatus.AVAILABLE", which never matches the + # lowercase branches below — older code had every available tool scoring 0.0 + # on reliability. + status = tool.get_status().value best_for = set(info.get("best_for", [])) intent = task_context.get("intent", "") @@ -477,6 +492,31 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore: else: task_fit *= 0.7 + # Premium-cinematic bonus: when a video task has cinematic/trailer intent, + # reward providers that ship the premium feature set — native synchronized + # audio, multi-shot single-generation, director-level camera control, + # lip-sync from quoted dialogue. This is what makes Seedance 2.0 (and + # peer premium APIs) meaningfully better than generic clip providers. + if asset_type == "video": + intent_words = _expand_synonyms(set(intent.lower().split())) | set(style_keywords) + cinematic_signal = bool( + intent_words & {"cinematic", "film", "movie", "trailer", "teaser", "dramatic", "epic", "premium"} + ) + if cinematic_signal: + premium_features = [ + supports.get("native_audio"), + supports.get("multi_shot"), + supports.get("camera_direction"), + supports.get("lip_sync"), + supports.get("cinematic_quality"), + ] + matched = sum(1 for f in premium_features if f) + if matched >= 3: + task_fit = min(1.0, task_fit + 0.15) + output_quality = min(1.0, output_quality + 0.10) + elif matched >= 1: + task_fit = min(1.0, task_fit + 0.05) + return ProviderScore( tool_name=info.get("name", "unknown"), provider=info.get("provider", "unknown"), diff --git a/pipeline_defs/cinematic.yaml b/pipeline_defs/cinematic.yaml index 89bffdb2..c3b7594b 100644 --- a/pipeline_defs/cinematic.yaml +++ b/pipeline_defs/cinematic.yaml @@ -171,12 +171,16 @@ stages: - audio_enhance - image_selector - video_selector + - pixabay_music + - freesound_music - music_gen tools_available: - subtitle_gen - audio_enhance - image_selector - video_selector + - pixabay_music + - freesound_music - music_gen checkpoint_required: true human_approval_default: false diff --git a/skills/INDEX.md b/skills/INDEX.md index 82987c60..ce6f6302 100644 --- a/skills/INDEX.md +++ b/skills/INDEX.md @@ -92,6 +92,7 @@ Key capability families to look for in the output: | Data Visualization | `creative/data-visualization.md` | Chart type selection, animation, label placement | `d3-viz`, `remotion-best-practices` | | Video Stitching | `creative/video-stitching.md` | Multi-clip assembly, AI clip chaining, spatial composition | `ffmpeg`, `video_toolkit` | | Video Gen Prompting | `creative/video-gen-prompting.md` | Universal video generation prompt vocabulary | `ai-video-gen`, `ltx2`, `create-video` | +| ↳ Seedance Prompting | `creative/prompting/seedance-prompting.md` | **Preferred premium default.** Seedance 2.0 8-component structure, multi-shot, lip-sync, reference-to-video | `seedance-2-0`, `ai-video-gen` | | ↳ Grok Prompting | `creative/prompting/grok-prompting.md` | Grok image/video prompting, edit flows, reference-image video | `grok-media` | | ↳ Sora Prompting | `creative/prompting/sora-prompting.md` | Sora 2 structured template, advanced fields | `ai-video-gen` | | ↳ VEO Prompting | `creative/prompting/veo-prompting.md` | VEO 3.1 14-component structure, art movements | `ai-video-gen` | @@ -306,4 +307,5 @@ Claude Code accesses them via symlinks in `.claude/skills/`. | **Animation** | `framer-motion`, `lottie-bodymovin` | `pproenca/dot-skills`, `dylantarre/animation-principles` | | **Design** | `tailwind-design-system`, `web-design-guidelines`, `vercel-react-best-practices`, `vercel-composition-patterns` | `wshobson/agents`, `vercel-labs/agent-skills` | | **AI Video (HeyGen)** | `heygen`, `avatar-video`, `create-video`, `faceswap`, `ai-video-gen`, `video-download`, `video-edit`, `video-translate`, `video-understand`, `visual-style` | `heygen-com/skills` | +| **AI Video (Premium)** | `seedance-2-0` — preferred premium default (cinematic, trailer, multi-shot, lip-sync, synced audio); accessed via `seedance_video` (fal.ai) or `heygen_video` Avatar Shots | Local OpenMontage skill | | **Infrastructure** | `acestep`, `ltx2`, `playwright-recording` | `digitalsamba/claude-code-video-toolkit` | diff --git a/skills/creative/prompting/seedance-prompting.md b/skills/creative/prompting/seedance-prompting.md new file mode 100644 index 00000000..9bc8a2ef --- /dev/null +++ b/skills/creative/prompting/seedance-prompting.md @@ -0,0 +1,138 @@ +# Seedance 2.0 — Prompting Guide + +> Layer 3 authority: `.agents/skills/seedance-2-0/SKILL.md` +> For universal vocabulary, see: `skills/creative/video-gen-prompting.md` + +## When to pick Seedance 2.0 + +Seedance 2.0 (ByteDance Seed team, released Feb 2026) is OpenMontage's **preferred premium default for cinematic, trailer, teaser, hype-edit, and motion-led clip work** whenever a paid gateway is configured (`FAL_KEY` via `seedance_video`, or HeyGen Video Agent / Avatar Shots). It is the only model in the fleet that delivers all of: + +- single-pass native synchronized audio (speech + SFX + ambience together, not post-sync), +- multi-shot generation inside a single prompt, +- director-level camera control, +- lip-sync from quoted dialogue, +- reference-conditioned generation with up to 9 images + 3 video clips + 3 audio clips, +- consistent character identity across shots. + +Elo 1269 on Artificial Analysis as of release — ahead of Veo 3, Sora 2, Runway Gen-4.5. + +Switch off Seedance 2.0 only when there is a real reason: strict budget (use the `fast` variant or LTX), explicit user preference (VEO/Sora/Kling), or a stylistic fit another model does better (VEO for photoreal landscape, Kling for anime). + +## Seedance 2.0 8-Component Prompt Structure + +Seedance is unusually literal about camera language, multi-shot cuts, and quoted dialogue. Use this structure — include what matters, omit what doesn't: + +1. **Shot / framing** — wide establishing, medium, close-up, Dutch angle, etc. +2. **Camera movement** — static, slow push-in, aerial, handheld, arc, dolly zoom +3. **Subject description** — the physical detail that must persist across shots (identity anchor) +4. **Action beats** — one beat per sentence, use `→` or explicit `Shot 1 / Shot 2` for multi-shot +5. **Setting / environment** — location, era, weather, time of day +6. **Lighting / palette** — one lighting idea, pick and commit +7. **Style / grade / era** — "anamorphic lens, teal-orange grade, 35mm film grain" +8. **Audio** — ambient, diegetic, music direction (textural only), quoted dialogue for lip-sync + +## Seedance-specific strengths + +| Capability | How to invoke it | +|---|---| +| **Native synced audio** | Describe the soundscape in the prompt. Leave `generate_audio=true`. | +| **Multi-shot in one generation** | Use `Shot 1 (...)`, `Shot 2 (...)` etc. Keep subject description consistent across shots. | +| **Director-level camera** | Use unambiguous terms: `slow dolly-in`, `arc shot`, `Dutch tilt`, `aerial push-in`, `handheld with micro-shake` | +| **Lip-sync from quoted dialogue** | `Character says: "line."` — each line ≤ ~6 words on fast cuts | +| **Reference-to-video** | Use the `reference-to-video` endpoint; name each asset in the prompt (`Reference 1: hero character — ...`) | +| **Character identity consistency** | Describe the same physical details in every shot — Seedance uses those as the identity anchor | + +## Multi-shot pattern + +Seedance honors explicit shot lists: + +``` +Shot 1 (wide aerial establishing, slow push-in): +Snow-covered Air Temple at dawn, spires catching first orange light. +Wind lifting prayer flags. + +Shot 2 (medium, low angle, handheld): +Aang — bald, blue arrow tattoo, orange robes — plants his staff on stone. +He squints into the rising sun. + +Shot 3 (extreme close-up, rack focus): +Rack focus from the glowing arrow tattoo on his forehead to the distant peaks. +Aang says: "It's time." + +Style: anamorphic lens, teal-orange cinematic grade, 35mm film grain. +Audio: rising orchestral swell with low taiko pulse, wind, distant wingbeats. +``` + +## Lip-sync pattern + +``` +Aang says: "I won't run anymore." +Sokka, half a step behind, replies: "Then we fight." +``` + +- Use `Character says: "..."` / `Character replies: "..."` exactly — mouth shapes key off the quoted strings. +- Keep lines short (≤ 6 words on fast-cut shots) to avoid drift. +- For a single-speaker monologue, keep the camera close and static on the speaker's shot. + +## Parameter cheat sheet + +| Parameter | Guidance | +|---|---| +| `duration` | `5`–`8` s hero, `10`–`12` s multi-shot scenes, `4` s inserts. `auto` when unsure. | +| `aspect_ratio` | `21:9` trailers, `16:9` broadcast, `9:16` Reels/Shorts/TikTok | +| `resolution` | `720p` default. `480p` for cost-capped previews only. | +| `generate_audio` | Keep `true` — sync audio is the moat. Strip in compose if unused. | +| `model_variant` | `standard` for hero + multi-shot + camera-heavy. `fast` for b-roll, previews, latency-capped jobs. | +| `seed` | Lock once a shot composition reads; iterate variants with the same seed. | + +## Iteration strategy + +1. **Block out shape** — `duration=5`, `fast`, one shot. Confirm composition. +2. **Lock the seed** — record it in the per-clip README. +3. **Upgrade to `standard`** — same seed, tighten camera + lighting language. +4. **Extend or multi-shot** — only after the single-shot version is clean. +5. **Promote to final** — write the prompt, seed, variant, and duration into the asset manifest so compose can re-render consistent retakes. + +## What to avoid + +| Don't | Why | +|---|---| +| Four-plus simultaneous actions in one shot | Motion coherence collapses. Split to multi-shot. | +| Readable text / logos inside the clip | Text rendering is unreliable. Handle text in Remotion overlay. | +| Conflicting lighting (`bright noon` + `neon night`) | Model picks one and ignores the other. | +| Long dialogue on fast-cut shots | Lip-sync drifts. | +| `fast` variant for slow-mo, multi-shot, or complex camera | Routinely misses on first try. Route to `standard`. | +| Request a full multi-instrument score from Seedance | Keep audio direction textural; real scoring belongs in `music` / `pixabay_music` / `elevenlabs` and mixes in compose. | +| Bypass `video_selector` without a reason | Loses scoring, fallback, and cost handling. | + +## Integration notes + +- **Cinematic pipeline:** Seedance 2.0 is the default. 21:9, multi-shot for montage, reference-to-video when the brief has a visual bible. +- **Animated explainer:** Use Seedance 2.0 only for establishing / mood / cold-open clips — core motion graphics stay in Remotion. +- **Screen demo / podcast / clip factory:** Not the right default. Only for stylized cold-opens. +- **Cost check:** `standard` at 10 s ≈ $3.03 / clip on fal.ai. `fast` at 5 s ≈ $1.21. Budget in the proposal stage. + +## Example — Airbender trailer hero beat (60 s total trailer, this is shot 3 of 7) + +``` +Shot 1 (wide aerial, slow push-in, 3s): +Snow-covered Air Temple at dawn, spires catching orange light, +prayer flags lifting in wind. + +Shot 2 (low angle medium, handheld, 3s): +Aang — bald, blue arrow tattoo on forehead, orange and yellow robes — +plants his staff on weathered stone, squints into the rising sun. + +Shot 3 (extreme close-up, rack focus, 3s): +Rack focus from the glowing arrow on his forehead to distant peaks. +Aang says: "It's time." + +Lighting: cold blue ambient with warm break on the horizon, +rim light from rising sun. +Style: anamorphic 2.39:1, teal-orange cinematic grade, 35mm film grain, +halation on speculars. +Audio: low taiko drums rising to orchestral swell on Shot 3, +wind through temple, distant wingbeats, leather staff-grip creak. +``` + +Parameters: `duration=10`, `aspect_ratio=21:9`, `resolution=720p`, `model_variant=standard`, `generate_audio=true`, seed locked after shot 2. diff --git a/skills/creative/video-gen-prompting.md b/skills/creative/video-gen-prompting.md index 24c7f11a..1b606be1 100644 --- a/skills/creative/video-gen-prompting.md +++ b/skills/creative/video-gen-prompting.md @@ -2,9 +2,11 @@ ## When to Use -When writing prompts for the video generation family (`video_selector`, `heygen_video`, -`wan_video`, `hunyuan_video`, `ltx_video_local`, `ltx_video_modal`, `cogvideo_video`). -This skill covers the universal prompt vocabulary that works across all video generation models. +When writing prompts for the video generation family (`video_selector`, `seedance_video`, +`heygen_video`, `wan_video`, `hunyuan_video`, `ltx_video_local`, `ltx_video_modal`, +`cogvideo_video`). This skill covers the universal prompt vocabulary that works across all +video generation models. For the **preferred premium default**, see the Seedance 2.0 row +in the table below. For model-specific tips, see the linked guides below. @@ -12,6 +14,7 @@ For model-specific tips, see the linked guides below. | Model | Guide | Key Insight | |-------|-------|-------------| +| **Seedance 2.0 (standard / fast)** | `creative/prompting/seedance-prompting.md` + Layer 3 `.agents/skills/seedance-2-0/` | **Preferred premium default** when `FAL_KEY` or HeyGen is configured. Single-pass synced audio, multi-shot generation, director-level camera, lip-sync from quoted dialogue, reference-to-video (9 img + 3 vid + 3 audio). Elo 1269 (#1 on Artificial Analysis). | | **Sora 2 / Sora 2 Pro** | [OpenAI Sora 2 Cookbook](https://developers.openai.com/cookbook/examples/sora/sora2_prompting_guide) | Richest structured template. Advanced fields: lenses, filtration, grade, diegetic sound, wardrobe, finishing. | | **VEO 3.1 / VEO 3** | [Vertex AI Prompt Guide](https://cloud.google.com/vertex-ai/generative-ai/docs/video/video-gen-prompt-guide) | Best vocabulary reference tables. 14-component prompt structure. | | **Grok Imagine Video** | `creative/prompting/grok-prompting.md` | Best when prompts need reference-image placeholders like `` and identity/product carryover. | diff --git a/skills/pipelines/cinematic/asset-director.md b/skills/pipelines/cinematic/asset-director.md index 7c829745..0a818561 100644 --- a/skills/pipelines/cinematic/asset-director.md +++ b/skills/pipelines/cinematic/asset-director.md @@ -27,7 +27,7 @@ Before authoring title cards, name plates, or SVG overlays, read **`skills/meta/ |-------|----------|---------| | Schema | `schemas/artifacts/asset_manifest.schema.json` | Artifact validation | | Prior artifacts | `state.artifacts["scene_plan"]["scene_plan"]`, `state.artifacts["script"]["script"]`, `state.artifacts["proposal"]["proposal_packet"]` | Scene intent and beat plan | -| Tools | `subtitle_gen`, `audio_enhance`, `image_selector`, `video_selector`, `music_gen` — selectors auto-discover all available providers from the registry | Optional support asset creation | +| Tools | `subtitle_gen`, `audio_enhance`, `image_selector`, `video_selector`, `pixabay_music` (free, default), `freesound_music` (free), `music_gen` (ElevenLabs, paid) — selectors auto-discover all available providers from the registry. **Default to `pixabay_music` before reaching for `music_gen`.** | Optional support asset creation | | Playbook | Active style playbook | Brand and typography consistency | ## Process @@ -54,7 +54,7 @@ If `proposal_packet.metadata.motion_required = true`, actual moving footage or g Before batch-generating support assets, produce one sample of each expensive generated type and show the user: 1. **Generated insert sample** (if using `image_selector` or `video_selector`): Generate one representative visual. Confirm it complements the source footage before batching. -2. **Music sample** (if using `music_gen`): Generate a short clip. Confirm mood and energy match the beat plan. +2. **Music sample** (try `pixabay_music` first — free, searchable by mood/BPM; fall back to `freesound_music` for cues and ambience; only reach for `music_gen` when the search tools miss the brief): sample or retrieve a short clip. Confirm mood and energy match the beat plan. If `motion_required = true`, the representative visual must be a video clip sample, not a still image sample. diff --git a/skills/pipelines/cinematic/compose-director.md b/skills/pipelines/cinematic/compose-director.md index 694adb2f..c6c346df 100644 --- a/skills/pipelines/cinematic/compose-director.md +++ b/skills/pipelines/cinematic/compose-director.md @@ -24,6 +24,20 @@ If the approved brief or scene plan makes motion a hard requirement, verify that - Do not convert the piece into an animatic unless the user explicitly approves that downgrade. - If the render engine changes materially, tell the user before rendering and explain why. +**Mandatory Remotion preflight (run before every render when the scene plan includes any Remotion scene type — title cards, stat cards, anime/hero_title, end-tag, overlays):** + +```bash +python -c " +from tools.tool_registry import registry +registry.discover() +info = registry.get('video_compose').get_info() +print('Render engines:', info.get('render_engines')) +print('Remotion note:', info.get('remotion_note')) +" +``` + +If Remotion is not in the available render engines, stop and report to the user per the Decision Communication Contract. Do not substitute a reduced-fidelity render path without approval. + ### 1. Use Frame Treatment Deliberately Only use letterbox, 24fps intent, or heavy grading if they help the piece. Do not apply them because the pipeline name says cinematic. diff --git a/tools/base_tool.py b/tools/base_tool.py index 024a5703..f9300d44 100644 --- a/tools/base_tool.py +++ b/tools/base_tool.py @@ -181,6 +181,14 @@ class BaseTool(ABC): # --- Verification --- user_visible_verification: list[str] = [] + # --- Optional telemetry / quality hints for the scoring engine --- + # If set (0.0-1.0), lib/scoring.py uses these directly instead of falling + # back to stability-based heuristics. Leave unset unless the tool has a + # real measured or well-calibrated value. + quality_score: Optional[float] = None + historical_success_rate: Optional[float] = None + latency_p50_seconds: Optional[float] = None + # ---- Status reporting ---- def get_status(self) -> ToolStatus: @@ -255,6 +263,9 @@ class BaseTool(ABC): "agent_skills": self.agent_skills, "related_skills": self.agent_skills, "user_visible_verification": self.user_visible_verification, + "quality_score": self.quality_score, + "historical_success_rate": self.historical_success_rate, + "latency_p50_seconds": self.latency_p50_seconds, } # ---- Cost estimation ---- diff --git a/tools/video/seedance_video.py b/tools/video/seedance_video.py index c85bcc16..2f7d84ac 100644 --- a/tools/video/seedance_video.py +++ b/tools/video/seedance_video.py @@ -27,11 +27,11 @@ from tools.base_tool import ( class SeedanceVideo(BaseTool): name = "seedance_video" - version = "0.1.0" + version = "0.2.0" tier = ToolTier.GENERATE capability = "video_generation" provider = "seedance" - stability = ToolStability.EXPERIMENTAL + stability = ToolStability.BETA execution_mode = ExecutionMode.SYNC determinism = Determinism.STOCHASTIC runtime = ToolRuntime.API @@ -41,25 +41,36 @@ class SeedanceVideo(BaseTool): "Set FAL_KEY to your fal.ai API key.\n" " Get one at https://fal.ai/dashboard/keys" ) - agent_skills = ["ai-video-gen"] + agent_skills = ["seedance-2-0", "ai-video-gen"] - capabilities = ["text_to_video", "image_to_video"] + capabilities = ["text_to_video", "image_to_video", "reference_to_video"] supports = { "text_to_video": True, "image_to_video": True, + "reference_to_video": True, + "multiple_reference_images": True, + "reference_image": True, "native_audio": True, "cinematic_quality": True, "camera_direction": True, "lip_sync": True, + "multi_shot": True, + "aspect_ratio": True, + "seed": True, } best_for = [ - "cinematic clips with native synchronized audio", - "director-level camera control and multi-shot editing", + "preferred premium video gen when FAL_KEY is available", + "cinematic trailers, teasers, and high-fidelity clips with native synchronized audio", + "director-level camera control and multi-shot editing in a single generation", "lip-sync from quoted dialogue in prompts", - "high-fidelity motion with real-world physics", + "reference-conditioned generation (up to 9 images + 3 video clips + 3 audio clips)", + "consistent character identity across shots", ] not_good_for = ["offline generation", "budget-constrained projects"] - fallback_tools = ["kling_video", "minimax_video", "veo_video"] + fallback_tools = ["veo_video", "kling_video", "minimax_video"] + # Premium model — beat out "experimental stability" baseline. The scoring + # engine reads quality_score directly when present (see lib/scoring.py). + quality_score = 0.95 input_schema = { "type": "object", @@ -68,7 +79,7 @@ class SeedanceVideo(BaseTool): "prompt": {"type": "string"}, "operation": { "type": "string", - "enum": ["text_to_video", "image_to_video"], + "enum": ["text_to_video", "image_to_video", "reference_to_video"], "default": "text_to_video", }, "model_variant": { @@ -102,10 +113,34 @@ class SeedanceVideo(BaseTool): "type": "string", "description": "Start frame image URL for image_to_video (jpg, png, webp)", }, + "image_path": { + "type": "string", + "description": "Local start-frame path for image_to_video. Auto-uploaded to fal.ai storage.", + }, "end_image_url": { "type": "string", "description": "Optional end frame URL for image_to_video", }, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Up to 9 reference image URLs for reference_to_video (identity / wardrobe / setting / style anchors).", + }, + "reference_image_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Local reference image paths for reference_to_video. Auto-uploaded to fal.ai storage.", + }, + "reference_video_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Up to 3 reference video clip URLs for reference_to_video (motion / camera / pacing anchors).", + }, + "reference_audio_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Up to 3 reference audio clip URLs for reference_to_video (voice / music / ambience anchors).", + }, "seed": { "type": "integer", "description": "Optional seed for reproducibility", @@ -180,11 +215,41 @@ class SeedanceVideo(BaseTool): if inputs.get("image_url"): payload["image_url"] = inputs["image_url"] elif inputs.get("image_path"): - from tools.video._shared import upload_image_to_fal + from tools.video._shared import upload_image_fal payload["image_url"] = upload_image_to_fal(inputs["image_path"]) if inputs.get("end_image_url"): payload["end_image_url"] = inputs["end_image_url"] + if operation == "reference_to_video": + ref_image_urls = list(inputs.get("reference_image_urls") or []) + for local_path in inputs.get("reference_image_paths") or []: + from tools.video._shared import upload_image_fal + ref_image_urls.append(upload_image_to_fal(local_path)) + # Seedance 2.0 reference-to-video ceilings: 9 images + 3 video + 3 audio. + if len(ref_image_urls) > 9: + return ToolResult( + success=False, + error=f"Seedance 2.0 reference_to_video accepts at most 9 reference images; got {len(ref_image_urls)}", + ) + ref_video_urls = list(inputs.get("reference_video_urls") or []) + if len(ref_video_urls) > 3: + return ToolResult( + success=False, + error=f"Seedance 2.0 reference_to_video accepts at most 3 reference videos; got {len(ref_video_urls)}", + ) + ref_audio_urls = list(inputs.get("reference_audio_urls") or []) + if len(ref_audio_urls) > 3: + return ToolResult( + success=False, + error=f"Seedance 2.0 reference_to_video accepts at most 3 reference audio clips; got {len(ref_audio_urls)}", + ) + if ref_image_urls: + payload["reference_image_urls"] = ref_image_urls + if ref_video_urls: + payload["reference_video_urls"] = ref_video_urls + if ref_audio_urls: + payload["reference_audio_urls"] = ref_audio_urls + headers = { "Authorization": f"Key {api_key}", "Content-Type": "application/json",