diff --git a/.agents/skills/seedance-2-0/SKILL.md b/.agents/skills/seedance-2-0/SKILL.md index f8526111..a5ec54ee 100644 --- a/.agents/skills/seedance-2-0/SKILL.md +++ b/.agents/skills/seedance-2-0/SKILL.md @@ -14,7 +14,7 @@ metadata: # Seedance 2.0 (ByteDance) -Seedance 2.0 is the ByteDance Seed team's unified multimodal video+audio model (released Feb 2026, globally available via partner APIs April 2026). It is currently the **preferred premium default** for cinematic, trailer, teaser, and motion-led work inside OpenMontage when a paid gateway is configured. +Seedance 2.0 is the ByteDance Seed team's unified multimodal video+audio model (released Feb 2026, globally available via partner APIs April 2026). It is the **preferred premium default** for cinematic, trailer, teaser, and motion-led work inside OpenMontage whenever any supporting gateway is configured. OpenMontage wraps four gateways directly (`seedance_video` → fal.ai, `seedance_replicate` → Replicate, `runway_video` with `model="seedance_2.0"` → Runway, `higgsfield_video` with `model="seedance_2.0"` → Higgsfield); BytePlus / Freepik / HeyGen-Video-Agent wrappers are on the roadmap. The scoring engine deduplicates by `provider="seedance"` so whichever gateway the user has configured wins automatically — agents should pass `preferred_provider="seedance"` to `video_selector` (or let the scorer pick) rather than routing to a specific gateway by name. ## Why it is the OpenMontage premium default @@ -34,16 +34,16 @@ Switch away only for a specific reason: strict budget (use the `fast` variant or ## Provider surfaces -| Surface | Env | OpenMontage tool | Notes | -|---|---|---|---| -| **fal.ai** (primary) | `FAL_KEY` | `seedance_video` | Model IDs below. Supports T2V, I2V, reference-to-video; `standard` and `fast` variants. Default in OpenMontage. | -| **HeyGen** | `HEYGEN_API_KEY` | `heygen_video` (Video Agent / Avatar Shots) | Only platform with consent-verified real-face use via Avatar Shots. The `seedance_pro` / `seedance_lite` legacy provider strings on HeyGen map to Seedance 1.x — 2.0 access flows through the Video Agent / Avatar Shots endpoints rather than a stable provider parameter. | -| **Replicate** | `REPLICATE_API_TOKEN` | not wrapped | `bytedance/seedance-2.0`, `bytedance/seedance-2.0-fast` | -| **Runway** | Runway credentials | not wrapped | Third-party model inside Runway, **Unlimited/Enterprise plans, non-US only** | -| **BytePlus ModelArk / Volcengine** | BytePlus token | not wrapped | Direct from ByteDance. Pro ~$0.15 / 5 s, Lite ~$0.010/s. Token-based. | -| **Freepik** | Freepik token | not wrapped | `POST /v1/ai/image-to-video/seedance-pro-1080p` for 1080p I2V | -| **Higgsfield** | Higgsfield plan | not wrapped | Emphasis on character identity + long-form chaining | -| **Pollo / PiAPI / Atlas Cloud / AIMLAPI** | various | not wrapped | Aggregators resell fal.ai or ByteDance endpoints | +| Surface | Env | OpenMontage tool | Status | Notes | +|---|---|---|---|---| +| **fal.ai** (primary) | `FAL_KEY` | `seedance_video` | ✅ wrapped | Model IDs below. Supports T2V, I2V, reference-to-video; `standard` and `fast` variants. Default in OpenMontage. | +| **Replicate** | `REPLICATE_API_TOKEN` | `seedance_replicate` | ✅ wrapped | `bytedance/seedance-2.0` + `bytedance/seedance-2.0-fast`. Standard Replicate prediction API. | +| **Runway** | `RUNWAY_API_KEY` | `runway_video` (model: `seedance_2.0`) | ✅ wrapped | Third-party Seedance 2.0 model inside Runway. **Unlimited/Enterprise plans, non-US only**. Selected via `model` param. | +| **Higgsfield** | `HIGGSFIELD_API_KEY` + `_SECRET` | `higgsfield_video` (model: `seedance_2.0`) | ✅ wrapped | Seedance 2.0 is the default model on this tool. Emphasis on character identity + long-form chaining. | +| **HeyGen** | `HEYGEN_API_KEY` | `heygen_video` (1.x only) + TODO | ⚠️ 1.x only | The `seedance_pro` / `seedance_lite` workflow provider strings on HeyGen map to Seedance 1.x. 2.0 access flows through Video Agent / Avatar Shots endpoints — a separate `seedance_heygen` tool is on the roadmap. | +| **BytePlus ModelArk / Volcengine** | BytePlus token | not wrapped | 🔜 roadmap | Direct from ByteDance. Pro ~$0.15 / 5 s, Lite ~$0.010/s. Token-based. | +| **Freepik** | Freepik token | not wrapped | 🔜 roadmap | `POST /v1/ai/image-to-video/seedance-pro-1080p` for 1080p I2V | +| **Pollo / PiAPI / Atlas Cloud / AIMLAPI** | various | not wrapped | 🔜 roadmap | Aggregators resell fal.ai or ByteDance endpoints | ### fal.ai model IDs (used by `seedance_video`) diff --git a/tools/video/_shared.py b/tools/video/_shared.py index 459a85c8..762d0dbf 100644 --- a/tools/video/_shared.py +++ b/tools/video/_shared.py @@ -23,8 +23,12 @@ HEYGEN_PROVIDERS = { "sora_v2": {"name": "Sora v2", "quality": "high", "speed": "slow"}, "sora_v2_pro": {"name": "Sora v2 Pro", "quality": "highest", "speed": "slow"}, "runway_gen4": {"name": "Runway Gen-4", "quality": "high", "speed": "medium"}, - "seedance_lite": {"name": "Seedance Lite", "quality": "medium", "speed": "fast"}, - "seedance_pro": {"name": "Seedance Pro", "quality": "high", "speed": "medium"}, + # NOTE: HeyGen's `seedance_lite` / `seedance_pro` provider strings map to + # Seedance 1.x. Seedance 2.0 on HeyGen is exposed through Video Agent and + # Avatar Shots endpoints, NOT via the workflow provider parameter. For 2.0 + # access today, use `seedance_video` (fal.ai) or `seedance_replicate`. + "seedance_lite": {"name": "Seedance Lite (1.x)", "quality": "medium", "speed": "fast"}, + "seedance_pro": {"name": "Seedance Pro (1.x)", "quality": "high", "speed": "medium"}, "ltx_distilled": {"name": "LTX Distilled", "quality": "low", "speed": "fastest"}, } diff --git a/tools/video/higgsfield_video.py b/tools/video/higgsfield_video.py index aaf83a2c..95024d3b 100644 --- a/tools/video/higgsfield_video.py +++ b/tools/video/higgsfield_video.py @@ -42,7 +42,7 @@ class HiggsFieldVideo(BaseTool): " Get them at https://cloud.higgsfield.ai/api-keys\n" " Alternatively, set HIGGSFIELD_KEY as a combined key:secret value." ) - agent_skills = ["ai-video-gen"] + agent_skills = ["seedance-2-0", "ai-video-gen"] capabilities = ["text_to_video", "image_to_video"] supports = { @@ -50,14 +50,23 @@ class HiggsFieldVideo(BaseTool): "image_to_video": True, "character_consistency": True, "multi_model_routing": True, + "native_audio": True, + "cinematic_quality": True, + "camera_direction": True, + "lip_sync": True, + "multi_shot": True, } best_for = [ - "character-consistent video generation (Soul ID)", - "multi-model access through a single API", - "photorealistic and fashion-aware content", + "preferred premium video gen on Higgsfield (Seedance 2.0 is the default model)", + "cinematic trailers, teasers, and high-fidelity clips with native synchronized audio", + "character-consistent video generation (Soul ID + Seedance 2.0 identity consistency)", + "director-level camera control and multi-shot editing in a single generation", + "lip-sync from quoted dialogue in prompts", + "multi-model access through a single API (Seedance 2.0, Kling, Veo, Sora, WAN)", ] not_good_for = ["offline generation", "fine-grained model control", "budget projects without subscription"] - fallback_tools = ["kling_video", "veo_video", "minimax_video"] + fallback_tools = ["seedance_video", "seedance_replicate", "kling_video", "veo_video", "minimax_video"] + quality_score = 0.9 input_schema = { "type": "object", @@ -72,14 +81,16 @@ class HiggsFieldVideo(BaseTool): "model": { "type": "string", "enum": [ + "seedance_2.0", + "seedance_2.0_fast", "kling_3.0", "veo_3.1", "sora_2", "wan_2.5", "soul_cinema", ], - "default": "kling_3.0", - "description": "Underlying model to use for generation", + "default": "seedance_2.0", + "description": "Underlying model. Defaults to Seedance 2.0 (preferred premium) — see .agents/skills/seedance-2-0/", }, "duration": { "type": "string", @@ -123,10 +134,13 @@ class HiggsFieldVideo(BaseTool): return ToolStatus.UNAVAILABLE def estimate_cost(self, inputs: dict[str, Any]) -> float: - model = inputs.get("model", "kling_3.0") + model = inputs.get("model", "seedance_2.0") duration = int(inputs.get("duration", "5")) - # Approximate per-clip costs based on Higgsfield credit pricing + # Approximate per-clip costs based on Higgsfield credit pricing. + # Seedance 2.0 on Higgsfield runs ~50-80 credits per 5s clip ≈ $0.50-$1.20. base_costs = { + "seedance_2.0": 0.80, + "seedance_2.0_fast": 0.50, "kling_3.0": 0.10, "wan_2.5": 0.10, "veo_3.1": 0.50, @@ -137,9 +151,11 @@ class HiggsFieldVideo(BaseTool): return base * (duration / 5) def estimate_runtime(self, inputs: dict[str, Any]) -> float: - model = inputs.get("model", "kling_3.0") - if model in ("veo_3.1", "sora_2"): + model = inputs.get("model", "seedance_2.0") + if model in ("veo_3.1", "sora_2", "seedance_2.0"): return 120.0 + if model == "seedance_2.0_fast": + return 60.0 return 60.0 def execute(self, inputs: dict[str, Any]) -> ToolResult: diff --git a/tools/video/runway_video.py b/tools/video/runway_video.py index 6de0eef3..f1d31fad 100644 --- a/tools/video/runway_video.py +++ b/tools/video/runway_video.py @@ -34,12 +34,17 @@ _COST_PER_SECOND = { "gen3a_turbo": 0.05, "gen4_turbo": 0.05, "gen4_aleph": 0.15, + # Third-party Seedance 2.0 inside Runway (Enterprise/Unlimited, non-US). + "seedance_2.0": 0.30, + "seedance_2.0_fast": 0.24, } _RUNTIME_SECONDS = { "gen3a_turbo": 25.0, "gen4_turbo": 30.0, "gen4_aleph": 60.0, + "seedance_2.0": 120.0, + "seedance_2.0_fast": 60.0, } @@ -59,21 +64,29 @@ class RunwayVideo(BaseTool): "Set RUNWAY_API_KEY to your Runway API secret.\n" " Get one at https://dev.runwayml.com/" ) - agent_skills = ["ai-video-gen"] + agent_skills = ["seedance-2-0", "ai-video-gen"] capabilities = ["text_to_video", "image_to_video"] supports = { "text_to_video": True, "image_to_video": True, "professional_control": True, + "native_audio": True, + "cinematic_quality": True, + "camera_direction": True, + "lip_sync": True, + "multi_shot": True, } best_for = [ - "highest overall video quality (#1 Elo rating)", + "preferred premium video gen on Runway when Seedance 2.0 model is selected", + "cinematic trailers, teasers, and high-fidelity clips with native synchronized audio (Seedance 2.0 path)", + "director-level camera control and multi-shot editing (Seedance 2.0) or Runway Gen-4 professional control", + "lip-sync from quoted dialogue in prompts (Seedance 2.0)", "professional video production", - "precise control over generation", ] not_good_for = ["budget projects", "offline generation", "very long clips"] - fallback_tools = ["kling_video", "veo_video", "minimax_video", "wan_video"] + fallback_tools = ["seedance_video", "seedance_replicate", "kling_video", "veo_video", "minimax_video", "wan_video"] + quality_score = 0.9 input_schema = { "type": "object", @@ -87,9 +100,16 @@ class RunwayVideo(BaseTool): }, "model": { "type": "string", - "enum": ["gen4_turbo", "gen4_aleph", "gen3a_turbo"], - "default": "gen4_turbo", - "description": "gen4_aleph is highest fidelity, gen4_turbo is balanced, gen3a_turbo is cheapest", + "enum": ["seedance_2.0", "seedance_2.0_fast", "gen4_turbo", "gen4_aleph", "gen3a_turbo"], + "default": "seedance_2.0", + "description": ( + "seedance_2.0 = preferred premium default (single-pass synced audio, multi-shot, lip-sync — " + "Runway Unlimited/Enterprise plan, non-US only). " + "seedance_2.0_fast = lower-cost Seedance variant. " + "gen4_aleph = Runway's highest-fidelity native model. " + "gen4_turbo = balanced Runway native. " + "gen3a_turbo = cheapest Runway native." + ), }, "duration": { "type": "integer", diff --git a/tools/video/seedance_replicate.py b/tools/video/seedance_replicate.py new file mode 100644 index 00000000..aafd84b9 --- /dev/null +++ b/tools/video/seedance_replicate.py @@ -0,0 +1,253 @@ +"""Seedance 2.0 (ByteDance) video generation via Replicate. + +Replicate hosts ByteDance's published Seedance 2.0 models: + - bytedance/seedance-2.0 (standard) + - bytedance/seedance-2.0-fast (fast tier) + +Same model family as the fal.ai path (tools/video/seedance_video.py) — +if you have both FAL_KEY and REPLICATE_API_TOKEN the scoring engine +deduplicates by provider=seedance and picks whichever registers first. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class SeedanceReplicate(BaseTool): + name = "seedance_replicate" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "video_generation" + provider = "seedance" + stability = ToolStability.BETA + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = ( + "Set REPLICATE_API_TOKEN to your Replicate API token.\n" + " Get one at https://replicate.com/account/api-tokens" + ) + agent_skills = ["seedance-2-0", "ai-video-gen"] + + capabilities = ["text_to_video", "image_to_video"] + supports = { + "text_to_video": True, + "image_to_video": True, + "reference_image": True, + "native_audio": True, + "cinematic_quality": True, + "camera_direction": True, + "lip_sync": True, + "multi_shot": True, + "aspect_ratio": True, + "seed": True, + } + best_for = [ + "preferred premium video gen when REPLICATE_API_TOKEN is available", + "cinematic trailers, teasers, and high-fidelity clips with native synchronized audio", + "director-level camera control and multi-shot editing in a single generation", + "lip-sync from quoted dialogue in prompts", + "consistent character identity across shots", + ] + not_good_for = ["offline generation", "budget-constrained projects"] + fallback_tools = ["seedance_video", "veo_video", "kling_video", "minimax_video"] + quality_score = 0.95 + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string"}, + "operation": { + "type": "string", + "enum": ["text_to_video", "image_to_video"], + "default": "text_to_video", + }, + "model_variant": { + "type": "string", + "enum": ["standard", "fast"], + "default": "standard", + "description": "standard = bytedance/seedance-2.0, fast = bytedance/seedance-2.0-fast", + }, + "duration": { + "type": "string", + "enum": ["auto", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"], + "default": "5", + }, + "aspect_ratio": { + "type": "string", + "enum": ["auto", "21:9", "16:9", "4:3", "1:1", "3:4", "9:16"], + "default": "16:9", + }, + "resolution": { + "type": "string", + "enum": ["480p", "720p"], + "default": "720p", + }, + "generate_audio": { + "type": "boolean", + "default": True, + }, + "image_url": { + "type": "string", + "description": "Start frame image URL for image_to_video", + }, + "seed": {"type": "integer"}, + "output_path": {"type": "string"}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True + ) + retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) + idempotency_key_fields = ["prompt", "model_variant", "operation", "duration", "seed"] + side_effects = ["writes video file to output_path", "calls Replicate API"] + user_visible_verification = [ + "Watch generated clip for motion coherence, audio sync, and visual quality" + ] + + def _get_api_token(self) -> str | None: + return os.environ.get("REPLICATE_API_TOKEN") + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if self._get_api_token() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + variant = inputs.get("model_variant", "standard") + duration = inputs.get("duration", "5") + secs = 5 if duration == "auto" else int(duration) + # Replicate bills per-second at roughly the same rate as fal.ai for this model family. + rate = 0.24 if variant == "fast" else 0.30 + return round(rate * secs, 2) + + def estimate_runtime(self, inputs: dict[str, Any]) -> float: + return 60.0 if inputs.get("model_variant") == "fast" else 120.0 + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + token = self._get_api_token() + if not token: + return ToolResult( + success=False, + error="REPLICATE_API_TOKEN not set. " + self.install_instructions, + ) + + import requests + + start = time.time() + variant = inputs.get("model_variant", "standard") + model_slug = ( + "bytedance/seedance-2.0-fast" if variant == "fast" else "bytedance/seedance-2.0" + ) + + payload_input: dict[str, Any] = {"prompt": inputs["prompt"]} + if inputs.get("duration") and inputs["duration"] != "auto": + payload_input["duration"] = int(inputs["duration"]) + if inputs.get("aspect_ratio") and inputs["aspect_ratio"] != "auto": + payload_input["aspect_ratio"] = inputs["aspect_ratio"] + if inputs.get("resolution"): + payload_input["resolution"] = inputs["resolution"] + if "generate_audio" in inputs: + payload_input["generate_audio"] = inputs["generate_audio"] + if inputs.get("seed") is not None: + payload_input["seed"] = inputs["seed"] + if inputs.get("operation") == "image_to_video" and inputs.get("image_url"): + payload_input["image"] = inputs["image_url"] + + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Prefer": "wait=60", + } + + try: + submit = requests.post( + f"https://api.replicate.com/v1/models/{model_slug}/predictions", + headers=headers, + json={"input": payload_input}, + timeout=90, + ) + submit.raise_for_status() + pred = submit.json() + + # Poll until completed (Replicate may return the result synchronously + # when Prefer: wait is honored, but fall back to polling). + while pred.get("status") in ("starting", "processing"): + time.sleep(3) + get_url = pred.get("urls", {}).get("get") + if not get_url: + return ToolResult(success=False, error="Replicate response missing poll URL") + poll = requests.get(get_url, headers=headers, timeout=30) + poll.raise_for_status() + pred = poll.json() + + status = pred.get("status") + if status != "succeeded": + return ToolResult( + success=False, + error=f"Replicate Seedance 2.0 generation {status}: {pred.get('error')}", + ) + + output = pred.get("output") + # Replicate returns either a string URL or a list. + video_url = output[0] if isinstance(output, list) else output + if not isinstance(video_url, str): + return ToolResult(success=False, error=f"Unexpected output shape from Replicate: {output!r}") + + video_response = requests.get(video_url, timeout=180) + video_response.raise_for_status() + + output_path = Path(inputs.get("output_path", "seedance_replicate_output.mp4")) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(video_response.content) + + except Exception as e: + return ToolResult( + success=False, + error=f"Replicate Seedance 2.0 generation failed: {e}", + ) + + from tools.video._shared import probe_output + + probed = probe_output(output_path) + return ToolResult( + success=True, + data={ + "provider": "seedance", + "gateway": "replicate", + "model": model_slug, + "prompt": inputs["prompt"], + "variant": variant, + "aspect_ratio": inputs.get("aspect_ratio", "16:9"), + "resolution": inputs.get("resolution", "720p"), + "generate_audio": inputs.get("generate_audio", True), + "seed": pred.get("input", {}).get("seed"), + "output": str(output_path), + "output_path": str(output_path), + "format": "mp4", + **probed, + }, + artifacts=[str(output_path)], + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + model=model_slug, + )