From 7ca04e66d8421b79a8e7591ced2b90beeb987788 Mon Sep 17 00:00:00 2001 From: calesthio Date: Sun, 5 Apr 2026 15:31:37 -0700 Subject: [PATCH] Add Grok media providers and improve selector routing --- .agents/skills/grok-media/SKILL.md | 131 +++++++++ .env.example | 1 + README.md | 15 +- docs/ARCHITECTURE.md | 9 +- docs/PROVIDERS.md | 41 ++- lib/scoring.py | 147 ++++++++++ skills/INDEX.md | 7 +- skills/creative/image-provider-usage.md | 6 + skills/creative/prompting/grok-prompting.md | 81 ++++++ skills/creative/video-gen-prompting.md | 2 + tools/audio/tts_selector.py | 44 ++- tools/graphics/grok_image.py | 296 ++++++++++++++++++++ tools/graphics/image_selector.py | 111 +++++++- tools/video/grok_video.py | 290 +++++++++++++++++++ tools/video/video_selector.py | 98 ++++++- 15 files changed, 1251 insertions(+), 28 deletions(-) create mode 100644 .agents/skills/grok-media/SKILL.md create mode 100644 skills/creative/prompting/grok-prompting.md create mode 100644 tools/graphics/grok_image.py create mode 100644 tools/video/grok_video.py diff --git a/.agents/skills/grok-media/SKILL.md b/.agents/skills/grok-media/SKILL.md new file mode 100644 index 00000000..fc56d4d1 --- /dev/null +++ b/.agents/skills/grok-media/SKILL.md @@ -0,0 +1,131 @@ +--- +name: grok-media +description: xAI Grok image and video generation guide covering authentication, endpoints, prompt structure, image editing, reference-image video, and async polling. +metadata: + author: OpenMontage + version: "1.0.0" + tags: xai, grok, image-generation, video-generation, media +--- + +# Grok Media + +Use this skill when working with xAI media models in OpenMontage. + +## Models + +- `grok-imagine-image` for image generation and image editing +- `grok-imagine-video` for text-to-video, image-to-video, and reference-image video + +## Authentication + +- Env var: `XAI_API_KEY` +- Base URL: `https://api.x.ai/v1` +- Header: `Authorization: Bearer $XAI_API_KEY` + +## Image API + +### Text-to-image + +- Endpoint: `POST /images/generations` +- Core fields: + - `model` + - `prompt` + - `n` + - `aspect_ratio` + - `resolution` + +### Image edit + +- Endpoint: `POST /images/edits` +- Use `image` for one source image +- Use `images` for multi-image compositing +- Each source image can be: + - a public HTTPS URL + - a base64 data URI + +### Image prompting + +- Grok responds well to direct natural language +- For edits, describe only the intended change and preserve everything else implicitly +- For multi-image merges, explicitly name how each source contributes +- Prefer one strong scene description over long style-stacking + +## Video API + +### Generation + +- Endpoint: `POST /videos/generations` +- Polling endpoint: `GET /videos/{request_id}` +- Success state: `status == "done"` +- Failure states to handle explicitly: `failed`, `expired` + +### Modes + +- Text-to-video: + - prompt-only generation +- Image-to-video: + - use `image: {"url": ...}` + - this anchors the starting frame +- Reference-to-video: + - use `reference_images: [{"url": ...}, ...]` + - this influences who/what appears in the video without locking the first frame + - prompts can reference inputs with placeholders like ``, `` + +### Video constraints + +- Grok video is best treated as short-form generation +- Current output resolutions are `480p` and `720p` +- Reference-image video supports multiple images and is useful for product placement, wardrobe transfer, and identity consistency +- Download outputs promptly; provider URLs may be temporary + +## Pricing + +- `grok-imagine-image`: `$0.02` per generated image +- `grok-imagine-image` edits/composites: add `$0.002` per input image +- `grok-imagine-video`: + - `480p`: `$0.05` per second + - `720p`: `$0.07` per second +- `grok-imagine-video` image-conditioned requests: add `$0.002` per input image + +## Grok-Specific Prompt Guidance + +### Images + +- Start with subject, action, setting +- Add one style anchor, not five +- For edits: + - describe the desired modification + - keep the rest of the image stable by omission, not by writing a giant preservation list + +### Video + +- Keep prompts scene-local: one shot, one main motion idea, one emotional beat +- For reference-conditioned video, explicitly map source images to roles: + - person from `` + - jacket from `` + - product from `` +- Camera and pacing language helps: + - slow push-in + - handheld follow + - locked-off medium shot + - high-energy whip pan transition + +## Good Fits + +- Image style transfer +- Image compositing from multiple sources +- Reference-conditioned short video +- Product-led motion clips +- Character-consistent scenes without hard first-frame lock + +## Weak Fits + +- Long-form clip generation +- Heavy reliance on deterministic seeds +- Overloaded prompts with multiple scene changes + +## Failure Handling + +- If generation submission succeeds but polling expires, surface it as a provider/runtime issue +- If a request fails, preserve the endpoint, mode, and prompt summary in the error +- Do not silently substitute a different provider after xAI was selected without user approval diff --git a/.env.example b/.env.example index 7bb82f98..93c321a3 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,7 @@ GOOGLE_API_KEY= # Google Imagen images, Google Cloud TTS (700+ voic # --- Voice --- ELEVENLABS_API_KEY= # TTS narration, music generation, sound effects OPENAI_API_KEY= # OpenAI TTS fallback and DALL-E image generation +XAI_API_KEY= # Grok image generation/editing and Grok video generation # Piper local voices do not require env vars; install `piper-tts` via pip # --- Music --- diff --git a/README.md b/README.md index 45ceecd9..834adf8e 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ SUNO_API_KEY=your-key # Full songs, instrumentals, any genre # Voice & images: ELEVENLABS_API_KEY=your-key # Premium TTS, AI music, sound effects OPENAI_API_KEY=your-key # OpenAI TTS, DALL-E 3 images +XAI_API_KEY=your-key # xAI Grok image edits/generation + Grok video generation GOOGLE_API_KEY=your-key # Google Imagen images, Google TTS (700+ voices) # More video providers: @@ -265,7 +266,7 @@ Most AI video tools give you a single clip from a prompt. OpenMontage gives you Edit your own talking-head footage. Generate a fully animated explainer from scratch. Cut a 2-hour podcast into a dozen social clips. Translate and dub your content into 10 languages. Build a cinematic brand teaser from stock footage and AI-generated scenes. **If a production team can make it, OpenMontage can orchestrate it.** - **11 production pipelines** — explainers, talking heads, screen demos, cinematic trailers, animations, podcasts, localization, and more -- **49 production tools** — spanning video generation, image creation, text-to-speech, music, audio mixing, subtitles, enhancement, and analysis +- **51 production tools** — spanning video generation, image creation, text-to-speech, music, audio mixing, subtitles, enhancement, and analysis - **400+ agent skills** — production skills, pipeline directors, creative techniques, quality checklists, and deep technology knowledge packs that teach the agent how to use every tool like an expert - **Reference-driven creation** — paste a video you like and the agent turns it into a grounded, differentiated production plan instead of forcing you to invent the perfect prompt from scratch - **Live web research built in** — before writing a single word of script, the agent runs 15-25+ web searches across YouTube, Reddit, news sites, and academic sources to ground your video in real, current data @@ -323,9 +324,9 @@ Final video output -- only if self-review passes ``` OpenMontage/ ├── tools/ # 48 Python tools (the agent's hands) -│ ├── video/ # 12 video gen providers + compose, stitch, trim +│ ├── video/ # 13 video gen tools + compose, stitch, trim │ ├── audio/ # 4 TTS providers + Suno/ElevenLabs music, mixing, enhancement -│ ├── graphics/ # 8 image gen providers + diagrams, code snippets, math +│ ├── graphics/ # 9 image/graphics generation tools + diagrams, code snippets, math │ ├── enhancement/ # Upscale, bg remove, face enhance, color grade │ ├── analysis/ # Transcription, scene detect, frame sampling │ ├── avatar/ # Talking head, lip sync @@ -369,6 +370,7 @@ Each tool declares which Layer 3 skills it relies on. The agent reads Layer 1 to | **Kling** | Cloud API | High quality, fast | | **Runway Gen-4** | Cloud API | Cinematic quality | | **Google Veo 3** | Cloud API | Long-form, cinematic. Via fal.ai or HeyGen. | +| **Grok Imagine Video** | Cloud API | Strong reference-image video and xAI-native short-form generation | | **MiniMax** | Cloud API | Cost-effective | | **HeyGen** | Cloud API | Multi-model gateway | | **WAN 2.1** | Local GPU | Free, 1.3B and 14B variants | @@ -381,12 +383,13 @@ Each tool declares which Layer 3 skills it relies on. The agent reads Layer 1 to
-Image Generation — 8 providers +Image Generation — 9 tools/providers | Provider | Type | Notes | |----------|------|-------| | **FLUX** | Cloud API | State-of-the-art quality | | **Google Imagen** | Cloud API | Imagen 4 — high-quality, multiple aspect ratios | +| **Grok Imagine Image** | Cloud API | Strong image edits, style transfer, and multi-image compositing | | **DALL-E 3** | Cloud API | OpenAI's image model | | **Recraft** | Cloud API | Design-focused generation | | **Local Diffusion** | Local GPU | Stable Diffusion, free | @@ -513,6 +516,10 @@ OpenMontage treats video production like real engineering — with quality gates Every tool selection (video generation, image generation, TTS, music) runs through a 7-dimension scoring engine: task fit (30%), output quality (20%), control features (15%), reliability (15%), cost efficiency (10%), latency (5%), continuity (5%). The winning provider and its score are logged in the decision trail with all alternatives considered. +Selectors normalize loose brief context before scoring. If the agent only knows something like "Pixar-style animated short with character consistency," the selector expands that into scorer-friendly intent and style signals instead of requiring a perfectly pre-shaped `task_context`. + +Selector outputs also surface the chosen provider's `agent_skills`, so the agent can immediately read the right Layer 3 provider skill before writing prompts. + ### Decision Audit Trail Every major creative and technical choice — provider selection, style/playbook choice, music track, voice selection, renderer family, any fallback or downgrade — is logged with alternatives considered, confidence scores, and reasoning. The cumulative decision log persists across all stages so you can trace exactly why the output looks the way it does. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7cc4bb75..0ac2b5df 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -143,8 +143,8 @@ Three selector tools abstract multi-provider capabilities: | Selector | Capability | Providers (priority order) | |----------|-----------|---------------------------| | `tts_selector` | Text-to-speech | ElevenLabs > Google TTS > OpenAI > Piper (offline) | -| `image_selector` | Image generation | FLUX > Google Imagen > DALL-E > Recraft > LocalDiffusion > Pexels/Pixabay (stock) | -| `video_selector` | Video generation | Kling > Runway > VEO > MiniMax > HeyGen > LTX (modal) > LTX (local) > CogVideo > Hunyuan > WAN > Pexels/Pixabay (stock) | +| `image_selector` | Image generation | FLUX > Grok > Google Imagen > DALL-E > Recraft > LocalDiffusion > Pexels/Pixabay (stock) | +| `video_selector` | Video generation | Grok > Kling > Runway > VEO > MiniMax > HeyGen > LTX (modal) > LTX (local) > CogVideo > Hunyuan > WAN > Pexels/Pixabay (stock) | Selectors route based on: user preference > availability > fallback order. They adapt input schemas between providers transparently. @@ -158,11 +158,11 @@ Selectors route based on: user preference > availability > fallback order. They **Enhancement (5):** upscale (Real-ESRGAN), bg_remove (rembg/U2Net), face_enhance, face_restore (CodeFormer/GFPGAN), color_grade (FFmpeg LUTs) -**Graphics (12):** flux_image, google_imagen, openai_image, recraft_image, local_diffusion, pexels_image, pixabay_image, image_selector, code_snippet, diagram_gen, math_animate (ManimCE), image_gen (deprecated) +**Graphics (13):** flux_image, grok_image, google_imagen, openai_image, recraft_image, local_diffusion, pexels_image, pixabay_image, image_selector, code_snippet, diagram_gen, math_animate (ManimCE), image_gen (deprecated) **Subtitle (1):** subtitle_gen -**Video (17):** heygen_video, veo_video, kling_video, runway_video, minimax_video, wan_video, hunyuan_video, cogvideo_video, ltx_video_local, ltx_video_modal, pexels_video, pixabay_video, video_selector, video_compose (FFmpeg), video_stitch, video_trimmer +**Video (17):** grok_video, heygen_video, veo_video, kling_video, runway_video, minimax_video, wan_video, hunyuan_video, cogvideo_video, ltx_video_local, ltx_video_modal, pexels_video, pixabay_video, video_selector, video_compose (FFmpeg), video_stitch, video_trimmer --- @@ -378,6 +378,7 @@ All config is validated via Pydantic models in `lib/config_model.py`. |----------|---------|---------| | `ELEVENLABS_API_KEY` | elevenlabs_tts, music_gen | TTS, music, sound effects | | `OPENAI_API_KEY` | openai_tts, openai_image | TTS fallback, DALL-E 3 | +| `XAI_API_KEY` | grok_image, grok_video | Grok image editing/generation, Grok video generation | | `FAL_KEY` | flux_image, kling_video, veo_video, minimax_video, recraft_image | fal.ai hosted models (FLUX, Veo, Kling, MiniMax, Recraft) | | `HEYGEN_API_KEY` | heygen_video | Multi-provider video generation | | `PEXELS_API_KEY` | pexels_image, pexels_video | Stock media | diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 4932729e..3e61988b 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -38,6 +38,7 @@ GOOGLE_API_KEY= # Google TTS + Google Imagen # VOICE + MUSIC ELEVENLABS_API_KEY= # TTS, music, sound effects (10K chars/month free) OPENAI_API_KEY= # OpenAI TTS + DALL-E 3 images +XAI_API_KEY= # xAI Grok image generation/editing + Grok video generation # MULTI-MODEL GATEWAY (one key, 6+ tools) FAL_KEY= # FLUX, Recraft, Kling, Veo, MiniMax video @@ -56,6 +57,41 @@ VIDEO_GEN_LOCAL_MODEL= # wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, ## Cloud Providers +### xAI — Grok Image + Video + +> **Best if you want one provider for image edits and reference-conditioned short video.** Grok covers both image generation/editing and video generation under one key. + +**Tools unlocked:** `grok_image`, `grok_video` +**Env var:** `XAI_API_KEY` + +#### Setup + +1. Create an xAI developer account +2. Generate an API key in the xAI developer console +3. Add to `.env`: `XAI_API_KEY=xai-...` + +#### What it's best for + +- Image editing and style transfer +- Multi-image composites into one generated frame +- Short reference-image videos where a person, garment, or product must carry into motion + +#### Pricing + +Current xAI docs pricing for the Grok media models: + +| Model | Price | +|------|-------| +| `grok-imagine-image` | $0.02 per generated image | +| `grok-imagine-image` input images (edits/composites) | $0.002 per input image | +| `grok-imagine-video` at 480p | $0.05/sec | +| `grok-imagine-video` at 720p | $0.07/sec | +| `grok-imagine-video` input images | $0.002 per input image | + +OpenMontage now uses those published rates in the Grok tool estimators. + +--- + ### fal.ai — Multi-Model Gateway > **Best bang for buck.** One API key unlocks 6 tools across image and video generation. @@ -565,6 +601,7 @@ These tools require only FFmpeg or Python packages — no GPU, no API key. | **ElevenLabs** | `ELEVENLABS_API_KEY` | `elevenlabs_tts`, `music_gen` | Free tier + paid | | **fal.ai** | `FAL_KEY` | `flux_image`, `recraft_image`, `kling_video`, `veo_video`, `minimax_video` | Pay-as-you-go | | **OpenAI** | `OPENAI_API_KEY` | `openai_tts`, `openai_image` | Paid only | +| **xAI** | `XAI_API_KEY` | `grok_image`, `grok_video` | Paid only | | **Runway** | `RUNWAY_API_KEY` | `runway_video` | Free trial + paid | | **HeyGen** | `HEYGEN_API_KEY` | `heygen_video` | Pay-as-you-go | | **Suno** | `SUNO_API_KEY` | `suno_music` | Pay-as-you-go | @@ -580,8 +617,8 @@ How many providers cover each capability: | Capability | Cloud Providers | Local Providers | Free Options | |-----------|----------------|-----------------|--------------| -| **Image Generation** | FLUX, Google Imagen, DALL-E 3, Recraft | Local Diffusion | Pexels, Pixabay (stock) | -| **Video Generation** | Kling, Runway, Veo, MiniMax, HeyGen | WAN, Hunyuan, CogVideo, LTX | Pexels, Pixabay (stock) | +| **Image Generation** | FLUX, Grok, Google Imagen, DALL-E 3, Recraft | Local Diffusion | Pexels, Pixabay (stock) | +| **Video Generation** | Grok, Kling, Runway, Veo, MiniMax, HeyGen | WAN, Hunyuan, CogVideo, LTX | Pexels, Pixabay (stock) | | **Text-to-Speech** | ElevenLabs, Google TTS, OpenAI | Piper | Piper, Google free tier, ElevenLabs free tier | | **Music Generation** | ElevenLabs, Suno | — | ElevenLabs free tier | | **Post-Production** | — | FFmpeg (compose, stitch, trim, mix, enhance, grade) | All free | diff --git a/lib/scoring.py b/lib/scoring.py index 7bf2458a..ed586890 100644 --- a/lib/scoring.py +++ b/lib/scoring.py @@ -10,6 +10,7 @@ Scores are normalized 0-1. Higher is better. from __future__ import annotations from dataclasses import dataclass, asdict, field +import re from typing import Any @@ -129,6 +130,7 @@ _SYNONYM_CLUSTERS: list[set[str]] = [ {"corporate", "business", "professional", "enterprise"}, {"social", "tiktok", "instagram", "reels", "shorts", "viral"}, {"animation", "animated", "motion-graphics", "motion", "kinetic"}, + {"pixar", "animation", "animated", "stylized", "storybook", "character"}, {"realistic", "photorealistic", "lifelike", "natural"}, {"stock", "footage", "b-roll", "library"}, {"avatar", "presenter", "talking-head", "spokesperson"}, @@ -136,6 +138,52 @@ _SYNONYM_CLUSTERS: list[set[str]] = [ {"music", "soundtrack", "background-music", "score", "ambient"}, ] +_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9+._-]*") +_GENERATED_VISUAL_TERMS = { + "animated", + "animation", + "anime", + "cartoon", + "character", + "cinematic", + "concept", + "fantasy", + "ghibli", + "illustration", + "pixar", + "render", + "scifi", + "short", + "story", + "stylized", + "surreal", +} +_REFERENCE_TERMS = { + "character", + "consistency", + "identity", + "preserve", + "product", + "reference", + "subject", + "wardrobe", +} +_IMAGE_EDIT_TERMS = { + "combine", + "composite", + "edit", + "merge", + "modify", + "repaint", + "replace", + "style-transfer", + "transfer", +} + + +def _tokenize_text(value: str) -> list[str]: + return _TOKEN_RE.findall((value or "").lower()) + def _expand_synonyms(words: set[str]) -> set[str]: """Expand a word set with synonyms from known clusters.""" expanded = set(words) @@ -235,6 +283,82 @@ def _compute_continuity( return 0.4 # Different provider = possible style break +def normalize_task_context( + task_context: dict[str, Any] | None, + *, + prompt: str = "", + capability: str = "", + operation: str = "", +) -> dict[str, Any]: + """Normalize loose task context into the scorer's expected shape.""" + context = dict(task_context or {}) + + needs = context.get("needs") or [] + if isinstance(needs, str): + needs = [needs] + + text_fragments: list[str] = [] + for key in ("intent", "style", "brief", "goal", "platform"): + value = context.get(key) + if isinstance(value, str) and value.strip(): + text_fragments.append(value.strip()) + text_fragments.extend(str(item).strip() for item in needs if str(item).strip()) + if prompt.strip(): + text_fragments.append(prompt.strip()) + + combined_text = " ".join(text_fragments).strip() + if not context.get("intent"): + context["intent"] = combined_text + + style_keywords = { + str(item).lower().strip() + for item in (context.get("style_keywords") or []) + if str(item).strip() + } + for source in [context.get("style"), context.get("platform"), *needs]: + if isinstance(source, str): + style_keywords.update(_tokenize_text(source)) + context["style_keywords"] = sorted(style_keywords) + + if not context.get("asset_type"): + asset_type_map = { + "video_generation": "video", + "image_generation": "image", + "tts": "voice", + "music_generation": "music", + } + if capability in asset_type_map: + context["asset_type"] = asset_type_map[capability] + + if "motion_required" not in context and capability == "video_generation": + context["motion_required"] = True + + if "budget_remaining_usd" not in context and context.get("budget_usd") is not None: + context["budget_remaining_usd"] = context["budget_usd"] + + text_tokens = set(_tokenize_text(combined_text)) + context["prefers_generated_visuals"] = bool(text_tokens & _GENERATED_VISUAL_TERMS) + context["wants_reference_conditioning"] = ( + operation == "reference_to_video" or bool(text_tokens & _REFERENCE_TERMS) + ) + context["wants_image_editing"] = ( + operation == "edit" or bool(text_tokens & _IMAGE_EDIT_TERMS) + ) + + return context + + +def _is_stock_like_provider(info: dict[str, Any]) -> bool: + provider = str(info.get("provider", "")).lower() + if provider in {"pexels", "pixabay"}: + return True + + words = set() + for desc in info.get("best_for", []): + words.update(_tokenize_text(str(desc))) + return bool(words & {"stock", "footage", "b-roll", "library"}) + + def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore: """Score a provider against a task context. @@ -248,6 +372,7 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore: - motion_required (bool): Whether motion is a hard requirement - asset_type (str): "image", "video", "audio", "music", "voice" """ + task_context = normalize_task_context(task_context) info = tool.get_info() status = str(tool.get_status()) @@ -330,6 +455,28 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore: if "video" not in cap: task_fit *= 0.2 # Heavy penalty + supports = info.get("supports", {}) + stock_like = _is_stock_like_provider(info) + asset_type = task_context.get("asset_type") + + if task_context.get("prefers_generated_visuals") and stock_like and asset_type in {"video", "image"}: + task_fit *= 0.55 + output_quality *= 0.85 + + if task_context.get("wants_reference_conditioning") and asset_type == "video": + if supports.get("reference_to_video") or supports.get("reference_image") or supports.get("multiple_reference_images"): + task_fit = min(1.0, task_fit + 0.18) + control = min(1.0, control + 0.12) + else: + task_fit *= 0.7 + + if task_context.get("wants_image_editing") and asset_type == "image": + if supports.get("image_edit") or supports.get("style_transfer") or supports.get("multiple_reference_images"): + task_fit = min(1.0, task_fit + 0.18) + control = min(1.0, control + 0.10) + else: + task_fit *= 0.7 + return ProviderScore( tool_name=info.get("name", "unknown"), provider=info.get("provider", "unknown"), diff --git a/skills/INDEX.md b/skills/INDEX.md index 1907a084..82987c60 100644 --- a/skills/INDEX.md +++ b/skills/INDEX.md @@ -92,6 +92,7 @@ Key capability families to look for in the output: | Data Visualization | `creative/data-visualization.md` | Chart type selection, animation, label placement | `d3-viz`, `remotion-best-practices` | | Video Stitching | `creative/video-stitching.md` | Multi-clip assembly, AI clip chaining, spatial composition | `ffmpeg`, `video_toolkit` | | Video Gen Prompting | `creative/video-gen-prompting.md` | Universal video generation prompt vocabulary | `ai-video-gen`, `ltx2`, `create-video` | +| ↳ Grok Prompting | `creative/prompting/grok-prompting.md` | Grok image/video prompting, edit flows, reference-image video | `grok-media` | | ↳ Sora Prompting | `creative/prompting/sora-prompting.md` | Sora 2 structured template, advanced fields | `ai-video-gen` | | ↳ VEO Prompting | `creative/prompting/veo-prompting.md` | VEO 3.1 14-component structure, art movements | `ai-video-gen` | | ↳ LTX Prompting | `creative/prompting/ltx-prompting.md` | LTX-2 6-element structure, audio prompting | `ltx2` | @@ -101,7 +102,7 @@ Key capability families to look for in the output: | Typography | `creative/typography.md` | Font selection, text sizing, safe zones, caption styling | â€" | | ManimCE Usage | `creative/manim-usage.md` | Scene composition, animation timing, color usage | `manimce-best-practices` | | Image Gen Usage | `creative/image-gen-usage.md` | Prompt consistency, hero reference, batch strategy | `flux-best-practices`, `bfl-api` | -| Image Provider Usage | `creative/image-provider-usage.md` | Provider selection (FLUX/OpenAI/Recraft/stock), cost-quality tradeoffs | `flux-best-practices`, `bfl-api` | +| Image Provider Usage | `creative/image-provider-usage.md` | Provider selection (FLUX/Grok/OpenAI/Recraft/stock), cost-quality tradeoffs | `flux-best-practices`, `bfl-api`, `grok-media` | | B-Roll Planning | `creative/broll-planning.md` | Stock vs. generated decision, query construction, footage evaluation | — | | Stock Sourcing Usage | `creative/stock-sourcing-usage.md` | Pexels/Pixabay usage, parameters, licensing, integration | — | | Scene Detect Usage | `creative/scene-detect-usage.md` | Threshold tuning, algorithm selection, content presets | â€" | @@ -298,7 +299,7 @@ Claude Code accesses them via symlinks in `.claude/skills/`. | **Video Composition** | `remotion-best-practices`, `remotion` | `remotion-dev/skills`, `digitalsamba/claude-code-video-toolkit` | | **Video Processing** | `ffmpeg`, `video_toolkit` | `digitalsamba/claude-code-video-toolkit` | | **TTS & Audio** | `text-to-speech`, `speech-to-text`, `music`, `sound-effects`, `elevenlabs`, `agents`, `setup-api-key` | `elevenlabs/skills`, `digitalsamba/claude-code-video-toolkit` | -| **Image Generation** | `flux-best-practices`, `bfl-api` | `black-forest-labs/skills` | +| **Image Generation** | `flux-best-practices`, `bfl-api`, `grok-media` | `black-forest-labs/skills`, local OpenMontage skill | | **Math Animation** | `manimce-best-practices`, `manimgl-best-practices`, `manim-composer` | `adithya-s-k/manim_skill` | | **3D Graphics** | `threejs-animation`, `threejs-fundamentals`, `threejs-geometry`, `threejs-interaction`, `threejs-lighting`, `threejs-loaders`, `threejs-materials`, `threejs-postprocessing`, `threejs-shaders`, `threejs-textures` | `cloudai-x/threejs-skills` | | **Diagrams** | `beautiful-mermaid`, `d3-viz` | `intellectronica/agent-skills`, `davila7/claude-code-templates` | @@ -306,5 +307,3 @@ Claude Code accesses them via symlinks in `.claude/skills/`. | **Design** | `tailwind-design-system`, `web-design-guidelines`, `vercel-react-best-practices`, `vercel-composition-patterns` | `wshobson/agents`, `vercel-labs/agent-skills` | | **AI Video (HeyGen)** | `heygen`, `avatar-video`, `create-video`, `faceswap`, `ai-video-gen`, `video-download`, `video-edit`, `video-translate`, `video-understand`, `visual-style` | `heygen-com/skills` | | **Infrastructure** | `acestep`, `ltx2`, `playwright-recording` | `digitalsamba/claude-code-video-toolkit` | - - diff --git a/skills/creative/image-provider-usage.md b/skills/creative/image-provider-usage.md index fadc1515..baed41f6 100644 --- a/skills/creative/image-provider-usage.md +++ b/skills/creative/image-provider-usage.md @@ -10,6 +10,7 @@ | Tool | Provider | Cost | Speed | Best For | |------|----------|------|-------|----------| | `flux_image` | FLUX 2 Pro via fal.ai | ~$0.03-0.05 | ~5-10s | Photorealism, general purpose, workhorse | +| `grok_image` | Grok Imagine Image (xAI) | $0.02/output + $0.002/input edit image | ~5-15s | Image edits, style transfer, multi-image compositing | | `openai_image` | GPT Image 1 (OpenAI) | ~$0.01-0.17 | ~5-15s | Complex instructions, text in images, multi-element | | `recraft_image` | Recraft V4 via fal.ai | ~$0.04-0.25 | ~5-10s | Logos, SVG vectors, brand assets, text rendering | | `local_diffusion` | Stable Diffusion (local) | Free | ~30s+ | Offline, privacy, free | @@ -35,6 +36,8 @@ | **Real-world photo** (city, nature, people) | `pexels_image` | Real photos > AI for realism | `pixabay_image` → `flux_image` | | **Technical diagram** | `diagram_gen` | Structured, editable | `flux_image` with diagram prompt | | **Abstract/conceptual illustration** | `flux_image` | AI excels at custom concepts | `openai_image` | +| **Style transfer / repaint of an existing image** | `grok_image` | Native edit flow, strong promptable transforms | `openai_image` | +| **Multi-image merge / composite** | `grok_image` | Can combine multiple source images into one scene | `openai_image` | | **Logo or brand asset** | `recraft_image` | SVG support, text accuracy | `openai_image` | | **Image with text/labels** | `openai_image` | Best text rendering (GPT Image 1) | `recraft_image` | | **Complex multi-element composition** | `openai_image` | Best instruction following | `flux_image` | @@ -46,6 +49,7 @@ ## Cost-Quality Tradeoff ``` + PRODUCTION PATH: Premium ├── Hero images: flux_image ($0.05/img) ├── Supporting visuals: flux_image ($0.03/img) @@ -69,6 +73,8 @@ PRODUCTION PATH: Offline └── Total: $0.00 (but slower, lower quality) ``` +Use `generation_mode="edit"` when the task starts from an existing image and should route only to edit-capable providers. + ## Using the Image Selector For most cases, use `image_selector` and let it route: diff --git a/skills/creative/prompting/grok-prompting.md b/skills/creative/prompting/grok-prompting.md new file mode 100644 index 00000000..a585b13c --- /dev/null +++ b/skills/creative/prompting/grok-prompting.md @@ -0,0 +1,81 @@ +# Grok Prompting + +Use this when the chosen provider is `grok_image` or `grok_video`. + +## When Grok Is The Right Pick + +- You need to edit an existing image instead of generating from scratch +- You need to merge multiple source images into one output +- You need a short video influenced by reference images without locking the first frame +- You want one provider for both image and video generation with similar prompt language + +## Grok Image + +### Best Prompt Shape + +``` +[subject] + [action or change] + [setting] + [one style anchor] + [lighting] +``` + +### Edit Prompts + +For image edits, describe the intended transformation directly: + +- "Render this as a pencil sketch with detailed shading." +- "Replace the plain t-shirt with a dark green bomber jacket." +- "Combine these two people into the same sunny park scene." + +Do not over-specify every unchanged detail unless preservation is critical. + +### Multi-Image Composites + +Tell Grok how to combine the inputs: + +- who comes from which source +- what should stay separate +- where the final scene takes place + +Example: + +``` +Place the person from image 1 and the person from image 2 on the same subway platform at dusk, +standing shoulder to shoulder, cinematic sodium-vapor lighting, realistic photography. +``` + +## Grok Video + +### Best Prompt Shape + +``` +[shot] + [camera movement] + [subject] + [main motion beat] + [environment] + [lighting] + [tone] +``` + +### Reference-Image Video + +Grok supports prompts that refer to source images with placeholders like ``. +Use that when you need identity, wardrobe, or product consistency. + +Example: + +``` +Medium full shot, slow push-in. The model from walks onto a clean white runway wearing +the jacket from . Soft studio lighting, premium fashion campaign, confident expression. +``` + +### Image-to-Video vs Reference-to-Video + +- Use image-to-video when the source image should act like the opening frame. +- Use reference-to-video when the source images should influence the content but not freeze the composition. + +## Common Mistakes + +- Treating Grok reference images like strict storyboards. They are influence inputs, not exact frame locks. +- Writing multiple scene changes into one clip request. +- Combining too many style labels with too little scene information. +- Using vague edit prompts like "make it better" instead of naming the change. + +## OpenMontage Guidance + +- For image edits or compositing, prefer `grok_image` over the selector's default workhorse tools. +- For reference-conditioned video, prefer `grok_video` when the brief depends on carrying people, clothing, or products from input images into motion. +- If the deliverable is pure cinematic motion without reference constraints, compare Grok against Runway, Veo, and Kling before locking the provider. diff --git a/skills/creative/video-gen-prompting.md b/skills/creative/video-gen-prompting.md index b9d85486..24c7f11a 100644 --- a/skills/creative/video-gen-prompting.md +++ b/skills/creative/video-gen-prompting.md @@ -14,6 +14,7 @@ For model-specific tips, see the linked guides below. |-------|-------|-------------| | **Sora 2 / Sora 2 Pro** | [OpenAI Sora 2 Cookbook](https://developers.openai.com/cookbook/examples/sora/sora2_prompting_guide) | Richest structured template. Advanced fields: lenses, filtration, grade, diegetic sound, wardrobe, finishing. | | **VEO 3.1 / VEO 3** | [Vertex AI Prompt Guide](https://cloud.google.com/vertex-ai/generative-ai/docs/video/video-gen-prompt-guide) | Best vocabulary reference tables. 14-component prompt structure. | +| **Grok Imagine Video** | `creative/prompting/grok-prompting.md` | Best when prompts need reference-image placeholders like `` and identity/product carryover. | | **LTX-2** | [LTX Prompting Guide](https://docs.ltx.video/api-documentation/prompting-guide) | 6-element structure. Audio/voice prompting. Strong "what to avoid" section. | | **HunyuanVideo 1.5** | [Tencent Prompt Handbook](https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/assets/HunyuanVideo_1_5_Prompt_Handbook_EN.md) | Formula: Subject + Motion + Scene + [Shot] + [Camera] + [Lighting] + [Style] + [Atmosphere]. | | **Runway Gen-4** | [Runway Prompting Guide](https://help.runwayml.com/hc/en-us/articles/39789879462419-Gen-4-Video-Prompting-Guide) | "Focus on motion, not appearance." One scene per clip. Simplicity wins. | @@ -175,6 +176,7 @@ Put dialogue in quotation marks: `Character says: "Hello world."` 3. **If a shot misfires** — strip back. Freeze camera, simplify action, try again. 4. **For consistency across clips** — repeat the same style/lighting/grade description. 5. **Use seed values** — when you find a good result, save the seed for variations. +6. **For Grok reference-image video** — assign each source image a clear role in the prompt using ``, ``, etc. ## Example: Generic Prompt Template diff --git a/tools/audio/tts_selector.py b/tools/audio/tts_selector.py index 16af1719..2d69af20 100644 --- a/tools/audio/tts_selector.py +++ b/tools/audio/tts_selector.py @@ -114,13 +114,13 @@ class TTSSelector(BaseTool): candidates = self._providers() if not candidates: return 0.0 - tool, _ = self._select_best_tool(inputs, candidates, inputs.get("task_context", {})) + tool, _ = self._select_best_tool(inputs, candidates, self._prepare_task_context(inputs)) return tool.estimate_cost(inputs) if tool else 0.0 def execute(self, inputs: dict[str, Any]) -> ToolResult: from lib.scoring import rank_providers - task_context = inputs.get("task_context", {}) + task_context = self._prepare_task_context(inputs) candidates = self._providers() # Rank mode — return scored provider rankings without generating @@ -129,8 +129,9 @@ class TTSSelector(BaseTool): return ToolResult( success=True, data={ - "rankings": [r.to_dict() for r in rankings], + "rankings": self._serialize_rankings(candidates, rankings), "explanation": "\n".join(r.explain() for r in rankings[:5]), + "normalized_task_context": task_context, }, ) @@ -142,9 +143,11 @@ class TTSSelector(BaseTool): result = tool.execute(inputs) if result.success: result.data.setdefault("selected_tool", tool.name) + result.data["selected_provider"] = tool.provider result.data["selection_reason"] = score.explain() if score else f"Selected {tool.provider} ({tool.name})" if score: result.data["provider_score"] = score.to_dict() + result.data.update(self._tool_context_payload(tool)) result.data["alternatives_considered"] = [ t.name for t in candidates if t.name != tool.name and t.get_status().value == "available" @@ -182,3 +185,38 @@ class TTSSelector(BaseTool): return tool_by_provider[score_item.provider], score_item return None, None + + def _prepare_task_context(self, inputs: dict[str, Any]) -> dict[str, Any]: + from lib.scoring import normalize_task_context + + return normalize_task_context( + inputs.get("task_context", {}), + prompt=inputs.get("text", ""), + capability=self.capability, + operation=inputs.get("operation", "generate"), + ) + + @staticmethod + def _tool_context_payload(tool: BaseTool) -> dict[str, Any]: + info = tool.get_info() + return { + "selected_tool_agent_skills": info.get("agent_skills", []), + "required_agent_skills": info.get("agent_skills", []), + "selected_tool_usage_location": info.get("usage_location"), + "selected_tool_best_for": info.get("best_for", []), + } + + def _serialize_rankings(self, candidates: list[BaseTool], rankings: list[object]) -> list[dict[str, Any]]: + tool_by_name = {tool.name: tool for tool in candidates} + serialized: list[dict[str, Any]] = [] + for score in rankings: + item = score.to_dict() + tool = tool_by_name.get(score.tool_name) + if tool: + info = tool.get_info() + item["agent_skills"] = info.get("agent_skills", []) + item["usage_location"] = info.get("usage_location") + item["best_for"] = info.get("best_for", []) + item["status"] = str(tool.get_status()) + serialized.append(item) + return serialized diff --git a/tools/graphics/grok_image.py b/tools/graphics/grok_image.py new file mode 100644 index 00000000..4889dd71 --- /dev/null +++ b/tools/graphics/grok_image.py @@ -0,0 +1,296 @@ +"""xAI Grok image generation and editing.""" + +from __future__ import annotations + +import base64 +import mimetypes +import os +import time +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +def _file_to_data_uri(path_str: str) -> str: + path = Path(path_str) + if not path.exists(): + raise FileNotFoundError(f"Input file not found: {path}") + mime_type, _ = mimetypes.guess_type(path.name) + if not mime_type: + mime_type = "application/octet-stream" + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + +def _normalize_image_input(url_value: str | None, path_value: str | None) -> dict[str, str] | None: + if url_value: + return {"url": url_value, "type": "image_url"} + if path_value: + return {"url": _file_to_data_uri(path_value), "type": "image_url"} + return None + + +class GrokImage(BaseTool): + name = "grok_image" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "image_generation" + provider = "grok" + stability = ToolStability.BETA + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = ( + "Set XAI_API_KEY to your xAI API key.\n" + " Get one from the xAI developer console" + ) + agent_skills = ["grok-media"] + + capabilities = [ + "generate_image", + "edit_image", + "text_to_image", + "image_to_image", + "style_transfer", + ] + supports = { + "image_edit": True, + "multiple_outputs": True, + "aspect_ratio": True, + "resolution": True, + "reference_image": True, + "multiple_reference_images": True, + } + best_for = [ + "single-image edits and style transfers", + "multi-image compositing into one generated frame", + "general-purpose image generation with aspect ratio control", + ] + not_good_for = ["offline generation", "strict seeded reproducibility"] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string"}, + "generation_mode": { + "type": "string", + "enum": ["generate", "edit"], + "default": "generate", + "description": "Use 'edit' when providing one or more source images.", + }, + "model": { + "type": "string", + "enum": ["grok-imagine-image"], + "default": "grok-imagine-image", + }, + "aspect_ratio": {"type": "string", "description": "Examples: 1:1, 3:2, 16:9, 9:16"}, + "resolution": { + "type": "string", + "enum": ["1k", "2k"], + "description": "xAI image output resolution tier", + }, + "n": { + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 1, + }, + "image_url": {"type": "string", "description": "Single source image URL for edit mode"}, + "image_path": {"type": "string", "description": "Single local source image path for edit mode"}, + "image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Multiple source image URLs for compositing edits", + }, + "image_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Multiple local source image paths for compositing edits", + }, + "output_path": {"type": "string"}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=100, network_required=True + ) + retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) + idempotency_key_fields = ["prompt", "generation_mode", "model", "aspect_ratio", "resolution", "n"] + side_effects = ["writes image file(s) to output_path", "calls xAI image API"] + user_visible_verification = ["Inspect generated image(s) for composition quality and edit fidelity"] + + def get_status(self) -> ToolStatus: + if os.environ.get("XAI_API_KEY"): + return ToolStatus.AVAILABLE + return ToolStatus.UNAVAILABLE + + @staticmethod + def _input_image_count(inputs: dict[str, Any]) -> int: + count = 0 + if inputs.get("image_url") or inputs.get("image_path"): + count += 1 + count += len(inputs.get("image_urls") or []) + count += len(inputs.get("image_paths") or []) + return count + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + output_count = int(inputs.get("n", 1)) + input_count = self._input_image_count(inputs) + # xAI currently publishes Grok Imagine Image at $0.02 per generated + # image plus $0.002 per input image for edits or composites. + return output_count * 0.02 + input_count * 0.002 + + def _build_payload(self, inputs: dict[str, Any]) -> tuple[str, dict[str, Any]]: + mode = inputs.get("generation_mode", "generate") + payload: dict[str, Any] = { + "model": inputs.get("model", "grok-imagine-image"), + "prompt": inputs["prompt"], + } + if inputs.get("aspect_ratio"): + payload["aspect_ratio"] = inputs["aspect_ratio"] + if inputs.get("resolution"): + payload["resolution"] = inputs["resolution"] + if inputs.get("n"): + payload["n"] = inputs["n"] + + primary_image = _normalize_image_input(inputs.get("image_url"), inputs.get("image_path")) + extra_images = [ + {"url": url, "type": "image_url"} + for url in (inputs.get("image_urls") or []) + ] + extra_images.extend( + {"url": _file_to_data_uri(path), "type": "image_url"} + for path in (inputs.get("image_paths") or []) + ) + + if primary_image or extra_images: + mode = "edit" + + if mode == "edit": + endpoint = "https://api.x.ai/v1/images/edits" + if primary_image and not extra_images: + payload["image"] = primary_image + else: + images = [] + if primary_image: + images.append(primary_image) + images.extend(extra_images) + if not images: + raise ValueError( + "Edit mode requires image_url/image_path or image_urls/image_paths" + ) + payload["images"] = images + else: + endpoint = "https://api.x.ai/v1/images/generations" + + return endpoint, payload + + @staticmethod + def _infer_extension(url: str) -> str: + suffix = Path(urlparse(url).path).suffix.lower() + if suffix in {".png", ".jpg", ".jpeg", ".webp"}: + return suffix + return ".png" + + @staticmethod + def _output_paths(output_path: str | None, count: int, extension: str) -> list[Path]: + if not output_path: + stem = "grok_image" + return [Path(f"{stem}_{idx + 1}{extension}") for idx in range(count)] + + path = Path(output_path) + suffix = path.suffix or extension + if count == 1: + return [path if path.suffix else path.with_suffix(suffix)] + + base = path.with_suffix("") if path.suffix else path + return [base.parent / f"{base.name}_{idx + 1}{suffix}" for idx in range(count)] + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + api_key = os.environ.get("XAI_API_KEY") + if not api_key: + return ToolResult( + success=False, + error="XAI_API_KEY not set. " + self.install_instructions, + ) + + import requests + + start = time.time() + try: + endpoint, payload = self._build_payload(inputs) + response = requests.post( + endpoint, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json=payload, + timeout=180, + ) + response.raise_for_status() + data = response.json() + + items = data.get("data", []) + if not items: + return ToolResult(success=False, error="xAI returned no image outputs") + + extension = ".png" + first_url = items[0].get("url") + if first_url: + extension = self._infer_extension(first_url) + + output_paths = self._output_paths(inputs.get("output_path"), len(items), extension) + artifacts: list[str] = [] + outputs: list[str] = [] + + for item, output_path in zip(items, output_paths): + output_path.parent.mkdir(parents=True, exist_ok=True) + if item.get("b64_json"): + output_path.write_bytes(base64.b64decode(item["b64_json"])) + else: + image_url = item.get("url") + if not image_url: + return ToolResult(success=False, error="xAI image output missing url") + download = requests.get(image_url, timeout=120) + download.raise_for_status() + output_path.write_bytes(download.content) + artifacts.append(str(output_path)) + outputs.append(str(output_path)) + + except Exception as e: + return ToolResult(success=False, error=f"Grok image generation failed: {e}") + + primary_output = outputs[0] + return ToolResult( + success=True, + data={ + "provider": "grok", + "model": payload["model"], + "prompt": inputs["prompt"], + "generation_mode": inputs.get("generation_mode", "generate"), + "output": primary_output, + "outputs": outputs, + "images_generated": len(outputs), + }, + artifacts=artifacts, + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + model=payload["model"], + ) diff --git a/tools/graphics/image_selector.py b/tools/graphics/image_selector.py index 80b164d6..d92a60e5 100644 --- a/tools/graphics/image_selector.py +++ b/tools/graphics/image_selector.py @@ -52,6 +52,33 @@ class ImageSelector(BaseTool): "width": {"type": "integer", "description": "Image width in pixels"}, "height": {"type": "integer", "description": "Image height in pixels"}, "seed": {"type": "integer", "description": "Random seed for reproducibility (generation providers only)"}, + "n": {"type": "integer", "description": "Number of image variations to request when supported."}, + "aspect_ratio": { + "type": "string", + "description": "Aspect ratio hint for providers that support ratio-based generation.", + }, + "resolution": { + "type": "string", + "description": "Resolution tier for providers that support named resolutions.", + }, + "generation_mode": { + "type": "string", + "enum": ["generate", "edit"], + "default": "generate", + "description": "Use 'edit' when providing one or more source images.", + }, + "image_url": {"type": "string", "description": "Single source image URL for edit-capable providers."}, + "image_path": {"type": "string", "description": "Single local source image path for edit-capable providers."}, + "image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Multiple source image URLs for compositing edits.", + }, + "image_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Multiple local source image paths for compositing edits.", + }, "preferred_provider": { "type": "string", "description": "Provider name or 'auto'. Valid values are discovered at runtime from the registry.", @@ -101,7 +128,7 @@ class ImageSelector(BaseTool): candidates = self._providers() if not candidates: return 0.0 - tool, _ = self._select_best_tool(inputs, candidates, inputs.get("task_context", {})) + tool, _ = self._select_best_tool(inputs, candidates, self._prepare_task_context(inputs)) return tool.estimate_cost(inputs) if tool else 0.0 def execute(self, inputs: dict[str, Any]) -> ToolResult: @@ -109,8 +136,8 @@ class ImageSelector(BaseTool): from lib.scoring import rank_providers logger = logging.getLogger(__name__) - task_context = inputs.get("task_context", {}) - candidates = self._providers() + task_context = self._prepare_task_context(inputs) + candidates = self._filter_candidates(inputs, self._providers()) # Rank mode — return scored provider rankings without generating if inputs.get("operation") == "rank": @@ -118,8 +145,9 @@ class ImageSelector(BaseTool): return ToolResult( success=True, data={ - "rankings": [r.to_dict() for r in rankings], + "rankings": self._serialize_rankings(candidates, rankings), "explanation": "\n".join(r.explain() for r in rankings[:5]), + "normalized_task_context": task_context, }, ) @@ -143,7 +171,20 @@ class ImageSelector(BaseTool): if hasattr(tool, 'input_schema'): props = tool.input_schema.get("properties", {}) stripped = [] - for passthrough_key in ("negative_prompt", "width", "height", "seed"): + for passthrough_key in ( + "negative_prompt", + "width", + "height", + "seed", + "n", + "aspect_ratio", + "resolution", + "generation_mode", + "image_url", + "image_path", + "image_urls", + "image_paths", + ): if passthrough_key in adapted and passthrough_key not in props: stripped.append(f"{passthrough_key}={adapted.pop(passthrough_key)}") if stripped: @@ -155,9 +196,11 @@ class ImageSelector(BaseTool): result = tool.execute(adapted) if result.success: result.data.setdefault("selected_tool", tool.name) + result.data["selected_provider"] = tool.provider result.data["selection_reason"] = score.explain() if score else f"Selected {tool.provider} ({tool.name})" if score: result.data["provider_score"] = score.to_dict() + result.data.update(self._tool_context_payload(tool)) result.data["alternatives_considered"] = [ t.name for t in candidates if t.name != tool.name and t.get_status().value == "available" @@ -177,6 +220,7 @@ class ImageSelector(BaseTool): allowed = set(inputs.get("allowed_providers") or []) if allowed: candidates = [tool for tool in candidates if tool.provider in allowed] + candidates = self._filter_candidates(inputs, candidates) rankings = rank_providers(candidates, task_context) @@ -195,3 +239,60 @@ class ImageSelector(BaseTool): return tool_by_provider[score_item.provider], score_item return None, None + + def _prepare_task_context(self, inputs: dict[str, Any]) -> dict[str, Any]: + from lib.scoring import normalize_task_context + + return normalize_task_context( + inputs.get("task_context", {}), + prompt=inputs.get("prompt", ""), + capability=self.capability, + operation=inputs.get("generation_mode", inputs.get("operation", "generate")), + ) + + @staticmethod + def _tool_context_payload(tool: BaseTool) -> dict[str, Any]: + info = tool.get_info() + return { + "selected_tool_agent_skills": info.get("agent_skills", []), + "required_agent_skills": info.get("agent_skills", []), + "selected_tool_usage_location": info.get("usage_location"), + "selected_tool_best_for": info.get("best_for", []), + } + + def _serialize_rankings(self, candidates: list[BaseTool], rankings: list[object]) -> list[dict[str, Any]]: + tool_by_name = {tool.name: tool for tool in candidates} + serialized: list[dict[str, Any]] = [] + for score in rankings: + item = score.to_dict() + tool = tool_by_name.get(score.tool_name) + if tool: + info = tool.get_info() + item["agent_skills"] = info.get("agent_skills", []) + item["usage_location"] = info.get("usage_location") + item["best_for"] = info.get("best_for", []) + item["supports"] = info.get("supports", {}) + item["status"] = str(tool.get_status()) + serialized.append(item) + return serialized + + def _filter_candidates(self, inputs: dict[str, Any], candidates: list[BaseTool]) -> list[BaseTool]: + wants_edit = ( + inputs.get("generation_mode") == "edit" + or inputs.get("image_url") + or inputs.get("image_path") + or inputs.get("image_urls") + or inputs.get("image_paths") + ) + if not wants_edit: + return candidates + + filtered: list[BaseTool] = [] + for tool in candidates: + props = getattr(tool, "input_schema", {}).get("properties", {}) + supports = getattr(tool, "supports", {}) + if supports.get("image_edit") or any( + key in props for key in ("image", "images", "image_url", "image_path", "image_urls", "image_paths") + ): + filtered.append(tool) + return filtered or candidates diff --git a/tools/video/grok_video.py b/tools/video/grok_video.py new file mode 100644 index 00000000..1c1e653e --- /dev/null +++ b/tools/video/grok_video.py @@ -0,0 +1,290 @@ +"""xAI Grok video generation.""" + +from __future__ import annotations + +import base64 +import mimetypes +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +def _file_to_data_uri(path_str: str) -> str: + path = Path(path_str) + if not path.exists(): + raise FileNotFoundError(f"Input file not found: {path}") + mime_type, _ = mimetypes.guess_type(path.name) + if not mime_type: + mime_type = "application/octet-stream" + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + +def _normalize_media_ref(url_value: str | None, path_value: str | None) -> dict[str, str] | None: + if url_value: + return {"url": url_value} + if path_value: + return {"url": _file_to_data_uri(path_value)} + return None + + +class GrokVideo(BaseTool): + name = "grok_video" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "video_generation" + provider = "grok" + stability = ToolStability.BETA + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = ( + "Set XAI_API_KEY to your xAI API key.\n" + " Get one from the xAI developer console" + ) + agent_skills = ["grok-media", "ai-video-gen"] + + capabilities = ["text_to_video", "image_to_video", "reference_to_video"] + supports = { + "text_to_video": True, + "image_to_video": True, + "reference_to_video": True, + "reference_image": True, + "multiple_reference_images": True, + } + best_for = [ + "reference-conditioned video generation", + "product placement or character-consistent motion clips", + "xAI-native image-guided and text-guided short videos", + ] + not_good_for = ["offline generation", "very long clips"] + fallback_tools = ["veo_video", "runway_video", "kling_video", "minimax_video"] + + input_schema = { + "type": "object", + "required": ["prompt"], + "properties": { + "prompt": {"type": "string"}, + "operation": { + "type": "string", + "enum": ["text_to_video", "image_to_video", "reference_to_video"], + "default": "text_to_video", + }, + "model": { + "type": "string", + "enum": ["grok-imagine-video"], + "default": "grok-imagine-video", + }, + "duration": { + "type": "integer", + "minimum": 2, + "maximum": 10, + "default": 5, + }, + "aspect_ratio": { + "type": "string", + "enum": ["16:9", "9:16", "1:1"], + "default": "16:9", + }, + "resolution": { + "type": "string", + "enum": ["480p", "720p"], + "default": "720p", + }, + "image_url": {"type": "string", "description": "Reference image URL for image_to_video"}, + "image_path": {"type": "string", "description": "Local reference image path for image_to_video"}, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Reference image URLs for reference_to_video", + }, + "reference_image_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Local reference image paths for reference_to_video", + }, + "output_path": {"type": "string"}, + "poll_interval_seconds": {"type": "integer", "minimum": 2, "default": 5}, + "timeout_seconds": {"type": "integer", "minimum": 30, "default": 900}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True + ) + retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) + idempotency_key_fields = ["prompt", "operation", "model", "duration", "aspect_ratio", "resolution"] + side_effects = ["writes video file to output_path", "calls xAI video API"] + user_visible_verification = ["Watch generated clip for motion quality and prompt fidelity"] + + def get_status(self) -> ToolStatus: + if os.environ.get("XAI_API_KEY"): + return ToolStatus.AVAILABLE + return ToolStatus.UNAVAILABLE + + @staticmethod + def _normalize_resolution(value: str | None) -> str: + if value == "540p": + return "480p" + return value or "720p" + + @staticmethod + def _input_image_count(inputs: dict[str, Any]) -> int: + count = 0 + if inputs.get("image_url") or inputs.get("image_path"): + count += 1 + count += len(inputs.get("reference_image_urls") or []) + count += len(inputs.get("reference_image_paths") or []) + return count + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + duration = int(inputs.get("duration", 5)) + resolution = self._normalize_resolution(inputs.get("resolution")) + base_per_second = 0.07 if resolution == "720p" else 0.05 + input_image_cost = self._input_image_count(inputs) * 0.002 + # xAI currently publishes Grok Imagine Video at $0.05/sec for 480p, + # $0.07/sec for 720p, plus $0.002 per input image. + return base_per_second * duration + input_image_cost + + def estimate_runtime(self, inputs: dict[str, Any]) -> float: + duration = int(inputs.get("duration", 5)) + return 90.0 + duration * 8.0 + + def _build_payload(self, inputs: dict[str, Any]) -> dict[str, Any]: + operation = inputs.get("operation", "text_to_video") + payload: dict[str, Any] = { + "model": inputs.get("model", "grok-imagine-video"), + "prompt": inputs["prompt"], + } + + if operation != "reference_to_video": + payload["duration"] = int(inputs.get("duration", 5)) + if inputs.get("aspect_ratio"): + payload["aspect_ratio"] = inputs["aspect_ratio"] + if inputs.get("resolution"): + payload["resolution"] = self._normalize_resolution(inputs["resolution"]) + + if operation == "image_to_video": + image = _normalize_media_ref(inputs.get("image_url"), inputs.get("image_path")) + if not image: + raise ValueError("image_to_video requires image_url or image_path") + payload["image"] = image + elif operation == "reference_to_video": + refs = [{"url": url} for url in (inputs.get("reference_image_urls") or [])] + refs.extend( + {"url": _file_to_data_uri(path)} + for path in (inputs.get("reference_image_paths") or []) + ) + if not refs: + raise ValueError( + "reference_to_video requires reference_image_urls or reference_image_paths" + ) + payload["reference_images"] = refs + payload["duration"] = int(inputs.get("duration", 5)) + if inputs.get("aspect_ratio"): + payload["aspect_ratio"] = inputs["aspect_ratio"] + if inputs.get("resolution"): + payload["resolution"] = self._normalize_resolution(inputs["resolution"]) + + return payload + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + api_key = os.environ.get("XAI_API_KEY") + if not api_key: + return ToolResult( + success=False, + error="XAI_API_KEY not set. " + self.install_instructions, + ) + + import requests + from tools.video._shared import probe_output + + start = time.time() + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + try: + payload = self._build_payload(inputs) + response = requests.post( + "https://api.x.ai/v1/videos/generations", + headers=headers, + json=payload, + timeout=60, + ) + response.raise_for_status() + request_id = response.json()["request_id"] + + timeout_seconds = int(inputs.get("timeout_seconds", 900)) + poll_interval = int(inputs.get("poll_interval_seconds", 5)) + deadline = time.time() + timeout_seconds + + result_data: dict[str, Any] | None = None + while time.time() < deadline: + result = requests.get( + f"https://api.x.ai/v1/videos/{request_id}", + headers={"Authorization": headers["Authorization"]}, + timeout=30, + ) + result.raise_for_status() + result_data = result.json() + status = result_data.get("status") + if status == "done": + break + if status in {"failed", "expired"}: + detail = result_data.get("error") or result_data.get("message") or status + return ToolResult(success=False, error=f"Grok video generation {status}: {detail}") + time.sleep(poll_interval) + + if not result_data or result_data.get("status") != "done": + return ToolResult(success=False, error="Grok video generation timed out") + + video_url = (result_data.get("video") or {}).get("url") + if not video_url: + return ToolResult(success=False, error="xAI video output missing url") + + download = requests.get(video_url, timeout=300) + download.raise_for_status() + output_path = Path(inputs.get("output_path", "grok_video_output.mp4")) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(download.content) + + except Exception as e: + return ToolResult(success=False, error=f"Grok video generation failed: {e}") + + probed = probe_output(output_path) + return ToolResult( + success=True, + data={ + "provider": "grok", + "model": payload["model"], + "prompt": inputs["prompt"], + "operation": inputs.get("operation", "text_to_video"), + "request_id": request_id, + "output": str(output_path), + "output_path": str(output_path), + "format": "mp4", + **probed, + }, + artifacts=[str(output_path)], + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - start, 2), + model=payload["model"], + ) diff --git a/tools/video/video_selector.py b/tools/video/video_selector.py index 9e18ed0a..bc864b8d 100644 --- a/tools/video/video_selector.py +++ b/tools/video/video_selector.py @@ -49,7 +49,11 @@ class VideoSelector(BaseTool): "default": "auto", }, "allowed_providers": {"type": "array", "items": {"type": "string"}}, - "operation": {"type": "string", "enum": ["text_to_video", "image_to_video", "rank"], "default": "text_to_video"}, + "operation": { + "type": "string", + "enum": ["text_to_video", "image_to_video", "reference_to_video", "rank"], + "default": "text_to_video", + }, "aspect_ratio": { "type": "string", "enum": ["16:9", "9:16", "1:1"], @@ -68,10 +72,24 @@ class VideoSelector(BaseTool): "type": "string", "description": "URL of a reference image for image_to_video.", }, + "reference_image_urls": { + "type": "array", + "items": {"type": "string"}, + "description": "Reference image URLs for providers that support reference-conditioned video.", + }, + "reference_image_paths": { + "type": "array", + "items": {"type": "string"}, + "description": "Local reference image paths for providers that support reference-conditioned video.", + }, "image_url": { "type": "string", "description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).", }, + "resolution": { + "type": "string", + "description": "Resolution hint for providers that support named output resolutions.", + }, "output_path": {"type": "string"}, }, } @@ -103,23 +121,23 @@ class VideoSelector(BaseTool): return ToolStatus.UNAVAILABLE def estimate_cost(self, inputs: dict[str, object]) -> float: - candidates = self._providers() + candidates = self._filter_candidates(inputs, self._providers()) if not candidates: return 0.0 - tool, _ = self._select_best_tool(inputs, candidates, inputs.get("task_context", {})) + tool, _ = self._select_best_tool(inputs, candidates, self._prepare_task_context(inputs)) return tool.estimate_cost(inputs) if tool else 0.0 def estimate_runtime(self, inputs: dict[str, object]) -> float: candidates = self._providers() if not candidates: return 0.0 - tool, _ = self._select_best_tool(inputs, candidates, inputs.get("task_context", {})) + tool, _ = self._select_best_tool(inputs, candidates, self._prepare_task_context(inputs)) return tool.estimate_runtime(inputs) if tool else 0.0 def execute(self, inputs: dict[str, object]) -> ToolResult: from lib.scoring import rank_providers - task_context = inputs.get("task_context", {}) + task_context = self._prepare_task_context(inputs) candidates = self._providers() # Rank mode — return scored provider rankings without generating @@ -128,8 +146,9 @@ class VideoSelector(BaseTool): return ToolResult( success=True, data={ - "rankings": [r.to_dict() for r in rankings], + "rankings": self._serialize_rankings(candidates, rankings), "explanation": "\n".join(r.explain() for r in rankings[:5]), + "normalized_task_context": task_context, }, ) @@ -159,9 +178,11 @@ class VideoSelector(BaseTool): result = tool.execute(adapted) if result.success: result.data.setdefault("selected_tool", tool.name) + result.data["selected_provider"] = tool.provider result.data["selection_reason"] = score.explain() if score else f"Selected {tool.provider} ({tool.name})" if score: result.data["provider_score"] = score.to_dict() + result.data.update(self._tool_context_payload(tool)) result.data["alternatives_considered"] = [ t.name for t in candidates if t.name != tool.name and t.get_status().value == "available" @@ -185,6 +206,7 @@ class VideoSelector(BaseTool): allowed = set(inputs.get("allowed_providers") or []) if allowed: candidates = [tool for tool in candidates if tool.provider in allowed] + candidates = self._filter_candidates(inputs, candidates) env_hint = os.environ.get("VIDEO_GEN_LOCAL_MODEL", "").lower() env_map = { @@ -219,3 +241,67 @@ class VideoSelector(BaseTool): return tool_by_provider[score.provider], score return None, None + + def _prepare_task_context(self, inputs: dict[str, object]) -> dict[str, object]: + from lib.scoring import normalize_task_context + + return normalize_task_context( + inputs.get("task_context", {}), + prompt=str(inputs.get("prompt", "")), + capability=self.capability, + operation=str(inputs.get("operation", "text_to_video")), + ) + + @staticmethod + def _tool_context_payload(tool: BaseTool) -> dict[str, object]: + info = tool.get_info() + return { + "selected_tool_agent_skills": info.get("agent_skills", []), + "required_agent_skills": info.get("agent_skills", []), + "selected_tool_usage_location": info.get("usage_location"), + "selected_tool_best_for": info.get("best_for", []), + } + + def _serialize_rankings(self, candidates: list[BaseTool], rankings: list[object]) -> list[dict[str, object]]: + tool_by_name = {tool.name: tool for tool in candidates} + serialized: list[dict[str, object]] = [] + for score in rankings: + item = score.to_dict() + tool = tool_by_name.get(score.tool_name) + if tool: + info = tool.get_info() + item["agent_skills"] = info.get("agent_skills", []) + item["usage_location"] = info.get("usage_location") + item["best_for"] = info.get("best_for", []) + item["supports"] = info.get("supports", {}) + item["status"] = str(tool.get_status()) + serialized.append(item) + return serialized + + def _filter_candidates( + self, + inputs: dict[str, object], + candidates: list[BaseTool], + ) -> list[BaseTool]: + operation = inputs.get("operation", "text_to_video") + if operation == "rank": + return candidates + + filtered: list[BaseTool] = [] + for tool in candidates: + supports = getattr(tool, "supports", {}) + props = getattr(tool, "input_schema", {}).get("properties", {}) + + if operation == "image_to_video": + if supports.get("image_to_video") or "image_url" in props or "reference_image_url" in props: + filtered.append(tool) + continue + + if operation == "reference_to_video": + if supports.get("reference_to_video") or "reference_image_urls" in props: + filtered.append(tool) + continue + + filtered.append(tool) + + return filtered or candidates