From 07276cfa939b9a37bf5e9787c3baa1d0242efbb6 Mon Sep 17 00:00:00 2001 From: calesthio Date: Sun, 29 Mar 2026 12:50:29 -0700 Subject: [PATCH] Fix fal.ai video tools and enforce pipeline-first production MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix Kling URL format: text_to_video → text-to-video (hyphens) - Fix Kling, MiniMax, Veo: switch from sync fal.run to queue API with polling - Add Rule Zero to AGENT_GUIDE: all production must go through pipelines - Make Layer 3 skill reading mandatory before calling generation tools - Add explicit do-nots: no ad-hoc scripts, no skipping director skills --- AGENT_GUIDE.md | 34 +++++++++++++++++++++++--- tools/video/kling_video.py | 46 ++++++++++++++++++++++++++++-------- tools/video/minimax_video.py | 42 +++++++++++++++++++++++++------- tools/video/veo_video.py | 42 +++++++++++++++++++++++++------- 4 files changed, 133 insertions(+), 31 deletions(-) diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index c703e320..30159173 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -4,6 +4,27 @@ Start here. This is the complete operating guide and agent contract for OpenMont For architecture, key files, and conventions see [`PROJECT_CONTEXT.md`](PROJECT_CONTEXT.md). +## Rule Zero — All Production Goes Through a Pipeline + +**Every video production request MUST go through the pipeline system. No exceptions.** + +When the user asks to make, create, produce, or generate any video content — a trailer, explainer, clip, animation, or any other video — the agent must: + +1. **Identify the pipeline.** Match the request to one of the pipelines in `pipeline_defs/`. If unclear, ask the user. +2. **Read the pipeline manifest.** `pipeline_defs/.yaml` — know the stages, tools, and quality gates. +3. **Run preflight.** Discover available tools via the registry. Present the capability menu. +4. **Execute stage by stage.** For EACH stage, read the stage director skill (`skills/pipelines//-director.md`) BEFORE doing any work in that stage. +5. **Read Layer 3 skills before calling tools.** Before using any tool with an `agent_skills` field, read the referenced skill in `.agents/skills/`. These contain provider-specific prompting guidance, parameter optimization, and quality techniques that dramatically improve output. + +**Do NOT:** +- Write ad-hoc Python scripts to call tools directly +- Skip the pipeline and go straight to API calls +- Generate assets without reading the stage director skill first +- Use a tool without checking its Layer 3 skill for prompting guidance +- Bypass preflight, checkpoints, or review + +The intelligence is in the skills, not in improvised code. An agent that reads the director skills and Layer 3 knowledge will produce significantly better output than one that calls tools directly with generic prompts. + ## What OpenMontage Is OpenMontage is an instruction-driven video production system. The AI agent IS the intelligence — it reads instructions (pipeline manifests + stage director skills + meta skills) and drives the pipeline using tools. @@ -415,9 +436,13 @@ OpenMontage has three instruction layers: Reading order: -1. registry / tool contract -2. relevant pipeline or creative skill -3. underlying vendor skill only if needed +1. registry / tool contract — discover what's available +2. relevant pipeline or creative skill — know HOW to use it in this context +3. underlying vendor skill — **mandatory before calling any generation tool** + +**Layer 3 is not optional.** Every generation tool (video, image, TTS, music) has an `agent_skills` field listing its Layer 3 skills. These skills contain provider-specific prompt engineering, parameter tuning, and quality techniques. Read them before writing prompts. The difference between a generic prompt and a skill-informed prompt is the difference between "usable" and "cinematic." + +Example: Before calling `kling_video`, read its `agent_skills` → `ai-video-gen` → get Kling-specific prompt structure, camera direction syntax, and quality keywords that the model responds to best. ## Quick Lookup @@ -432,6 +457,9 @@ Reading order: ## What Not To Do +- **Do not bypass the pipeline.** Never write ad-hoc scripts to call tools directly. All production goes through pipeline stages with director skills. See Rule Zero. +- **Do not call generation tools without reading their Layer 3 skill.** Check the tool's `agent_skills` field, read the referenced skill, then craft your prompts using that guidance. +- **Do not skip stage director skills.** Before executing any pipeline stage, read its director skill. The skill contains the quality bar, the workflow, and the review criteria. - Do not use deleted legacy names such as `tts_cloud`, `tts_engine`, or `video_gen`. - Do not hardcode provider names, API key names, or setup URLs. Read them from the registry's `install_instructions` and `dependencies` fields. - Do not begin asset generation before user approval on the production plan. diff --git a/tools/video/kling_video.py b/tools/video/kling_video.py index 1527769f..a386ac8c 100644 --- a/tools/video/kling_video.py +++ b/tools/video/kling_video.py @@ -129,7 +129,9 @@ class KlingVideo(BaseTool): start = time.time() operation = inputs.get("operation", "text_to_video") variant = inputs.get("model_variant", "v3/standard") - model_path = f"kling-video/{variant}/{operation}" + # fal.ai uses hyphens in endpoint paths (text-to-video, not text_to_video) + operation_path = operation.replace("_", "-") + model_path = f"kling-video/{variant}/{operation_path}" payload: dict[str, Any] = {"prompt": inputs["prompt"]} if inputs.get("duration"): @@ -139,18 +141,42 @@ class KlingVideo(BaseTool): if operation == "image_to_video" and inputs.get("image_url"): payload["image_url"] = inputs["image_url"] + headers = { + "Authorization": f"Key {api_key}", + "Content-Type": "application/json", + } + try: - response = requests.post( - f"https://fal.run/fal-ai/{model_path}", - headers={ - "Authorization": f"Key {api_key}", - "Content-Type": "application/json", - }, + # Submit to queue API (async) — sync endpoint times out for video gen + submit_resp = requests.post( + f"https://queue.fal.run/fal-ai/{model_path}", + headers=headers, json=payload, - timeout=300, + timeout=30, ) - response.raise_for_status() - data = response.json() + submit_resp.raise_for_status() + queue_data = submit_resp.json() + status_url = queue_data["status_url"] + response_url = queue_data["response_url"] + + # Poll until complete + while True: + time.sleep(5) + status_resp = requests.get(status_url, headers=headers, timeout=15) + status_resp.raise_for_status() + status = status_resp.json().get("status", "UNKNOWN") + if status == "COMPLETED": + break + if status in ("FAILED", "CANCELLED"): + return ToolResult( + success=False, + error=f"Kling video generation {status.lower()}", + ) + + # Fetch result + result_resp = requests.get(response_url, headers=headers, timeout=30) + result_resp.raise_for_status() + data = result_resp.json() video_url = data["video"]["url"] video_response = requests.get(video_url, timeout=120) diff --git a/tools/video/minimax_video.py b/tools/video/minimax_video.py index 80b314be..bdbba74d 100644 --- a/tools/video/minimax_video.py +++ b/tools/video/minimax_video.py @@ -137,18 +137,42 @@ class MiniMaxVideo(BaseTool): if operation == "image_to_video" and inputs.get("image_url"): payload["image_url"] = inputs["image_url"] + headers = { + "Authorization": f"Key {api_key}", + "Content-Type": "application/json", + } + try: - response = requests.post( - f"https://fal.run/fal-ai/{model_path}", - headers={ - "Authorization": f"Key {api_key}", - "Content-Type": "application/json", - }, + # Submit to queue API (async) — sync endpoint times out for video gen + submit_resp = requests.post( + f"https://queue.fal.run/fal-ai/{model_path}", + headers=headers, json=payload, - timeout=300, + timeout=30, ) - response.raise_for_status() - data = response.json() + submit_resp.raise_for_status() + queue_data = submit_resp.json() + status_url = queue_data["status_url"] + response_url = queue_data["response_url"] + + # Poll until complete + while True: + time.sleep(5) + status_resp = requests.get(status_url, headers=headers, timeout=15) + status_resp.raise_for_status() + status = status_resp.json().get("status", "UNKNOWN") + if status == "COMPLETED": + break + if status in ("FAILED", "CANCELLED"): + return ToolResult( + success=False, + error=f"MiniMax video generation {status.lower()}", + ) + + # Fetch result + result_resp = requests.get(response_url, headers=headers, timeout=30) + result_resp.raise_for_status() + data = result_resp.json() video_url = data["video"]["url"] video_response = requests.get(video_url, timeout=120) diff --git a/tools/video/veo_video.py b/tools/video/veo_video.py index f0f7acd2..8e37de9a 100644 --- a/tools/video/veo_video.py +++ b/tools/video/veo_video.py @@ -158,18 +158,42 @@ class VeoVideo(BaseTool): if operation == "image_to_video" and inputs.get("image_url"): payload["image_url"] = inputs["image_url"] + headers = { + "Authorization": f"Key {api_key}", + "Content-Type": "application/json", + } + try: - response = requests.post( - f"https://fal.run/fal-ai/{model_path}", - headers={ - "Authorization": f"Key {api_key}", - "Content-Type": "application/json", - }, + # Submit to queue API (async) — sync endpoint times out for video gen + submit_resp = requests.post( + f"https://queue.fal.run/fal-ai/{model_path}", + headers=headers, json=payload, - timeout=300, + timeout=30, ) - response.raise_for_status() - data = response.json() + submit_resp.raise_for_status() + queue_data = submit_resp.json() + status_url = queue_data["status_url"] + response_url = queue_data["response_url"] + + # Poll until complete + while True: + time.sleep(5) + status_resp = requests.get(status_url, headers=headers, timeout=15) + status_resp.raise_for_status() + status = status_resp.json().get("status", "UNKNOWN") + if status == "COMPLETED": + break + if status in ("FAILED", "CANCELLED"): + return ToolResult( + success=False, + error=f"Veo video generation {status.lower()}", + ) + + # Fetch result + result_resp = requests.get(response_url, headers=headers, timeout=30) + result_resp.raise_for_status() + data = result_resp.json() video_url = data["video"]["url"] video_response = requests.get(video_url, timeout=120)