From 9482eddeff1f11cc352ca1d2d49ce26981ee3023 Mon Sep 17 00:00:00 2001 From: calesthio Date: Mon, 3 Aug 2026 02:14:01 -0700 Subject: [PATCH] fix: recover bounded defects from PR backlog --- .env.example | 107 +++++--- AGENT_GUIDE.md | 7 +- backlot/server.py | 27 +- backlot/state.py | 2 + lib/checkpoint.py | 139 ++++++++-- lib/clip_embedder.py | 18 +- lib/pipeline_loader.py | 4 +- remotion-composer/package-lock.json | 21 +- remotion-composer/package.json | 1 + remotion-composer/src/CinematicRenderer.tsx | 26 +- remotion-composer/src/CollageBurst.tsx | 23 +- remotion-composer/src/Explainer.tsx | 89 ++++--- remotion-composer/src/LyricOverlay.tsx | 23 +- remotion-composer/src/TitledVideo.tsx | 29 +- remotion-composer/src/cinematic/types.ts | 1 + .../src/components/AnimeScene.tsx | 18 +- .../src/components/CaptionOverlay.tsx | 4 +- remotion-composer/src/components/EndTag.tsx | 4 +- .../src/components/HeroTitle.tsx | 4 +- .../src/components/ProductReveal.tsx | 4 +- .../src/components/ProviderChip.tsx | 2 +- .../src/components/ScreenshotScene.tsx | 21 +- remotion-composer/src/lib/resolveAsset.ts | 28 ++ schemas/artifacts/__init__.py | 2 +- schemas/artifacts/edit_decisions.schema.json | 4 + .../pipelines/pipeline_manifest.schema.json | 2 +- scripts/backlot_screenshot_stage.py | 9 + skills/INDEX.md | 4 +- skills/pipelines/cinematic/idea-director.md | 15 +- .../pipelines/cinematic/proposal-director.md | 8 +- .../documentary-montage/idea-director.md | 7 +- skills/pipelines/explainer/idea-director.md | 4 +- .../pipelines/explainer/proposal-director.md | 14 +- skills/pipelines/explainer/scene-director.md | 22 +- skills/pipelines/explainer/script-director.md | 4 +- styles/playbook_loader.py | 4 +- tests/backlot/test_gate_scenarios.py | 31 ++- tests/backlot/test_ui_bug_bash.py | 32 ++- .../test_agent_instruction_integrity.py | 31 +++ tests/contracts/test_backlot_contract.py | 26 ++ tests/contracts/test_env_example.py | 19 ++ tests/contracts/test_phase2_contracts.py | 8 +- tests/contracts/test_phase3_contracts.py | 21 +- .../test_pipeline_manifest_categories.py | 9 + ...test_remotion_video_transition_contract.py | 28 ++ tests/contracts/test_utf8_file_io.py | 30 +++ tests/lib/test_checkpoint_prerequisites.py | 152 +++++++++++ tests/lib/test_clip_embedder_compat.py | 26 ++ .../tools/test_audio_mixer_target_duration.py | 94 +++++++ tests/tools/test_bg_remove_api.py | 28 ++ .../tools/test_cinematic_remotion_adapter.py | 177 +++++++++++++ .../test_corpus_builder_total_failure.py | 59 +++++ tests/tools/test_google_vertex_backends.py | 97 +++++++ tests/tools/test_hyperframes_compose.py | 30 +++ tests/tools/test_mps_device.py | 8 +- tests/tools/test_remotion_audio_mux.py | 59 +++++ .../test_transcriber_device_selection.py | 83 ++++++ tools/analysis/transcriber.py | 110 +++++--- tools/audio/audio_mixer.py | 48 +++- tools/audio/google_music.py | 3 +- tools/enhancement/bg_remove.py | 2 +- tools/enhancement/upscale.py | 5 + tools/google_credentials.py | 19 +- tools/graphics/google_imagen.py | 3 +- tools/video/corpus_builder.py | 32 +++ tools/video/hyperframes_compose.py | 48 ++++ tools/video/veo_video.py | 21 +- tools/video/video_compose.py | 249 ++++++++++++++++-- 68 files changed, 1913 insertions(+), 376 deletions(-) create mode 100644 remotion-composer/src/lib/resolveAsset.ts create mode 100644 tests/contracts/test_agent_instruction_integrity.py create mode 100644 tests/contracts/test_env_example.py create mode 100644 tests/contracts/test_pipeline_manifest_categories.py create mode 100644 tests/contracts/test_remotion_video_transition_contract.py create mode 100644 tests/contracts/test_utf8_file_io.py create mode 100644 tests/lib/test_checkpoint_prerequisites.py create mode 100644 tests/lib/test_clip_embedder_compat.py create mode 100644 tests/tools/test_audio_mixer_target_duration.py create mode 100644 tests/tools/test_bg_remove_api.py create mode 100644 tests/tools/test_cinematic_remotion_adapter.py create mode 100644 tests/tools/test_corpus_builder_total_failure.py create mode 100644 tests/tools/test_google_vertex_backends.py create mode 100644 tests/tools/test_remotion_audio_mux.py create mode 100644 tests/tools/test_transcriber_device_selection.py diff --git a/.env.example b/.env.example index d60f8987..51d973e4 100644 --- a/.env.example +++ b/.env.example @@ -2,72 +2,103 @@ # Copy this to .env and fill in your keys # --- Image + video gateway --- -FAL_KEY= # FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images - # Get one at https://fal.ai/dashboard/keys -FAL_AI_API_KEY= # Alias for FAL_KEY (some SDKs/docs use this name); either one is read. +# FLUX images, Google Veo video, Kling video, MiniMax video, Recraft images. +# Get one at https://fal.ai/dashboard/keys +FAL_KEY= +# Alias for FAL_KEY (some SDKs/docs use this name); either one is read. +FAL_AI_API_KEY= # --- Replicate --- -REPLICATE_API_TOKEN= # Replicate-hosted video gen (seedance_replicate). Needed to make the - # Replicate-backed Seedance path selectable alongside the fal.ai one. - # Get one at https://replicate.com/account/api-tokens +# Replicate-hosted video gen (seedance_replicate). Needed to make the +# Replicate-backed Seedance path selectable alongside the fal.ai one. +# Get one at https://replicate.com/account/api-tokens +REPLICATE_API_TOKEN= # --- Higgsfield --- -HIGGSFIELD_API_KEY= # Higgsfield Cloud key (higgsfield_video). Pair with the secret below, -HIGGSFIELD_API_SECRET= # or use the combined HIGGSFIELD_KEY=":" form instead. +# Higgsfield Cloud key (higgsfield_video). Pair with the secret below, +# or use the combined HIGGSFIELD_KEY=":" form instead. +HIGGSFIELD_API_KEY= +HIGGSFIELD_API_SECRET= # HIGGSFIELD_KEY= # Combined key:secret — set this INSTEAD of the _KEY/_SECRET pair if you prefer. # --- Kling official direct API --- -KLING_API_KEY= # Official Kling API key; enables video, image, TTS, avatar, lip sync -KLING_API_BASE_URL= # Optional endpoint override; leave blank for default https://api-singapore.klingai.com - # Mainland China accounts can use https://api-beijing.klingai.com +# Official Kling API key; enables video, image, TTS, avatar, lip sync. +KLING_API_KEY= +# Optional endpoint override; leave blank for default https://api-singapore.klingai.com +# Mainland China accounts can use https://api-beijing.klingai.com +KLING_API_BASE_URL= # --- Google (one key unlocks image gen + TTS + video) --- -GOOGLE_API_KEY= # Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages), - # Gemini Omni video (generation + conversational editing, paid tier) - # Get one at https://aistudio.google.com/apikey +# Google Imagen images, Google Cloud TTS (700+ voices, 50+ languages), +# Gemini Omni video (generation + conversational editing, paid tier). +# Get one at https://aistudio.google.com/apikey +GOOGLE_API_KEY= # GEMINI_API_KEY= # Alias for GOOGLE_API_KEY (takes precedence when both are set) # Alternative to the API key: service-account JSON auth. # TTS uses Cloud Text-to-Speech; Imagen routes to Vertex AI. -GOOGLE_APPLICATION_CREDENTIALS= # path to a service-account JSON key file -GOOGLE_CLOUD_PROJECT= # GCP project id (required for Imagen via Vertex AI) -GOOGLE_CLOUD_LOCATION= # Vertex AI region, default us-central1 +# Path to a service-account JSON key file. +GOOGLE_APPLICATION_CREDENTIALS= +# GCP project id (required for Imagen via Vertex AI). +GOOGLE_CLOUD_PROJECT= +# Vertex AI region, default us-central1. +GOOGLE_CLOUD_LOCATION= # --- Voice --- -ELEVENLABS_API_KEY= # TTS narration, music generation, sound effects -OPENAI_API_KEY= # OpenAI TTS fallback and GPT Image 2 image generation -XAI_API_KEY= # Grok image generation/editing and Grok video generation -DOUBAO_SPEECH_API_KEY= # Volcengine Doubao Speech TTS (new console API Key) -DOUBAO_SPEECH_VOICE_TYPE= # Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts +# TTS narration, music generation, sound effects. +ELEVENLABS_API_KEY= +# OpenAI TTS fallback and GPT Image 2 image generation. +OPENAI_API_KEY= +# Grok image generation/editing and Grok video generation. +XAI_API_KEY= +# Volcengine Doubao Speech TTS (new console API Key). +DOUBAO_SPEECH_API_KEY= +# Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts. +DOUBAO_SPEECH_VOICE_TYPE= # Piper local voices do not require env vars; install `piper-tts` via pip # --- DashScope (Alibaba Cloud Bailian) --- -DASHSCOPE_API_KEY= # Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans) - # Get one at https://dashscope.aliyun.com/ +# Qwen image gen (qwen-image-2.0-pro), TTS (qwen3-tts-flash), ASR with word timestamps (qwen3-asr-flash-filetrans). +# Get one at https://dashscope.aliyun.com/ +DASHSCOPE_API_KEY= # --- Music --- -SUNO_API_KEY= # Suno AI music generation (full songs, instrumentals, any genre) +# Suno AI music generation (full songs, instrumentals, any genre). +SUNO_API_KEY= # --- Video Generation --- -HEYGEN_API_KEY= # HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key) -RUNWAY_API_KEY= # Runway Gen-4 (direct API, alternative to fal.ai routing) -VOLC_ACCESSKEY= # Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing) -VOLC_SECRETKEY= # Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage -VIDEO_GEN_LOCAL_ENABLED= # Set to "true" for local video gen (needs GPU + diffusers) -VIDEO_GEN_LOCAL_MODEL= # Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b -MODAL_LTX2_ENDPOINT_URL= # Modal self-hosted LTX-2 endpoint (optional) +# HeyGen API (VEO, Sora, Runway, Kling, Seedance via single key). +HEYGEN_API_KEY= +# Runway Gen-4 (direct API, alternative to fal.ai routing). +RUNWAY_API_KEY= +# Volcengine Jimeng (即梦 AI) video generation via official API (HMAC-SHA256 V4 signing). +VOLC_ACCESSKEY= +# Secret Access Key paired with VOLC_ACCESSKEY. Get both at https://console.volcengine.com/iam/keymanage +VOLC_SECRETKEY= +# Set to "true" for local video gen (needs GPU + diffusers). +VIDEO_GEN_LOCAL_ENABLED= +# Local model: wan2.1-1.3b, wan2.1-14b, hunyuan-1.5, ltx2-local, cogvideo-5b. +VIDEO_GEN_LOCAL_MODEL= +# Modal self-hosted LTX-2 endpoint (optional). +MODAL_LTX2_ENDPOINT_URL= # --- Stock Media --- -PEXELS_API_KEY= # Pexels stock footage/images (free) -PIXABAY_API_KEY= # Pixabay stock footage/images (free) -UNSPLASH_ACCESS_KEY= # Unsplash stock images (free developer key) +# Pexels stock footage/images (free). +PEXELS_API_KEY= +# Pixabay stock footage/images (free). +PIXABAY_API_KEY= +# Unsplash stock images (free developer key). +UNSPLASH_ACCESS_KEY= # --- Analysis --- -HF_TOKEN= # HuggingFace token — enables speaker diarization in transcriber +# HuggingFace token — enables speaker diarization in transcriber. +HF_TOKEN= # Speech-to-text: optional Azure AI Speech (Fast Transcription). When set, the # agent prefers azure_stt for cloud STT; the local faster-whisper transcriber # remains the default offline path. -AZURE_SPEECH_KEY= # Azure AI Speech resource key ('Keys and Endpoint' page) -AZURE_SPEECH_REGION= # Speech resource region, e.g. eastus +# Azure AI Speech resource key ('Keys and Endpoint' page). +AZURE_SPEECH_KEY= +# Speech resource region, e.g. eastus. +AZURE_SPEECH_REGION= # AZURE_SPEECH_ENDPOINT= # Optional: full custom endpoint URL (overrides region) # --- Avatar (local installs) --- diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 2a0b6bd5..fe464643 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -528,9 +528,10 @@ Music is a critical part of any video. **Surface the music situation to the user Check music availability in this order and present the options: -1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one. -2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly — include quota status if known. -3. **Royalty-free sources:** Note if the user can provide their own track (e.g., from YouTube Audio Library, Jamendo, or other free sources). Offer the `music_library/` drop path. +1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. If tracks exist, list durations and let the user pick one. +2. **Royalty-free search:** Check `registry.get_by_capability("music_search")` for configured search/download tools. Report licensing constraints and whether a key is required. +3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly. +4. **Bring your own:** Note that the user can provide a track (for example from YouTube Audio Library or Jamendo) through the `music_library/` drop path. **Always present the user with explicit choices:** - Use a track from their library (which one?) diff --git a/backlot/server.py b/backlot/server.py index 7e40e8fc..1016a256 100644 --- a/backlot/server.py +++ b/backlot/server.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio import json import time +from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Optional @@ -147,18 +148,22 @@ async def _watch_projects() -> None: hub.publish(pid) +@asynccontextmanager +async def _lifespan(app: FastAPI): + """Own and cleanly stop the project watcher with FastAPI's lifespan API.""" + + task = asyncio.create_task(_watch_projects()) + app.state.watch_task = task + try: + yield + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + def create_app() -> FastAPI: - app = FastAPI(title="Backlot", docs_url=None, redoc_url=None) - - @app.on_event("startup") - async def _startup() -> None: - app.state.watch_task = asyncio.create_task(_watch_projects()) - - @app.on_event("shutdown") - async def _shutdown() -> None: - task = getattr(app.state, "watch_task", None) - if task: - task.cancel() + app = FastAPI(title="Backlot", docs_url=None, redoc_url=None, lifespan=_lifespan) # ---- API ---------------------------------------------------------- diff --git a/backlot/state.py b/backlot/state.py index 3cecfe3b..678cb829 100644 --- a/backlot/state.py +++ b/backlot/state.py @@ -9,6 +9,7 @@ from __future__ import annotations import json import re +from functools import lru_cache from pathlib import Path from typing import Any, Optional @@ -59,6 +60,7 @@ def _rel(project_dir: Path, path: Path) -> str: # Pipeline / stages # --------------------------------------------------------------------------- +@lru_cache(maxsize=32) def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]: """Stage order + gate flags from the manifest; graceful fallback.""" if pipeline_type and pipeline_type != "unknown": diff --git a/lib/checkpoint.py b/lib/checkpoint.py index e35e6164..f0cc4b99 100644 --- a/lib/checkpoint.py +++ b/lib/checkpoint.py @@ -95,9 +95,29 @@ class CheckpointValidationError(ValueError): """Raised when a checkpoint or its canonical artifacts are invalid.""" +def _validate_style_playbook(style_playbook: str | None) -> None: + """Fail closed when a checkpoint names a visual identity that cannot load.""" + + if style_playbook is None: + return + try: + from styles.playbook_loader import list_playbooks, load_playbook + + load_playbook(style_playbook) + except Exception as exc: + try: + available = list_playbooks() + except Exception: + available = [] + raise CheckpointValidationError( + f"Unknown or invalid style_playbook {style_playbook!r}. " + f"Available playbooks: {available}. Underlying error: {exc}" + ) from exc + + @lru_cache(maxsize=1) def _load_checkpoint_schema() -> dict[str, Any]: - with open(CHECKPOINT_SCHEMA_PATH) as f: + with open(CHECKPOINT_SCHEMA_PATH, encoding="utf-8") as f: return json.load(f) @@ -192,6 +212,7 @@ def init_project( Idempotent: re-running preserves the original created_at and merges fields. Returns the project directory. """ + _validate_style_playbook(style_playbook) base = pipeline_dir or PROJECTS_DIR project_dir = base / project_id for sub in ( @@ -208,7 +229,7 @@ def init_project( marker: dict[str, Any] = {} if marker_path.exists(): try: - with open(marker_path) as f: + with open(marker_path, encoding="utf-8") as f: marker = json.load(f) except (json.JSONDecodeError, OSError): marker = {} @@ -221,7 +242,7 @@ def init_project( if style_playbook is not None: marker["style_playbook"] = style_playbook - with open(marker_path, "w") as f: + with open(marker_path, "w", encoding="utf-8") as f: json.dump(marker, f, indent=2) return project_dir @@ -260,6 +281,71 @@ def _stage_requires_approval(pipeline_type: Optional[str], stage: str) -> Option return get_stage_human_approval_default(manifest, stage) +def _enforce_stage_prerequisites( + pipeline_dir: Path, + project_id: str, + pipeline_type: str | None, + stage: str, + status: str, +) -> None: + """Require completed, approved predecessors before advancing a stage. + + ``in_progress`` and failure heartbeats remain writable so an operator can + inspect or resume a broken run. Only lifecycle advancement + (``awaiting_human``/``completed``) is gated. + """ + + if status not in {"awaiting_human", "completed"}: + return + if not pipeline_type or pipeline_type == "unknown": + return + + stages = get_pipeline_stages(pipeline_type) + if stage not in stages: + return + + incomplete: list[str] = [] + unapproved: list[str] = [] + for predecessor in stages[: stages.index(stage)]: + path = _checkpoint_path(pipeline_dir, project_id, predecessor) + if not path.exists(): + incomplete.append(predecessor) + continue + try: + with open(path, encoding="utf-8") as handle: + checkpoint = json.load(handle) + validate_checkpoint(checkpoint) + except (OSError, json.JSONDecodeError, CheckpointValidationError): + incomplete.append(predecessor) + continue + if ( + checkpoint.get("project_id") != project_id + or checkpoint.get("pipeline_type") != pipeline_type + or checkpoint.get("stage") != predecessor + ): + incomplete.append(predecessor) + continue + if checkpoint.get("status") != "completed": + incomplete.append(predecessor) + continue + if _stage_requires_approval(pipeline_type, predecessor) and not checkpoint.get( + "human_approved" + ): + unapproved.append(predecessor) + + if incomplete or unapproved: + details = [] + if incomplete: + details.append(f"incomplete or missing: {incomplete}") + if unapproved: + details.append(f"completed without required approval: {unapproved}") + raise CheckpointValidationError( + f"PREREQUISITE VIOLATION: stage {stage!r} cannot advance; " + + "; ".join(details) + + f". Pipeline order: {stages}." + ) + + def _archive_superseded_checkpoint(path: Path, stage: str) -> None: """Copy an existing checkpoint into history/ before it is overwritten. @@ -275,7 +361,7 @@ def _archive_superseded_checkpoint(path: Path, stage: str) -> None: if not path.exists(): return try: - with open(path) as f: + with open(path, encoding="utf-8") as f: existing = json.load(f) except (json.JSONDecodeError, OSError): existing = {} @@ -314,7 +400,7 @@ def _merge_decision_log( """ path = _decision_log_path(pipeline_dir, project_id) if path.exists(): - with open(path) as f: + with open(path, encoding="utf-8") as f: existing = json.load(f) else: existing = { @@ -329,7 +415,7 @@ def _merge_decision_log( existing["decisions"].append(decision) path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: + with open(path, "w", encoding="utf-8") as f: json.dump(existing, f, indent=2) @@ -351,19 +437,22 @@ def write_checkpoint( metadata: Optional[dict] = None, ) -> Path: """Write a checkpoint file for a pipeline stage.""" - # Backfill a missing pipeline_type from the project marker so that - # omitting the kwarg doesn't quietly bypass gate enforcement. - if not pipeline_type: - marker = None - marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME - if marker_path.exists(): - try: - with open(marker_path) as f: - marker = json.load(f) - except (json.JSONDecodeError, OSError): - marker = None - if isinstance(marker, dict) and marker.get("pipeline_type"): + # Backfill identity fields from the project marker so omitted kwargs + # cannot bypass either gate enforcement or style validation. + marker = None + marker_path = pipeline_dir / project_id / PROJECT_MARKER_FILENAME + if marker_path.exists() and (not pipeline_type or not style_playbook): + try: + with open(marker_path, encoding="utf-8") as f: + marker = json.load(f) + except (json.JSONDecodeError, OSError): + marker = None + if isinstance(marker, dict): + if not pipeline_type and marker.get("pipeline_type"): pipeline_type = marker["pipeline_type"] + if not style_playbook and marker.get("style_playbook"): + style_playbook = marker["style_playbook"] + _validate_style_playbook(style_playbook) valid_stages = ( set(get_pipeline_stages(pipeline_type)) if pipeline_type @@ -404,6 +493,14 @@ def write_checkpoint( f"re-write with status='completed', human_approved=True." ) + _enforce_stage_prerequisites( + pipeline_dir, + project_id, + pipeline_type, + stage, + status, + ) + checkpoint = { "version": "1.0", "project_id": project_id, @@ -456,7 +553,7 @@ def write_checkpoint( # current checkpoint; then archive the superseded file and swap in the # new one atomically. tmp_path = path.with_suffix(".json.tmp") - with open(tmp_path, "w") as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(checkpoint, f, indent=2) # Preserve run history: a superseded completed/awaiting_human checkpoint # is copied to history/ (stage versioning, gate audit trail, replay). @@ -474,7 +571,7 @@ def read_checkpoint( path = _checkpoint_path(pipeline_dir, project_id, stage) if not path.exists(): return None - with open(path) as f: + with open(path, encoding="utf-8") as f: checkpoint = json.load(f) validate_checkpoint(checkpoint) return checkpoint @@ -496,7 +593,7 @@ def get_latest_checkpoint( if not checkpoints: return None - with open(checkpoints[0]) as f: + with open(checkpoints[0], encoding="utf-8") as f: checkpoint = json.load(f) validate_checkpoint(checkpoint) return checkpoint diff --git a/lib/clip_embedder.py b/lib/clip_embedder.py index b9ee82eb..82f0e1cd 100644 --- a/lib/clip_embedder.py +++ b/lib/clip_embedder.py @@ -61,6 +61,20 @@ def model_info() -> dict: } +def _as_feature_tensor(features): + """Normalize CLIP feature return values across transformers versions. + + Transformers 4 returned the projected tensor directly. Transformers 5 may + wrap that tensor in a model-output object whose ``pooler_output`` contains + the same shared-space embedding. Do not project it again: the vision + projection expects the pre-projection width, while ``pooler_output`` is + already the final CLIP width. + """ + + pooled = getattr(features, "pooler_output", None) + return features if pooled is None else pooled + + def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray: """Embed a list of image files into a (N, 512) float32 matrix. @@ -82,7 +96,7 @@ def embed_images(image_paths: Sequence[Union[str, Path]]) -> np.ndarray: inputs = _PROCESSOR(images=images, return_tensors="pt").to(_DEVICE) with torch.no_grad(): - features = _MODEL.get_image_features(**inputs) + features = _as_feature_tensor(_MODEL.get_image_features(**inputs)) features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8) arr = features.cpu().numpy().astype(np.float32, copy=False) # Close PIL handles to avoid leaking file handles on Windows @@ -116,7 +130,7 @@ def embed_texts(texts: Sequence[str]) -> np.ndarray: max_length=77, ).to(_DEVICE) with torch.no_grad(): - features = _MODEL.get_text_features(**inputs) + features = _as_feature_tensor(_MODEL.get_text_features(**inputs)) features = features / features.norm(dim=-1, keepdim=True).clamp_min(1e-8) return features.cpu().numpy().astype(np.float32, copy=False) diff --git a/lib/pipeline_loader.py b/lib/pipeline_loader.py index 6ced59c7..e2f2003d 100644 --- a/lib/pipeline_loader.py +++ b/lib/pipeline_loader.py @@ -26,7 +26,7 @@ from functools import lru_cache @lru_cache(maxsize=1) def _load_manifest_schema() -> dict: - with open(SCHEMA_PATH) as f: + with open(SCHEMA_PATH, encoding="utf-8") as f: return json.load(f) @@ -61,7 +61,7 @@ def load_pipeline(name: str, defs_dir: Optional[Path] = None) -> dict[str, Any]: if not path.exists(): raise FileNotFoundError(f"Pipeline manifest not found: {path}") - with open(path) as f: + with open(path, encoding="utf-8") as f: manifest = yaml.safe_load(f) schema = _load_manifest_schema() diff --git a/remotion-composer/package-lock.json b/remotion-composer/package-lock.json index 4b527227..da9eabb2 100644 --- a/remotion-composer/package-lock.json +++ b/remotion-composer/package-lock.json @@ -15,6 +15,7 @@ "@remotion/player": "^4.0.484", "@remotion/transitions": "^4.0.484", "d3-geo": "^3.1.1", + "fast-uri": "^3.1.5", "react": "^18.2.0", "react-dom": "^18.2.0", "remotion": "^4.0.484", @@ -1951,9 +1952,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "funding": [ { "type": "github", @@ -2253,9 +2254,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "funding": [ { "type": "github", @@ -2345,9 +2346,9 @@ "license": "ISC" }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "funding": [ { "type": "opencollective", @@ -2364,7 +2365,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/remotion-composer/package.json b/remotion-composer/package.json index 2e5bfad9..e913bb2c 100644 --- a/remotion-composer/package.json +++ b/remotion-composer/package.json @@ -15,6 +15,7 @@ "@remotion/player": "^4.0.484", "@remotion/transitions": "^4.0.484", "d3-geo": "^3.1.1", + "fast-uri": "^3.1.5", "react": "^18.2.0", "react-dom": "^18.2.0", "remotion": "^4.0.484", diff --git a/remotion-composer/src/CinematicRenderer.tsx b/remotion-composer/src/CinematicRenderer.tsx index f3c27412..926c59c7 100644 --- a/remotion-composer/src/CinematicRenderer.tsx +++ b/remotion-composer/src/CinematicRenderer.tsx @@ -8,31 +8,13 @@ import { Sequence, interpolate, spring, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; -function resolveAsset(src: string): string { - if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) { - return src; - } - const clean = src.replace(/^file:\/\/\/?/, ""); - if (clean.startsWith("/") || /^[A-Za-z]:[/\\]/.test(clean)) { - const posix = clean.replace(/\\/g, "/"); - // POSIX absolute paths already have a leading "/" — file:// + posix - // gives exactly three slashes. Windows drive paths (C:/...) need the - // extra slash added explicitly. Do not merge these branches — adding - // "file:///" unconditionally double-slashes POSIX paths (file:////...). - if (posix.startsWith("/")) { - return `file://${posix}`; - } - return `file:///${posix}`; - } - return staticFile(clean); -} import { CinematicRendererProps, CinematicTone, CinematicVideoScene } from "./cinematic/types"; import { CaptionOverlay } from "./components/CaptionOverlay"; +import { resolveAsset } from "./lib/resolveAsset"; const FPS = 30; @@ -57,10 +39,11 @@ const toneGradient = (tone: CinematicTone) => { const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => { const frame = useCurrentFrame(); - const { durationInFrames, fps } = useVideoConfig(); + const { fps } = useVideoConfig(); + const durationInFrames = Math.max(1, Math.round(scene.durationSeconds * fps)); const fadeInFrames = scene.fadeInFrames ?? 10; const fadeOutFrames = scene.fadeOutFrames ?? 10; - const fadeOutStart = Math.max(fadeInFrames, durationInFrames - fadeOutFrames); + const fadeOutStart = Math.max(0, durationInFrames - fadeOutFrames); const fadeInOpacity = fadeInFrames === 0 ? 1 @@ -98,6 +81,7 @@ const SceneVideo: React.FC<{ scene: CinematicVideoScene }> = ({ scene }) => { src={resolveAsset(scene.src)} trimBefore={trimBefore} trimAfter={trimAfter} + playbackRate={scene.playbackRate} style={{ width: "100%", height: "100%", diff --git a/remotion-composer/src/CollageBurst.tsx b/remotion-composer/src/CollageBurst.tsx index b0400095..77cf0c74 100644 --- a/remotion-composer/src/CollageBurst.tsx +++ b/remotion-composer/src/CollageBurst.tsx @@ -6,12 +6,12 @@ import { interpolate, random, spring, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import React from "react"; import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay"; +import { resolveAsset } from "./lib/resolveAsset"; const { fontFamily: playfairFamily } = loadPlayfair("normal", { weights: ["400", "700"], @@ -22,23 +22,6 @@ const { fontFamily: playfairItalic } = loadPlayfair("italic", { subsets: ["latin"], }); -function resolveAsset(src: string): string { - if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src; - const clean = src.replace(/^file:\/\/\/?/, ""); - if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) { - const posix = clean.replace(/\\/g, "/"); - // POSIX absolute paths already have a leading "/" — file:// + posix - // gives exactly three slashes. Windows drive paths (C:/...) need the - // extra slash added explicitly. Do not merge these branches — adding - // "file:///" unconditionally double-slashes POSIX paths (file:////...). - if (posix.startsWith("/")) { - return `file://${posix}`; - } - return `file:///${posix}`; - } - return staticFile(clean); -} - export type CollageTransition = | "pop" | "slide-zoom" @@ -63,13 +46,13 @@ export interface CollageClip { seed?: number; } -export interface CollageBurstProps { +export type CollageBurstProps = { backgroundSrc: string; backgroundInSeconds?: number; curtainStartSeconds: number; curtainEndSeconds: number; clips: CollageClip[]; -} +}; // ---------------------------------------------------------------------------- // Opening text — elegant serif card that lives in the pre-reveal black, then diff --git a/remotion-composer/src/Explainer.tsx b/remotion-composer/src/Explainer.tsx index cf7ae03f..91b0da67 100644 --- a/remotion-composer/src/Explainer.tsx +++ b/remotion-composer/src/Explainer.tsx @@ -6,34 +6,10 @@ import { Sequence, interpolate, spring, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import { loadFont } from "@remotion/google-fonts/SpaceGrotesk"; - -// Resolve asset path — handle URLs, absolute paths (Windows/Unix), and public/ relative paths -function resolveAsset(src: string): string { - if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) { - return src; - } - // Strip any file:// prefix - const clean = src.replace(/^file:\/\/\/?/, ""); - // Absolute paths (Unix: /foo, Windows: C:\foo or C:/foo) — convert to file:// URI - // staticFile() only accepts relative paths within public/, so absolute paths must bypass it - if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) { - const posix = clean.replace(/\\/g, "/"); - // POSIX absolute paths already have a leading "/" — file:// + posix - // gives exactly three slashes. Windows drive paths (C:/...) need the - // extra slash added explicitly. Do not merge these branches — adding - // "file:///" unconditionally double-slashes POSIX paths (file:////...). - if (posix.startsWith("/")) { - return `file://${posix}`; - } - return `file:///${posix}`; - } - return staticFile(clean); -} import { TextCard } from "./components/TextCard"; import { StatCard } from "./components/StatCard"; import { CalloutBox } from "./components/CalloutBox"; @@ -54,6 +30,7 @@ import type { TerminalStep } from "./components/TerminalScene"; import { ScreenshotScene } from "./components/ScreenshotScene"; import type { ScreenshotStep } from "./components/ScreenshotScene"; import { ProviderChip } from "./components/ProviderChip"; +import { resolveAsset } from "./lib/resolveAsset"; import type { ParticleType } from "./components/ParticleOverlay"; import { resolveTheme, type ThemeConfig, DEFAULT_THEME } from "./Root"; @@ -254,6 +231,7 @@ interface Cut { animation?: string; transition_in?: string; transition_out?: string; + transition_duration?: number; transform?: { animation?: string; scale?: number; @@ -425,22 +403,49 @@ const ImageScene: React.FC<{ src: string; animation?: string }> = ({ // Enhanced Video Scene // --------------------------------------------------------------------------- -const VideoScene: React.FC<{ src: string; startFrom?: number }> = ({ +const VideoScene: React.FC<{ + src: string; + startFrom?: number; + transitionIn?: string; + transitionOut?: string; + transitionDuration?: number; + sceneDurationSeconds: number; + backgroundColor?: string; +}> = ({ src, startFrom = 0, + transitionIn, + transitionOut, + transitionDuration, + sceneDurationSeconds, + backgroundColor = "#0F172A", }) => { const frame = useCurrentFrame(); - const { fps, durationInFrames } = useVideoConfig(); + const { fps } = useVideoConfig(); + const durationInFrames = Math.max(1, Math.round(sceneDurationSeconds * fps)); - const fadeIn = spring({ frame, fps, config: { damping: 20 } }); - const fadeOutStart = durationInFrames - 8; - const fadeOut = interpolate(frame, [fadeOutStart, durationInFrames], [1, 0.3], { - extrapolateLeft: "clamp", - extrapolateRight: "clamp", - }); + const hardIn = ["cut", "none"].includes((transitionIn || "").toLowerCase()); + const hardOut = ["cut", "none"].includes((transitionOut || "").toLowerCase()); + const transitionFrames = Math.max( + 1, + Math.round((transitionDuration ?? 8 / fps) * fps), + ); + const fadeIn = hardIn + ? 1 + : interpolate(frame, [0, transitionFrames], [0, 1], { + extrapolateLeft: "clamp", + extrapolateRight: "clamp", + }); + const fadeOutStart = Math.max(0, durationInFrames - transitionFrames); + const fadeOut = hardOut + ? 1 + : interpolate(frame, [fadeOutStart, durationInFrames], [1, 0], { + extrapolateLeft: "clamp", + extrapolateRight: "clamp", + }); return ( - + = ({ cut, theme } if (cut.source && isVideo(cut.source)) { - return maybeWrapWithBg(); + return maybeWrapWithBg( + , + ); } // Final fallback — try as image if source exists, otherwise show text_card @@ -741,7 +756,7 @@ const OverlayRenderer: React.FC<{ overlay: Overlay }> = ({ overlay }) => { if (overlay.type === "section_title") { return ( = ({ overlay }) => { if (overlay.type === "stat_reveal") { return ( = ({ overlay }) => { ); } if (overlay.type === "hero_title") { - return ; + return ; } if (overlay.type === "provider_chip" && overlay.providers) { return ( diff --git a/remotion-composer/src/LyricOverlay.tsx b/remotion-composer/src/LyricOverlay.tsx index 4896cf1e..287110f7 100644 --- a/remotion-composer/src/LyricOverlay.tsx +++ b/remotion-composer/src/LyricOverlay.tsx @@ -3,46 +3,29 @@ import { Audio, OffthreadVideo, interpolate, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import React from "react"; import { loadFont as loadPlayfair } from "@remotion/google-fonts/PlayfairDisplay"; +import { resolveAsset } from "./lib/resolveAsset"; const { fontFamily: playfairItalic } = loadPlayfair("italic", { weights: ["400", "700"], subsets: ["latin"], }); -function resolveAsset(src: string): string { - if (src.startsWith("http://") || src.startsWith("https://") || src.startsWith("data:")) return src; - const clean = src.replace(/^file:\/\/\/?/, ""); - if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) { - const posix = clean.replace(/\\/g, "/"); - // POSIX absolute paths already have a leading "/" — file:// + posix - // gives exactly three slashes. Windows drive paths (C:/...) need the - // extra slash added explicitly. Do not merge these branches — adding - // "file:///" unconditionally double-slashes POSIX paths (file:////...). - if (posix.startsWith("/")) { - return `file://${posix}`; - } - return `file:///${posix}`; - } - return staticFile(clean); -} - export interface Lyric { text: string; inSeconds: number; outSeconds: number; } -export interface LyricOverlayProps { +export type LyricOverlayProps = { videoSrc: string; lyrics: Lyric[]; bottomY?: number; // 0..1, vertical center of subtitle band -} +}; const LyricLine: React.FC<{ lyric: Lyric; bottomY: number }> = ({ lyric, bottomY }) => { const frame = useCurrentFrame(); diff --git a/remotion-composer/src/TitledVideo.tsx b/remotion-composer/src/TitledVideo.tsx index 0129271c..0f710cae 100644 --- a/remotion-composer/src/TitledVideo.tsx +++ b/remotion-composer/src/TitledVideo.tsx @@ -5,12 +5,12 @@ import { Sequence, interpolate, spring, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import { getVideoMetadata } from "@remotion/media-utils"; import { loadFont } from "@remotion/google-fonts/PlayfairDisplay"; +import { resolveAsset } from "./lib/resolveAsset"; // Editorial serif for the tagline — Playfair Display at its boldest weight. // Loaded once at module scope so every render reuses the same font face. @@ -19,7 +19,7 @@ const { fontFamily } = loadFont("normal", { subsets: ["latin"], }); -export interface TitledVideoProps { +export type TitledVideoProps = { videoSrc: string; tagline: string; // When the tagline starts animating in, in seconds from the start of the video. @@ -32,33 +32,10 @@ export interface TitledVideoProps { fontSize?: number; // Accent color used for the underline and the glow halo. accentColor?: string; -} +}; // Resolve asset path — handle URLs, absolute paths, and public/ relative paths. // Mirrors the helper in Explainer.tsx so absolute Windows/Unix paths work. -function resolveAsset(src: string): string { - if ( - src.startsWith("http://") || - src.startsWith("https://") || - src.startsWith("data:") - ) { - return src; - } - const clean = src.replace(/^file:\/\/\/?/, ""); - if (clean.startsWith("/") || /^[A-Za-z]:[\\/]/.test(clean)) { - const posix = clean.replace(/\\/g, "/"); - // POSIX absolute paths already have a leading "/" — file:// + posix - // gives exactly three slashes. Windows drive paths (C:/...) need the - // extra slash added explicitly. Do not merge these branches — adding - // "file:///" unconditionally double-slashes POSIX paths (file:////...). - if (posix.startsWith("/")) { - return `file://${posix}`; - } - return `file:///${posix}`; - } - return staticFile(clean); -} - // --------------------------------------------------------------------------- // EditorialTagline — big bold serif, upper-third, drawn underline, warm glow. // Letter-by-letter spring entrance. Designed to feel like a printed headline, diff --git a/remotion-composer/src/cinematic/types.ts b/remotion-composer/src/cinematic/types.ts index 67d8689b..7669968d 100644 --- a/remotion-composer/src/cinematic/types.ts +++ b/remotion-composer/src/cinematic/types.ts @@ -12,6 +12,7 @@ export interface CinematicVideoScene extends CinematicBaseScene { tone?: CinematicTone; trimBeforeSeconds?: number; trimAfterSeconds?: number; + playbackRate?: number; filter?: string; fadeInFrames?: number; fadeOutFrames?: number; diff --git a/remotion-composer/src/components/AnimeScene.tsx b/remotion-composer/src/components/AnimeScene.tsx index acfcd694..c2ab1741 100644 --- a/remotion-composer/src/components/AnimeScene.tsx +++ b/remotion-composer/src/components/AnimeScene.tsx @@ -3,27 +3,11 @@ import { Img, interpolate, spring, - staticFile, useCurrentFrame, useVideoConfig, } from "remotion"; import { ParticleOverlay, type ParticleType } from "./ParticleOverlay"; - -/** - * Resolve asset path — use staticFile() for local paths, passthrough URLs. - * Duplicated from Explainer.tsx to keep the component self-contained. - */ -function resolveAsset(src: string): string { - if ( - src.startsWith("http://") || - src.startsWith("https://") || - src.startsWith("data:") - ) { - return src; - } - const clean = src.replace(/^file:\/\/\/?/, ""); - return staticFile(clean); -} +import { resolveAsset } from "../lib/resolveAsset"; // --------------------------------------------------------------------------- // Types diff --git a/remotion-composer/src/components/CaptionOverlay.tsx b/remotion-composer/src/components/CaptionOverlay.tsx index e31123c6..1f9cf207 100644 --- a/remotion-composer/src/components/CaptionOverlay.tsx +++ b/remotion-composer/src/components/CaptionOverlay.tsx @@ -14,7 +14,7 @@ export interface WordCaption { endMs: number; } -interface CaptionOverlayProps { +type CaptionOverlayProps = { words: WordCaption[]; // How many words to show at once in a "page" wordsPerPage?: number; @@ -23,7 +23,7 @@ interface CaptionOverlayProps { highlightColor?: string; backgroundColor?: string; fontFamily?: string; -} +}; interface CaptionPage { words: WordCaption[]; diff --git a/remotion-composer/src/components/EndTag.tsx b/remotion-composer/src/components/EndTag.tsx index 6ffbfefb..ab785c51 100644 --- a/remotion-composer/src/components/EndTag.tsx +++ b/remotion-composer/src/components/EndTag.tsx @@ -6,7 +6,7 @@ import { useVideoConfig, } from "remotion"; -export interface EndTagProps { +export type EndTagProps = { text: string; palette?: "cool_offwhite_on_black" | "warm_ivory_on_black"; // Optional extra fade hold controls (all in seconds) @@ -19,7 +19,7 @@ export interface EndTagProps { // AbsoluteFill drops its background fill — caller is responsible for // rendering with an alpha-capable codec (VP9/WebM or ProRes 4444). overlay?: boolean; -} +}; const PALETTES = { cool_offwhite_on_black: { diff --git a/remotion-composer/src/components/HeroTitle.tsx b/remotion-composer/src/components/HeroTitle.tsx index 381dd173..d4b6321d 100644 --- a/remotion-composer/src/components/HeroTitle.tsx +++ b/remotion-composer/src/components/HeroTitle.tsx @@ -6,10 +6,10 @@ import { useVideoConfig, } from "remotion"; -interface HeroTitleProps { +type HeroTitleProps = { title: string; subtitle?: string; -} +}; export const HeroTitle: React.FC = ({ title, subtitle }) => { const frame = useCurrentFrame(); diff --git a/remotion-composer/src/components/ProductReveal.tsx b/remotion-composer/src/components/ProductReveal.tsx index c8967ab9..4ad81dba 100644 --- a/remotion-composer/src/components/ProductReveal.tsx +++ b/remotion-composer/src/components/ProductReveal.tsx @@ -9,14 +9,14 @@ import { Easing, } from "remotion"; -export interface ProductRevealProps { +export type ProductRevealProps = { productImage: string; productName: string; price: string; tagline: string; closer: string; accentColor?: string; -} +}; export const ProductReveal: React.FC = ({ productImage, diff --git a/remotion-composer/src/components/ProviderChip.tsx b/remotion-composer/src/components/ProviderChip.tsx index fdc69372..0f11230a 100644 --- a/remotion-composer/src/components/ProviderChip.tsx +++ b/remotion-composer/src/components/ProviderChip.tsx @@ -53,7 +53,7 @@ export const ProviderChip: React.FC = ({ const translateY = interpolate(springIn, [0, 1], [12, 0]); return ( - +
+ src.startsWith("http://") || + src.startsWith("https://") || + src.startsWith("data:"); + +const isWindowsAbsolutePath = (src: string): boolean => + /^[A-Za-z]:[\\/]/.test(src); + +/** Resolve public assets and absolute filesystem paths consistently. */ +export function resolveAsset(src: string): string { + if (isRemoteAsset(src)) { + return src; + } + + const withoutScheme = src.replace(/^file:\/\//i, ""); + const clean = /^\/[A-Za-z]:[\\/]/.test(withoutScheme) + ? withoutScheme.slice(1) + : withoutScheme; + + if (clean.startsWith("/") || isWindowsAbsolutePath(clean)) { + const posix = clean.replace(/\\/g, "/"); + return posix.startsWith("/") ? `file://${posix}` : `file:///${posix}`; + } + + return staticFile(clean); +} diff --git a/schemas/artifacts/__init__.py b/schemas/artifacts/__init__.py index acf15edf..147b2093 100644 --- a/schemas/artifacts/__init__.py +++ b/schemas/artifacts/__init__.py @@ -39,7 +39,7 @@ def load_schema(name: str) -> dict: path = SCHEMA_DIR / f"{name}.schema.json" if not path.exists(): raise FileNotFoundError(f"Schema not found: {path}") - with open(path) as f: + with open(path, encoding="utf-8") as f: return json.load(f) diff --git a/schemas/artifacts/edit_decisions.schema.json b/schemas/artifacts/edit_decisions.schema.json index 63eeca4f..b4dc0089 100644 --- a/schemas/artifacts/edit_decisions.schema.json +++ b/schemas/artifacts/edit_decisions.schema.json @@ -57,6 +57,10 @@ "minimum": 0, "description": "Duration of transition in seconds" }, + "backgroundColor": { + "type": "string", + "description": "Backing color used behind this cut by composition runtimes" + }, "reason": { "type": "string" } }, "additionalProperties": false diff --git a/schemas/pipelines/pipeline_manifest.schema.json b/schemas/pipelines/pipeline_manifest.schema.json index ebf73b40..a7737e43 100644 --- a/schemas/pipelines/pipeline_manifest.schema.json +++ b/schemas/pipelines/pipeline_manifest.schema.json @@ -11,7 +11,7 @@ "description": { "type": "string" }, "category": { "type": "string", - "enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "custom"] + "enum": ["talking_head", "generated", "hybrid", "screen_recording", "animation", "cinematic", "documentary", "custom"] }, "stability": { "type": "string", diff --git a/scripts/backlot_screenshot_stage.py b/scripts/backlot_screenshot_stage.py index 0230a6f6..e968d5a9 100644 --- a/scripts/backlot_screenshot_stage.py +++ b/scripts/backlot_screenshot_stage.py @@ -184,6 +184,15 @@ def stage_project(pid: str, title: str, palette: str, scenes: list, *, brief["topic"] = title cp("research", "completed", {"research_brief": brief}) + proposal = sample_artifact("proposal_packet") + cp("proposal", "awaiting_human", {"proposal_packet": proposal}) + cp( + "proposal", + "completed", + {"proposal_packet": proposal}, + human_approved=True, + ) + script = script_artifact(title, scenes) plan = scene_plan_artifact(scenes, hero) (art_dir / "decision_log.json").write_text(json.dumps(decision_log(pid), indent=2)) diff --git a/skills/INDEX.md b/skills/INDEX.md index 228cd62f..01481798 100644 --- a/skills/INDEX.md +++ b/skills/INDEX.md @@ -60,7 +60,9 @@ Key capability families to look for in the output: | `analysis` | — | Mixed providers | | `character_animation` | — | Local character specs, SVG rigs, pose libraries, action timelines, previews, and QA | | `graphics` | — | Local rendering tools | -| `music_generation` | — | Single-provider | +| `music_library` | — | Discovers user-provided local tracks | +| `music_search` | — | Discovers royalty-free search/download providers | +| `music_generation` | — | Discovers paid/local generation providers | | `subtitle` | — | Pure Python | | `avatar` | — | Local GPU models | | `video_post` | — | FFmpeg-based local tools | diff --git a/skills/pipelines/cinematic/idea-director.md b/skills/pipelines/cinematic/idea-director.md index 0f332c81..d6cbc7ce 100644 --- a/skills/pipelines/cinematic/idea-director.md +++ b/skills/pipelines/cinematic/idea-director.md @@ -88,23 +88,26 @@ Cinematic videos live and die by their audio. **Surface the music situation befo Check availability in this order: -1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. List available tracks with durations and moods. Let the user choose. -2. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, and cost per track. -3. **Royalty-free sources:** Note that the user can provide a track from YouTube Audio Library, Jamendo, or other free sources by dropping it in `music_library/`. +1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List available tracks with durations and moods. +2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report available providers, licensing constraints, and any key requirement. +3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs. +4. **Bring your own:** The user can drop a track from another licensed source into `music_library/`. Present explicit options: ``` MUSIC PLAN ├── Your music library: [N tracks / empty] +├── Royalty-free search: [providers / unavailable] ├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost] └── Bring your own: Drop a track in music_library/ before asset stage Options: (a) Use a library track (which one?) - (b) Provide your own track - (c) Generate via API (if available) - (d) Proceed without music (not recommended for cinematic) + (b) Search a royalty-free provider + (c) Provide your own track + (d) Generate via API (if available) + (e) Proceed without music (not recommended for cinematic) ``` Record the decision in `brief.metadata.music_strategy` with the chosen source and path/prompt. diff --git a/skills/pipelines/cinematic/proposal-director.md b/skills/pipelines/cinematic/proposal-director.md index 1bbdcbd4..13d291e1 100644 --- a/skills/pipelines/cinematic/proposal-director.md +++ b/skills/pipelines/cinematic/proposal-director.md @@ -222,14 +222,16 @@ Let the user select, combine, modify, or redirect entirely. Cinematic videos live and die by their audio. Surface the music situation before the user approves. Check availability in this order: -1. **User music library (`music_library/`)** — list available tracks -2. **Music generation APIs** — report status, cost, and quality honestly -3. **Bring-your-own path** — user can drop a track in `music_library/` +1. **User music library** — query `registry.get_by_capability("music_library")` and list available tracks +2. **Royalty-free search** — query `registry.get_by_capability("music_search")` and report providers/licensing +3. **Music generation APIs** — query `registry.get_by_capability("music_generation")` and report status, cost, and quality honestly +4. **Bring-your-own path** — user can drop a track in `music_library/` Present explicit options: ``` MUSIC PLAN ├── Your music library: [N tracks / empty] +├── Royalty-free search: [providers / unavailable] ├── AI generation: [provider] — [AVAILABLE/UNAVAILABLE] [cost] └── Bring your own: Drop a track in music_library/ before asset stage diff --git a/skills/pipelines/documentary-montage/idea-director.md b/skills/pipelines/documentary-montage/idea-director.md index 204187d2..4e4ab81e 100644 --- a/skills/pipelines/documentary-montage/idea-director.md +++ b/skills/pipelines/documentary-montage/idea-director.md @@ -89,11 +89,14 @@ like abandoned footage at compose time. Do not assume silence will earn itself. If the user has not mentioned music, ASSUME THEY WANT IT and pick: - user-provided track (put path in `music_plan.source_path`), -- music library pick (list what's in `music_library/`), +- music library pick (query `registry.get_by_capability("music_library")` and list tracks), +- royalty-free search (query `registry.get_by_capability("music_search")`, report provider and license), - generated (name the tool and prompt seed with register), - explicit opt-out (`source: "none"` + `opt_out_reason`). -**Warn the user if no music source is available.** Do not silently +Before declaring no source available, also query +`registry.get_by_capability("music_generation")`. **Warn the user if no music +source is available.** Do not silently defer this — it becomes an expensive surprise at the asset stage. ### 5. Note End-Tag Intent (MANDATORY) diff --git a/skills/pipelines/explainer/idea-director.md b/skills/pipelines/explainer/idea-director.md index f500804b..5dda09b9 100644 --- a/skills/pipelines/explainer/idea-director.md +++ b/skills/pipelines/explainer/idea-director.md @@ -129,7 +129,9 @@ If any dimension scores below 3, iterate before submitting. The reviewer will ch ### Step 7: Submit -Call `handle_explainer_idea(state, {"brief": brief_json})` to validate and persist. +Validate `brief_json` against the canonical brief schema, persist it through +the checkpoint protocol, and attach the stage review. There is no separate +explainer submit function. ## Playbook Selection Guide diff --git a/skills/pipelines/explainer/proposal-director.md b/skills/pipelines/explainer/proposal-director.md index db718b72..ffb482eb 100644 --- a/skills/pipelines/explainer/proposal-director.md +++ b/skills/pipelines/explainer/proposal-director.md @@ -369,9 +369,9 @@ Music is a critical part of the video's feel. **Surface the music situation to t **Check music availability in this order:** -1. **User music library (`music_library/`):** Check if this folder exists and contains tracks. If so, list available tracks with durations and let the user pick one. -2. **Music generation APIs:** Check which music tools are available via the registry (`registry.get_by_capability("music_generation")`). Report their status honestly. -3. **Stock music sources:** Note if stock music is available via any provider. +1. **User music library:** Check `registry.get_by_capability("music_library")` and inspect `music_library/`. List tracks with durations. +2. **Royalty-free search:** Check `registry.get_by_capability("music_search")`. Report providers, licensing constraints, and key requirements. +3. **Music generation APIs:** Check `registry.get_by_capability("music_generation")`. Report status, quota, cost, and quality tradeoffs honestly. **Present to the user:** @@ -381,15 +381,17 @@ MUSIC PLAN │ ├── cosmic_interstellar_space.mp3 (3:13) — ambient, cosmic │ ├── cinematic_epic.mp3 (2:45) — dramatic, building │ └── lofi_beat.mp3 (4:00) — chill, electronic +├── Royalty-free search: [providers / unavailable] ├── AI generation: music_gen (ElevenLabs) — UNAVAILABLE (plan limit) └── Recommendation: Use "cosmic_interstellar_space.mp3" from your library OR provide a different track before asset generation Would you like to: (a) Use a track from your library (which one?) - (b) Provide a different track (drop it in music_library/) - (c) Generate one via API (if available) - (d) Proceed without music + (b) Search a royalty-free provider + (c) Provide a different track (drop it in music_library/) + (d) Generate one via API (if available) + (e) Proceed without music ``` **If no music source is available:** Tell the user explicitly. Do NOT let this surface as a surprise at the asset stage. Offer the `music_library/` path so they can add a track before production starts. diff --git a/skills/pipelines/explainer/scene-director.md b/skills/pipelines/explainer/scene-director.md index 9642d666..131eb455 100644 --- a/skills/pipelines/explainer/scene-director.md +++ b/skills/pipelines/explainer/scene-director.md @@ -63,9 +63,17 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v } ``` -#### Scene Types and When to Use Them +#### Render Templates and Scene Types -| Type | Best For | Available Tools | Duration Guidance | +The canonical `scene_plan.scenes[].type` vocabulary is `talking_head`, `broll`, +`animation`, `character_scene`, `diagram`, `text_card`, `transition`, +`generated`, and `screen_recording`. Names such as `hero_title`, `stat_card`, +and chart/card variants below are downstream Remotion `cut.type` templates, +not valid scene-plan types. During scene planning, use `text_card` or +`animation` as appropriate and state the intended render template in the scene +description; the Edit stage converts that intent into `cut.type`. + +| Render template or scene type | Best For | Available Tools | Duration Guidance | |------|----------|-----------------|-------------------| | `hero_title` | Opening titles, dramatic reveals | Remotion HeroTitle (theme-driven title treatment) | 3-5s | | `stat_card` | Big dramatic numbers, impactful metrics | Remotion StatCard (large stat + subtitle) | 4-6s | @@ -84,7 +92,11 @@ Transform each script section into 1-3 visual scenes. Each scene is a distinct v | `broll` | Context, real-world examples | Stock or generated footage | 3-6s | | `screen_recording` | Code demos, UI walkthroughs | Recorded or simulated | 5-15s | -**Zero-key scene selection:** When no image/video generation is available, prefer `hero_title`, `stat_card`, `bar_chart`, `line_chart`, `pie_chart`, `kpi_grid`, `comparison`, `callout`, `progress_bar`, and `text_card`. These render entirely from Remotion components with zero external dependencies and can still feel distinct if you derive color, typography, and pacing from the subject instead of defaulting to a generic dashboard aesthetic. +**Zero-key scene selection:** When no image/video generation is available, +plan `text_card`, `animation`, or `diagram` scenes and name an appropriate +downstream template (`hero_title`, `stat_card`, charts, `comparison`, +`callout`, or `progress_bar`) in each description. These render entirely from +Remotion components with zero external dependencies. ### Step 4: Apply the Visual Technique Library @@ -226,7 +238,9 @@ If any dimension scores below 3, revise. ### Step 8: Submit -Call `handle_explainer_scene_plan(state, {"scene_plan": scene_plan_json})` to validate and persist. +Validate `scene_plan_json` against the canonical scene-plan schema, persist it +through the checkpoint protocol, and attach the stage review. There is no +separate explainer submit function. ## Common Pitfalls diff --git a/skills/pipelines/explainer/script-director.md b/skills/pipelines/explainer/script-director.md index f00404c8..f33a9e93 100644 --- a/skills/pipelines/explainer/script-director.md +++ b/skills/pipelines/explainer/script-director.md @@ -206,7 +206,9 @@ If any dimension scores below 3, revise before submitting. ### Step 7: Submit -Call `handle_explainer_script(state, {"script": script_json})` to validate and persist. +Validate `script_json` against the canonical script schema, persist it through +the checkpoint protocol, and attach the stage review. There is no separate +explainer submit function. ### Mid-Production Fact Verification diff --git a/styles/playbook_loader.py b/styles/playbook_loader.py index 0ece61f6..70155eb4 100644 --- a/styles/playbook_loader.py +++ b/styles/playbook_loader.py @@ -26,7 +26,7 @@ SCHEMA_PATH = ( def _load_playbook_schema() -> dict: - with open(SCHEMA_PATH) as f: + with open(SCHEMA_PATH, encoding="utf-8") as f: return json.load(f) @@ -45,7 +45,7 @@ def load_playbook(name: str, styles_dir: Optional[Path] = None) -> dict[str, Any if not path.exists(): raise FileNotFoundError(f"Playbook not found: {path}") - with open(path) as f: + with open(path, encoding="utf-8") as f: playbook = yaml.safe_load(f) validate_playbook(playbook) diff --git a/tests/backlot/test_gate_scenarios.py b/tests/backlot/test_gate_scenarios.py index d00920c3..6d96c4de 100644 --- a/tests/backlot/test_gate_scenarios.py +++ b/tests/backlot/test_gate_scenarios.py @@ -7,7 +7,11 @@ import pytest from backlot import state as state_mod from backlot.state import load_board_state -from lib.checkpoint import CheckpointValidationError, write_checkpoint +from lib.checkpoint import ( + CANONICAL_STAGE_ARTIFACTS, + CheckpointValidationError, + write_checkpoint, +) def _script_artifact() -> dict: @@ -23,6 +27,22 @@ def _manifest_artifact() -> dict: return {"version": "1.0", "assets": [], "total_cost_usd": 0.0} +def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None: + from tests.contracts.test_phase0_contracts import sample_artifact + + for stage in stages: + artifact_name = CANONICAL_STAGE_ARTIFACTS[stage] + write_checkpoint( + tmp_path, + project_id, + stage, + "completed", + {artifact_name: sample_artifact(artifact_name)}, + pipeline_type=pipeline_type, + human_approved=True, + ) + + def _write(path: Path, data: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data), encoding="utf-8") @@ -73,6 +93,15 @@ def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypat def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path): + _approve_predecessors( + tmp_path, + "film", + "cinematic", + "research", + "proposal", + "script", + "scene_plan", + ) write_checkpoint( tmp_path, "film", diff --git a/tests/backlot/test_ui_bug_bash.py b/tests/backlot/test_ui_bug_bash.py index 8f5cf3e5..7c1e3458 100644 --- a/tests/backlot/test_ui_bug_bash.py +++ b/tests/backlot/test_ui_bug_bash.py @@ -10,7 +10,8 @@ import urllib.request import pytest -from lib.checkpoint import init_project, write_checkpoint +from lib.checkpoint import CANONICAL_STAGE_ARTIFACTS, init_project, write_checkpoint +from lib.pipeline_loader import get_stage_order, load_pipeline from scripts import backlot_screenshot_stage from tests.contracts.test_phase0_contracts import sample_artifact @@ -32,6 +33,28 @@ APPROVAL_CASES = [ ] +def _complete_predecessors(root, project_id: str, pipeline_type: str, stage: str) -> None: + order = get_stage_order(load_pipeline(pipeline_type)) + for predecessor in order[: order.index(stage)]: + artifact_name = CANONICAL_STAGE_ARTIFACTS.get(predecessor) + if artifact_name: + artifact = sample_artifact(artifact_name) + if artifact_name == "edit_decisions": + artifact["render_runtime"] = "ffmpeg" + artifacts = {artifact_name: artifact} + else: + artifacts = {} + write_checkpoint( + root, + project_id, + predecessor, + "completed", + artifacts, + pipeline_type=pipeline_type, + human_approved=True, + ) + + def _build_approval_projects() -> None: root = backlot_screenshot_stage.STAGE_DIR for project_id, pipeline_type, stage, artifact_name, _visible_text in APPROVAL_CASES: @@ -55,6 +78,7 @@ def _build_approval_projects() -> None: pipeline_type=pipeline_type, pipeline_dir=root, ) + _complete_predecessors(root, project_id, pipeline_type, stage) write_checkpoint( root, project_id, @@ -80,6 +104,12 @@ def _build_approval_projects() -> None: pipeline_type="character-animation", pipeline_dir=root, ) + _complete_predecessors( + root, + "gate-character-design", + "character-animation", + "character_design", + ) write_checkpoint( root, "gate-character-design", diff --git a/tests/contracts/test_agent_instruction_integrity.py b/tests/contracts/test_agent_instruction_integrity.py new file mode 100644 index 00000000..82443ab5 --- /dev/null +++ b/tests/contracts/test_agent_instruction_integrity.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _read(relative_path: str) -> str: + return (REPO_ROOT / relative_path).read_text(encoding="utf-8") + + +def test_music_plans_discover_all_music_capabilities() -> None: + instruction_files = [ + "AGENT_GUIDE.md", + "skills/pipelines/cinematic/idea-director.md", + "skills/pipelines/cinematic/proposal-director.md", + "skills/pipelines/documentary-montage/idea-director.md", + "skills/pipelines/explainer/proposal-director.md", + ] + + for relative_path in instruction_files: + text = _read(relative_path) + for capability in ("music_library", "music_search", "music_generation"): + assert f'get_by_capability("{capability}")' in text, ( + f"{relative_path} omits the {capability!r} music source" + ) + + +def test_explainer_directors_do_not_reference_fictitious_submit_functions() -> None: + for stage in ("idea", "script", "scene"): + text = _read(f"skills/pipelines/explainer/{stage}-director.md") + assert "handle_explainer_" not in text diff --git a/tests/contracts/test_backlot_contract.py b/tests/contracts/test_backlot_contract.py index 50245baa..82a257fc 100644 --- a/tests/contracts/test_backlot_contract.py +++ b/tests/contracts/test_backlot_contract.py @@ -6,6 +6,7 @@ import json import pytest from lib.checkpoint import ( + CANONICAL_STAGE_ARTIFACTS, CheckpointValidationError, HISTORY_DIRNAME, PROJECT_MARKER_FILENAME, @@ -27,6 +28,22 @@ def _minimal_script() -> dict: } +def _approve_predecessors(tmp_path, project_id, pipeline_type, *stages) -> None: + from tests.contracts.test_phase0_contracts import sample_artifact + + for stage in stages: + artifact_name = CANONICAL_STAGE_ARTIFACTS[stage] + write_checkpoint( + tmp_path, + project_id, + stage, + "completed", + artifacts={artifact_name: sample_artifact(artifact_name)}, + pipeline_type=pipeline_type, + human_approved=True, + ) + + class TestGateEnforcement: """GI-4: gated stages cannot be completed without approval evidence.""" @@ -39,6 +56,9 @@ class TestGateEnforcement: ) def test_awaiting_human_is_the_correct_gate_state(self, tmp_path): + _approve_predecessors( + tmp_path, "proj", "animated-explainer", "research", "proposal" + ) path = write_checkpoint( tmp_path, "proj", "script", "awaiting_human", artifacts={"script": _minimal_script()}, @@ -51,6 +71,9 @@ class TestGateEnforcement: assert cp["human_approval_required"] is True def test_completed_with_approval_passes(self, tmp_path): + _approve_predecessors( + tmp_path, "proj", "animated-explainer", "research", "proposal" + ) path = write_checkpoint( tmp_path, "proj", "script", "completed", artifacts={"script": _minimal_script()}, @@ -84,6 +107,9 @@ class TestCheckpointHistory: """Superseded checkpoints are archived, not destroyed.""" def test_overwrite_archives_previous(self, tmp_path): + _approve_predecessors( + tmp_path, "proj", "animated-explainer", "research", "proposal" + ) write_checkpoint( tmp_path, "proj", "script", "awaiting_human", artifacts={"script": _minimal_script()}, diff --git a/tests/contracts/test_env_example.py b/tests/contracts/test_env_example.py new file mode 100644 index 00000000..a2dd245d --- /dev/null +++ b/tests/contracts/test_env_example.py @@ -0,0 +1,19 @@ +from pathlib import Path + +from dotenv import dotenv_values + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def test_env_example_does_not_turn_comments_into_credentials() -> None: + """A fresh copy of .env.example must leave every documented key unset.""" + + values = dotenv_values(REPO_ROOT / ".env.example") + false_credentials = { + key: value + for key, value in values.items() + if isinstance(value, str) and value.lstrip().startswith("#") + } + + assert false_credentials == {} diff --git a/tests/contracts/test_phase2_contracts.py b/tests/contracts/test_phase2_contracts.py index 6c76931f..9c0746cf 100644 --- a/tests/contracts/test_phase2_contracts.py +++ b/tests/contracts/test_phase2_contracts.py @@ -174,10 +174,14 @@ class TestPhase2ErrorHandling: # Either succeeds (provider available) or fails gracefully assert isinstance(r, ToolResult) - def test_diagram_gen_empty_boxes(self): + def test_diagram_gen_empty_boxes(self, tmp_path): tool = DiagramGen() if tool.get_status() == ToolStatus.AVAILABLE: - r = tool.execute({"diagram_type": "boxes", "boxes": []}) + r = tool.execute({ + "diagram_type": "boxes", + "boxes": [], + "output_path": str(tmp_path / "empty-boxes.png"), + }) assert isinstance(r, ToolResult) diff --git a/tests/contracts/test_phase3_contracts.py b/tests/contracts/test_phase3_contracts.py index 3fa4a3b0..d582becd 100644 --- a/tests/contracts/test_phase3_contracts.py +++ b/tests/contracts/test_phase3_contracts.py @@ -9,6 +9,7 @@ import builtins import base64 import os import shutil +from types import SimpleNamespace from pathlib import Path from unittest.mock import MagicMock, patch @@ -571,16 +572,26 @@ class TestVeoVideo: called_kwargs = mock_client.models.generate_videos.call_args[1] assert called_kwargs["image"] is not None - def test_vertex_ai_mode_rejection(self): + def test_vertex_ai_mode_requires_inline_video_bytes(self): tool = VeoVideo() mock_client = MagicMock() mock_client.vertexai = True - if hasattr(mock_client, "_api_client"): - delattr(mock_client, "_api_client") + mock_client.models.generate_videos.return_value = SimpleNamespace( + done=True, + error=None, + response=SimpleNamespace( + generated_videos=[ + SimpleNamespace(video=SimpleNamespace(video_bytes=None)) + ] + ), + ) with ( patch.dict(os.environ, {"GEMINI_API_KEY": "test_key"}), - patch("google.genai.Client", return_value=mock_client), + patch( + "tools.google_credentials.get_genai_client", + return_value=mock_client, + ), ): inputs = { "prompt": "cinematic shot", @@ -589,7 +600,7 @@ class TestVeoVideo: res = tool.execute(inputs) assert res.success is False assert res.error is not None - assert "only supported using the Gemini Developer API" in res.error + assert "without inline bytes" in res.error def test_missing_local_image_paths(self): tool = VeoVideo() diff --git a/tests/contracts/test_pipeline_manifest_categories.py b/tests/contracts/test_pipeline_manifest_categories.py new file mode 100644 index 00000000..54983bb1 --- /dev/null +++ b/tests/contracts/test_pipeline_manifest_categories.py @@ -0,0 +1,9 @@ +"""Contracts for the category vocabulary used by shipped pipelines.""" + +from lib.pipeline_loader import load_pipeline + + +def test_documentary_pipeline_uses_a_schema_supported_category() -> None: + manifest = load_pipeline("documentary-montage") + + assert manifest["category"] == "documentary" diff --git a/tests/contracts/test_remotion_video_transition_contract.py b/tests/contracts/test_remotion_video_transition_contract.py new file mode 100644 index 00000000..3d30934b --- /dev/null +++ b/tests/contracts/test_remotion_video_transition_contract.py @@ -0,0 +1,28 @@ +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def test_video_scene_honors_hard_cut_tokens_and_backing_color() -> None: + source = (REPO_ROOT / "remotion-composer/src/Explainer.tsx").read_text( + encoding="utf-8" + ) + + assert '["cut", "none"].includes((transitionIn || "").toLowerCase())' in source + assert '["cut", "none"].includes((transitionOut || "").toLowerCase())' in source + assert "transitionIn={cut.transition_in}" in source + assert "transitionOut={cut.transition_out}" in source + assert "sceneDurationSeconds={cut.out_seconds - cut.in_seconds}" in source + assert "Math.round(sceneDurationSeconds * fps)" in source + assert "durationInFrames - transitionFrames" in source + assert "backgroundColor={cut.backgroundColor}" in source + + +def test_cinematic_fades_are_bounded_by_each_scene_duration() -> None: + source = (REPO_ROOT / "remotion-composer/src/CinematicRenderer.tsx").read_text( + encoding="utf-8" + ) + + assert "Math.round(scene.durationSeconds * fps)" in source + assert "durationInFrames - fadeOutFrames" in source diff --git a/tests/contracts/test_utf8_file_io.py b/tests/contracts/test_utf8_file_io.py new file mode 100644 index 00000000..d3a7ba30 --- /dev/null +++ b/tests/contracts/test_utf8_file_io.py @@ -0,0 +1,30 @@ +import ast +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNTIME_FILES = [ + "lib/checkpoint.py", + "lib/pipeline_loader.py", + "schemas/artifacts/__init__.py", + "styles/playbook_loader.py", +] + + +@pytest.mark.parametrize("relative_path", RUNTIME_FILES) +def test_pipeline_contract_files_use_explicit_utf8(relative_path: str) -> None: + path = REPO_ROOT / relative_path + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + + bare_opens = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Name) or node.func.id != "open": + continue + if not any(keyword.arg == "encoding" for keyword in node.keywords): + bare_opens.append(node.lineno) + + assert bare_opens == [], f"bare open() calls at lines {bare_opens}" diff --git a/tests/lib/test_checkpoint_prerequisites.py b/tests/lib/test_checkpoint_prerequisites.py new file mode 100644 index 00000000..ee9b5f97 --- /dev/null +++ b/tests/lib/test_checkpoint_prerequisites.py @@ -0,0 +1,152 @@ +import json + +import pytest +from tests.contracts.test_phase0_contracts import sample_artifact + +from lib.checkpoint import ( + CheckpointValidationError, + init_project, + write_checkpoint, +) + + +def _script_artifact() -> dict: + return { + "version": "1.0", + "title": "Smoke", + "total_duration_seconds": 1, + "sections": [ + { + "id": "s1", + "text": "One second.", + "start_seconds": 0, + "end_seconds": 1, + } + ], + } + + +def test_later_stage_cannot_skip_a_missing_predecessor(tmp_path) -> None: + init_project( + "run", + title="Run", + pipeline_type="framework-smoke", + pipeline_dir=tmp_path, + ) + + with pytest.raises(CheckpointValidationError, match="PREREQUISITE VIOLATION"): + write_checkpoint( + tmp_path, + "run", + "script", + "completed", + {"script": _script_artifact()}, + pipeline_type="framework-smoke", + human_approved=True, + ) + + +def test_later_stage_rejects_unapproved_gated_predecessor(tmp_path) -> None: + project_dir = init_project( + "run", + title="Run", + pipeline_type="framework-smoke", + pipeline_dir=tmp_path, + ) + predecessor_path = write_checkpoint( + tmp_path, + "run", + "research", + "awaiting_human", + {"research_brief": sample_artifact("research_brief")}, + pipeline_type="framework-smoke", + ) + predecessor = json.loads(predecessor_path.read_text(encoding="utf-8")) + predecessor["status"] = "completed" + predecessor["human_approved"] = False + predecessor_path.write_text(json.dumps(predecessor), encoding="utf-8") + + with pytest.raises(CheckpointValidationError, match="completed without required approval"): + write_checkpoint( + tmp_path, + "run", + "script", + "completed", + {"script": _script_artifact()}, + pipeline_type="framework-smoke", + human_approved=True, + ) + + +def test_malformed_predecessor_cannot_forge_completion(tmp_path) -> None: + project_dir = init_project( + "run", + title="Run", + pipeline_type="framework-smoke", + pipeline_dir=tmp_path, + ) + (project_dir / "checkpoint_research.json").write_text( + json.dumps({"status": "completed", "human_approved": True}), + encoding="utf-8", + ) + + with pytest.raises(CheckpointValidationError, match="incomplete or missing"): + write_checkpoint( + tmp_path, + "run", + "script", + "completed", + {"script": _script_artifact()}, + pipeline_type="framework-smoke", + human_approved=True, + ) + + +def test_in_progress_heartbeat_is_not_blocked_by_prerequisites(tmp_path) -> None: + init_project( + "run", + title="Run", + pipeline_type="framework-smoke", + pipeline_dir=tmp_path, + ) + + path = write_checkpoint( + tmp_path, + "run", + "script", + "in_progress", + {}, + pipeline_type="framework-smoke", + ) + + assert path.exists() + + +def test_unknown_style_playbook_fails_before_project_creation(tmp_path) -> None: + with pytest.raises(CheckpointValidationError, match="style_playbook"): + init_project( + "run", + title="Run", + pipeline_type="framework-smoke", + pipeline_dir=tmp_path, + style_playbook="does-not-exist", + ) + + assert not (tmp_path / "run").exists() + + +def test_marker_derived_unknown_playbook_blocks_later_writes(tmp_path) -> None: + project_dir = tmp_path / "run" + project_dir.mkdir() + (project_dir / "project.json").write_text( + json.dumps({ + "version": "1.0", + "project_id": "run", + "pipeline_type": "framework-smoke", + "style_playbook": "does-not-exist", + }), + encoding="utf-8", + ) + + with pytest.raises(CheckpointValidationError, match="style_playbook"): + write_checkpoint(tmp_path, "run", "research", "in_progress", {}) diff --git a/tests/lib/test_clip_embedder_compat.py b/tests/lib/test_clip_embedder_compat.py new file mode 100644 index 00000000..fc9e4741 --- /dev/null +++ b/tests/lib/test_clip_embedder_compat.py @@ -0,0 +1,26 @@ +from lib.clip_embedder import _as_feature_tensor + + +class _Tensor: + pass + + +class _ModelOutput: + def __init__(self, pooler_output): + self.pooler_output = pooler_output + self.last_hidden_state = object() + + +def test_transformers_4_tensor_passes_through() -> None: + tensor = _Tensor() + assert _as_feature_tensor(tensor) is tensor + + +def test_transformers_5_output_unwraps_projected_pooler_output() -> None: + tensor = _Tensor() + assert _as_feature_tensor(_ModelOutput(tensor)) is tensor + + +def test_missing_pooler_output_does_not_replace_features_with_none() -> None: + output = _ModelOutput(None) + assert _as_feature_tensor(output) is output diff --git a/tests/tools/test_audio_mixer_target_duration.py b/tests/tools/test_audio_mixer_target_duration.py new file mode 100644 index 00000000..58068450 --- /dev/null +++ b/tests/tools/test_audio_mixer_target_duration.py @@ -0,0 +1,94 @@ +import shutil +import subprocess +import math +import struct +import wave +from pathlib import Path + +import pytest + +from tools.audio.audio_mixer import AudioMixer + + +pytestmark = pytest.mark.skipif( + shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, + reason="ffmpeg and ffprobe are required", +) + + +def _tone(path: Path, frequency: int, duration: float) -> None: + subprocess.run( + [ + "ffmpeg", "-y", "-f", "lavfi", "-i", + f"sine=frequency={frequency}:duration={duration}", str(path), + ], + capture_output=True, + check=True, + timeout=30, + ) + + +def _duration(path: Path) -> float: + result = subprocess.run( + [ + "ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "csv=p=0", str(path), + ], + capture_output=True, + check=True, + text=True, + timeout=30, + ) + return float(result.stdout.strip()) + + +def _tail_rms(path: Path, start_seconds: float, end_seconds: float) -> float: + with wave.open(str(path), "rb") as handle: + sample_width = handle.getsampwidth() + assert sample_width == 2 + frame_rate = handle.getframerate() + handle.setpos(int(start_seconds * frame_rate)) + raw = handle.readframes(int((end_seconds - start_seconds) * frame_rate)) + samples = struct.unpack(f"<{len(raw) // 2}h", raw) + return math.sqrt(sum(sample * sample for sample in samples) / len(samples)) + + +def test_full_mix_can_pin_the_composition_length(tmp_path) -> None: + speech = tmp_path / "speech.wav" + music = tmp_path / "music.wav" + output = tmp_path / "mix.wav" + _tone(speech, 440, 1) + _tone(music, 220, 3) + + result = AudioMixer().execute({ + "operation": "full_mix", + "tracks": [ + {"path": str(speech), "role": "speech"}, + {"path": str(music), "role": "music"}, + ], + "ducking": {"enabled": True}, + "normalize": False, + "target_duration": 3, + "output_path": str(output), + }) + + assert result.success, result.error + assert result.data["target_duration"] == 3 + assert _duration(output) == pytest.approx(3.0, abs=0.15) + assert _tail_rms(output, 2.0, 2.5) > 100 + + +@pytest.mark.parametrize("target", [0, -1, "not-a-number"]) +def test_full_mix_rejects_invalid_target_duration(tmp_path, target) -> None: + tone = tmp_path / "tone.wav" + _tone(tone, 440, 0.25) + + result = AudioMixer().execute({ + "operation": "full_mix", + "tracks": [{"path": str(tone), "role": "speech"}], + "target_duration": target, + "output_path": str(tmp_path / "mix.wav"), + }) + + assert result.success is False + assert "target_duration" in result.error diff --git a/tests/tools/test_bg_remove_api.py b/tests/tools/test_bg_remove_api.py new file mode 100644 index 00000000..a6d47a6b --- /dev/null +++ b/tests/tools/test_bg_remove_api.py @@ -0,0 +1,28 @@ +import sys +from unittest.mock import MagicMock + +from PIL import Image + + +def test_bg_remove_selects_model_through_rembg_session(monkeypatch, tmp_path) -> None: + fake_rembg = MagicMock() + fake_rembg.new_session.return_value = "selected-session" + fake_rembg.remove.side_effect = lambda image, **kwargs: image.convert("RGBA") + monkeypatch.setitem(sys.modules, "rembg", fake_rembg) + + input_path = tmp_path / "input.png" + Image.new("RGB", (8, 8), (10, 20, 30)).save(input_path) + + from tools.enhancement.bg_remove import BgRemove + + result = BgRemove().execute({ + "input_path": str(input_path), + "output_path": str(tmp_path / "output.png"), + "model": "isnet-general-use", + }) + + assert result.success, result.error + fake_rembg.new_session.assert_called_once_with("isnet-general-use") + kwargs = fake_rembg.remove.call_args.kwargs + assert kwargs["session"] == "selected-session" + assert "model_name" not in kwargs diff --git a/tests/tools/test_cinematic_remotion_adapter.py b/tests/tools/test_cinematic_remotion_adapter.py new file mode 100644 index 00000000..b3b02fa8 --- /dev/null +++ b/tests/tools/test_cinematic_remotion_adapter.py @@ -0,0 +1,177 @@ +import json +from pathlib import Path + +import pytest + +from tools.video.video_compose import VideoCompose + + +def test_cinematic_cut_adapter_builds_a_sequential_timeline() -> None: + scenes = VideoCompose._cuts_to_cinematic_scenes([ + { + "id": "v1", + "source": "clip.mp4", + "in_seconds": 2, + "out_seconds": 6, + "transition_in": "cut", + "transition_out": "none", + }, + { + "id": "title", + "source": "", + "type": "hero_title", + "text": "The signal arrives", + "in_seconds": 0, + "out_seconds": 3, + }, + ]) + + assert scenes[0] == { + "id": "v1", + "startSeconds": 0.0, + "durationSeconds": 4.0, + "kind": "video", + "src": "clip.mp4", + "trimBeforeSeconds": 2.0, + "trimAfterSeconds": 6.0, + "playbackRate": 1.0, + "fadeInFrames": 0, + "fadeOutFrames": 0, + } + assert scenes[1]["kind"] == "title" + assert scenes[1]["startSeconds"] == 4.0 + assert scenes[1]["text"] == "The signal arrives" + + +def test_cinematic_cut_adapter_preserves_playback_speed() -> None: + scenes = VideoCompose._cuts_to_cinematic_scenes([ + { + "id": "fast", + "source": "clip.mp4", + "in_seconds": 2, + "out_seconds": 6, + "speed": 2, + } + ]) + + assert scenes[0]["durationSeconds"] == 2 + assert scenes[0]["playbackRate"] == 2 + + +@pytest.mark.parametrize("uri_style", ["standard", "legacy_windows"]) +def test_remotion_media_staging_decodes_file_uris(tmp_path, uri_style) -> None: + source = tmp_path / "clip with space.mp4" + source.write_bytes(b"video") + public_dir = tmp_path / "public" + uri = source.as_uri() + if uri_style == "legacy_windows" and len(source.drive) == 2: + uri = f"file://{source.drive}{source.as_posix()[2:]}".replace(" ", "%20") + props = {"scenes": [{"src": uri}]} + + staged_count = VideoCompose._stage_remotion_media(props, public_dir) + + assert staged_count == 1 + assert props["scenes"][0]["src"] != uri + assert (public_dir / props["scenes"][0]["src"]).read_bytes() == b"video" + + +def test_remotion_render_adapts_cuts_and_stages_local_video(monkeypatch, tmp_path) -> None: + source = tmp_path / "source.mp4" + source.write_bytes(b"not-a-real-video") + output = tmp_path / "render.mp4" + captured = {} + + def fake_run_command(self, command, **kwargs): + captured["command"] = command + captured["timeout"] = kwargs["timeout"] + props_arg = next(arg for arg in command if arg.startswith("--props=")) + captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text()) + public_arg = next(arg for arg in command if arg.startswith("--public-dir=")) + public_dir = Path(public_arg.split("=", 1)[1]) + captured["staged_exists_during_render"] = ( + public_dir / captured["props"]["scenes"][0]["src"] + ).exists() + output.write_bytes(b"rendered") + + monkeypatch.setattr(VideoCompose, "run_command", fake_run_command) + + result = VideoCompose()._remotion_render({ + "edit_decisions": { + "renderer_family": "cinematic-trailer", + "cuts": [ + { + "id": "v1", + "source": str(source), + "in_seconds": 0, + "out_seconds": 2, + } + ], + }, + "output_path": str(output), + }) + + assert result.success, result.error + assert "cuts" not in captured["props"] + assert captured["props"]["scenes"][0]["kind"] == "video" + assert captured["staged_exists_during_render"] is True + assert result.data["staged_media_count"] == 1 + + +def test_remotion_timeout_scales_with_scene_count(monkeypatch, tmp_path) -> None: + output = tmp_path / "render.mp4" + captured = {} + + def fake_run_command(self, command, **kwargs): + captured["timeout"] = kwargs["timeout"] + output.write_bytes(b"rendered") + + monkeypatch.setattr(VideoCompose, "run_command", fake_run_command) + cuts = [ + { + "id": f"title-{index}", + "source": "", + "type": "hero_title", + "text": str(index), + "in_seconds": 0, + "out_seconds": 1, + } + for index in range(50) + ] + + result = VideoCompose()._remotion_render({ + "edit_decisions": {"renderer_family": "cinematic-trailer", "cuts": cuts}, + "output_path": str(output), + }) + + assert result.success, result.error + assert captured["timeout"] == 750 + + +def test_remotion_render_preserves_direct_cinematic_scenes(monkeypatch, tmp_path) -> None: + output = tmp_path / "render.mp4" + captured = {} + + def fake_run_command(self, command, **kwargs): + props_arg = next(arg for arg in command if arg.startswith("--props=")) + captured["props"] = json.loads(Path(props_arg.split("=", 1)[1]).read_text()) + output.write_bytes(b"rendered") + + monkeypatch.setattr(VideoCompose, "run_command", fake_run_command) + scene = { + "id": "authored", + "kind": "title", + "text": "Keep me", + "startSeconds": 0, + "durationSeconds": 1, + } + + result = VideoCompose()._remotion_render({ + "composition_data": { + "renderer_family": "cinematic-trailer", + "scenes": [scene], + }, + "output_path": str(output), + }) + + assert result.success, result.error + assert captured["props"]["scenes"] == [scene] diff --git a/tests/tools/test_corpus_builder_total_failure.py b/tests/tools/test_corpus_builder_total_failure.py new file mode 100644 index 00000000..c0180380 --- /dev/null +++ b/tests/tools/test_corpus_builder_total_failure.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass + +import pytest + +import tools.video.stock_sources as stock_sources +from tools.video.corpus_builder import CorpusBuilder + + +@dataclass +class _Candidate: + clip_id: str + + +class _Source: + name = "fake" + + def __init__(self, count: int) -> None: + self.count = count + + def is_available(self) -> bool: + return True + + def search(self, query, filters): + return [_Candidate(f"clip-{index}") for index in range(self.count)] + + +@pytest.fixture +def run_builder(monkeypatch, tmp_path): + def run(count: int, processor): + monkeypatch.setattr(stock_sources, "available_sources", lambda: [_Source(count)]) + monkeypatch.setattr(stock_sources, "source_summary", lambda: {}) + monkeypatch.setattr(CorpusBuilder, "_process_candidate", processor) + return CorpusBuilder().execute({ + "corpus_dir": str(tmp_path / f"corpus-{count}"), + "queries": [{"query": "city at night"}], + "max_new_clips": 50, + }) + + return run + + +def test_all_candidate_failures_fail_closed_with_diagnostics(run_builder) -> None: + def broken_clip_stack(*args, **kwargs): + raise AttributeError("BaseModelOutput has no attribute norm") + + result = run_builder(4, broken_clip_stack) + + assert result.success is False + assert result.data["candidates_seen"] == 4 + assert result.data["clips_failed"] == 4 + assert "corpus index is empty" in result.error + assert "BaseModelOutput" in result.error + + +def test_no_candidates_is_a_valid_empty_search(run_builder) -> None: + result = run_builder(0, lambda *args, **kwargs: None) + + assert result.success is True + assert result.data["candidates_seen"] == 0 diff --git a/tests/tools/test_google_vertex_backends.py b/tests/tools/test_google_vertex_backends.py new file mode 100644 index 00000000..375ba14f --- /dev/null +++ b/tests/tools/test_google_vertex_backends.py @@ -0,0 +1,97 @@ +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +def test_blank_google_location_uses_documented_default(monkeypatch) -> None: + monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "") + from tools.google_credentials import resolve_google_location + + assert resolve_google_location() == "us-central1" + + +def test_google_music_requests_the_global_vertex_location(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("GOOGLE_API_KEY", "test-key") + import tools.google_credentials as credentials + + captured = {} + + def stop_before_network(http_options=None, location=None): + captured["location"] = location + raise RuntimeError("stop before network") + + monkeypatch.setattr(credentials, "get_genai_client", stop_before_network) + + from tools.audio.google_music import GoogleMusic + + result = GoogleMusic().execute({ + "prompt": "solo piano", + "output_path": str(tmp_path / "music.mp3"), + }) + + assert result.success is False + assert captured["location"] == "global" + + +class _VideoAsset: + uri = None + + def __init__(self, video_bytes): + self.video_bytes = video_bytes + + def save(self, path): + Path(path).write_bytes(self.video_bytes or b"") + + +class _Models: + def __init__(self, video_bytes): + self.video_bytes = video_bytes + + def generate_videos(self, **kwargs): + asset = _VideoAsset(self.video_bytes) + generated = SimpleNamespace(video=asset) + response = SimpleNamespace(generated_videos=[generated]) + return SimpleNamespace(done=True, error=None, response=response) + + +class _VertexClient: + vertexai = True + + def __init__(self, video_bytes): + self.models = _Models(video_bytes) + self.files = SimpleNamespace( + download=lambda **kwargs: pytest.fail("Vertex must not call files.download") + ) + + +@pytest.mark.parametrize( + ("video_bytes", "expected_success"), + [(b"VIDEO_BYTES", True), (None, False)], +) +def test_veo_accepts_vertex_and_requires_inline_bytes( + monkeypatch, tmp_path, video_bytes, expected_success +) -> None: + monkeypatch.setenv("GOOGLE_API_KEY", "test-key") + import tools.google_credentials as credentials + + monkeypatch.setattr( + credentials, + "get_genai_client", + lambda http_options=None, location=None: _VertexClient(video_bytes), + ) + + from tools.video.veo_video import VeoVideo + + output = tmp_path / "video.mp4" + result = VeoVideo().execute({ + "backend": "google", + "prompt": "wind moving across grassland", + "output_path": str(output), + }) + + assert result.success is expected_success + if expected_success: + assert output.read_bytes() == b"VIDEO_BYTES" + else: + assert "without inline bytes" in result.error diff --git a/tests/tools/test_hyperframes_compose.py b/tests/tools/test_hyperframes_compose.py index 6438a4fa..d3780fcf 100644 --- a/tests/tools/test_hyperframes_compose.py +++ b/tests/tools/test_hyperframes_compose.py @@ -250,6 +250,11 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch): "_resolve_npm_package", classmethod(lambda cls: {"version": "0.4.5"}), ) + monkeypatch.setattr( + HyperFramesCompose, + "_probe_cli", + classmethod(lambda cls: {"status": "ok"}), + ) rc = HyperFramesCompose()._runtime_check() # Local binaries must still pass for this to go green. if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]: @@ -259,6 +264,31 @@ def test_runtime_check_succeeds_when_npm_resolves(monkeypatch): assert rc["reasons"] == [] +def test_runtime_check_fails_when_published_cli_crashes(monkeypatch): + monkeypatch.setattr( + HyperFramesCompose, + "_resolve_npm_package", + classmethod(lambda cls: {"version": "0.7.89"}), + ) + monkeypatch.setattr( + HyperFramesCompose, + "_probe_cli", + classmethod( + lambda cls: { + "error": 'doctor failed: The "file" argument must be of type string' + } + ), + ) + + rc = HyperFramesCompose()._runtime_check() + + if rc["node_major"] is None or not rc["ffmpeg_available"] or not rc["npx_available"]: + pytest.skip("Local runtime floor not met on this machine") + assert rc["runtime_available"] is False + assert rc["cli_probe_error"] is not None + assert any("not executable" in reason for reason in rc["reasons"]) + + def test_video_compose_render_engines_follow_hyperframes_runtime_check(monkeypatch): """Regression: `video_compose.get_info()['render_engines']['hyperframes']` must track the true availability, not just the local-binary floor. diff --git a/tests/tools/test_mps_device.py b/tests/tools/test_mps_device.py index 844d633e..b3f5fdbd 100644 --- a/tests/tools/test_mps_device.py +++ b/tests/tools/test_mps_device.py @@ -213,8 +213,9 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch): # Build a fake RealESRGANer whose __init__ DOES accept device= class FakeRealESRGANer: - def __init__(self, *, scale, model_path, model, dni_weight, half, device=None): + def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10, device=None): self.called_with_device = device + self.called_with_tile = tile fake_realesrganer_cls = FakeRealESRGANer monkeypatch.setitem(sys.modules, "torch", fake_torch) @@ -239,6 +240,7 @@ def test_upscale_build_upsampler_uses_signature_guard(monkeypatch): tool = upscale.Upscale() result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False) assert result.called_with_device == "device(mps)" + assert result.called_with_tile == 256 def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch): @@ -252,8 +254,9 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch): # Build a fake RealESRGANer whose __init__ does NOT accept device= class FakeRealESRGANerNoDevice: - def __init__(self, *, scale, model_path, model, dni_weight, half): + def __init__(self, *, scale, model_path, model, dni_weight, half, tile=0, tile_pad=10): self.called_with_device = None # no device param + self.called_with_tile = tile fake_realesrganer_cls = FakeRealESRGANerNoDevice monkeypatch.setitem(sys.modules, "torch", fake_torch) @@ -279,6 +282,7 @@ def test_upscale_build_upsampler_skips_device_when_unsupported(monkeypatch): # Should NOT raise TypeError about unexpected keyword argument 'device' result = tool._build_upsampler(scale=4, model_name="RealESRGAN_x4plus", denoise_strength=0.5, face_enhance=False) assert result.called_with_device is None + assert result.called_with_tile == 256 # ------------------------------------------------------------------ diff --git a/tests/tools/test_remotion_audio_mux.py b/tests/tools/test_remotion_audio_mux.py new file mode 100644 index 00000000..210129b8 --- /dev/null +++ b/tests/tools/test_remotion_audio_mux.py @@ -0,0 +1,59 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +from tools.video.video_compose import VideoCompose + + +pytestmark = pytest.mark.skipif( + shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, + reason="ffmpeg and ffprobe are required", +) + + +def _run(command: list[str]) -> None: + subprocess.run(command, capture_output=True, check=True, timeout=30) + + +def test_external_audio_mux_adds_audible_stream_without_changing_video_length(tmp_path) -> None: + video = tmp_path / "video.mp4" + audio = tmp_path / "audio.wav" + _run([ + "ffmpeg", "-y", "-f", "lavfi", "-i", + "color=c=red:s=320x180:d=2:r=30", + "-c:v", "libx264", "-pix_fmt", "yuv420p", str(video), + ]) + _run([ + "ffmpeg", "-y", "-f", "lavfi", "-i", + "sine=frequency=440:duration=1", str(audio), + ]) + + result = VideoCompose()._mux_external_audio(video, audio) + + assert result.success, result.error + streams = subprocess.run( + [ + "ffprobe", "-v", "error", "-show_entries", + "stream=codec_type", "-of", "csv=p=0", str(video), + ], + capture_output=True, + check=True, + text=True, + timeout=30, + ).stdout.splitlines() + duration = float(subprocess.run( + [ + "ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "csv=p=0", str(video), + ], + capture_output=True, + check=True, + text=True, + timeout=30, + ).stdout.strip()) + + assert "video" in streams + assert "audio" in streams + assert duration == pytest.approx(2.0, abs=0.15) diff --git a/tests/tools/test_transcriber_device_selection.py b/tests/tools/test_transcriber_device_selection.py new file mode 100644 index 00000000..11d50012 --- /dev/null +++ b/tests/tools/test_transcriber_device_selection.py @@ -0,0 +1,83 @@ +import sys +from types import SimpleNamespace + +from tools.analysis.transcriber import Transcriber + + +class _Info: + language = "en" + duration = 1.0 + + +def test_transcriber_uses_ctranslate2_cuda_without_torch(monkeypatch, tmp_path) -> None: + devices = [] + + class FakeWhisperModel: + def __init__(self, model_size, *, device, compute_type): + devices.append((device, compute_type)) + + def transcribe(self, *args, **kwargs): + return iter(()), _Info() + + monkeypatch.setitem( + sys.modules, + "faster_whisper", + SimpleNamespace(WhisperModel=FakeWhisperModel), + ) + monkeypatch.setitem( + sys.modules, + "ctranslate2", + SimpleNamespace( + get_cuda_device_count=lambda: 1, + get_supported_compute_types=lambda device: {"float16", "float32"}, + ), + ) + input_path = tmp_path / "audio.wav" + input_path.write_bytes(b"fake") + + result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)}) + + assert result.success, result.error + assert devices == [("cuda", "float16")] + assert result.data["device"] == "cuda" + + +def test_transcriber_falls_back_when_cuda_fails_during_iteration(monkeypatch, tmp_path) -> None: + devices = [] + + class FakeWhisperModel: + def __init__(self, model_size, *, device, compute_type): + self.device = device + devices.append((device, compute_type)) + + def transcribe(self, *args, **kwargs): + if self.device == "cuda": + def broken_iterator(): + raise RuntimeError("cublas64_12.dll not found") + yield + + return broken_iterator(), _Info() + return iter(()), _Info() + + monkeypatch.setitem( + sys.modules, + "faster_whisper", + SimpleNamespace(WhisperModel=FakeWhisperModel), + ) + monkeypatch.setitem( + sys.modules, + "ctranslate2", + SimpleNamespace( + get_cuda_device_count=lambda: 1, + get_supported_compute_types=lambda device: {"float16"}, + ), + ) + input_path = tmp_path / "audio.wav" + input_path.write_bytes(b"fake") + + result = Transcriber().execute({"input_path": str(input_path), "output_dir": str(tmp_path)}) + + assert result.success, result.error + assert devices == [("cuda", "float16"), ("cpu", "int8")] + assert result.data["device"] == "cpu" + assert "cublas64_12.dll" in result.data["gpu_fallback_reason"] diff --git a/tools/analysis/transcriber.py b/tools/analysis/transcriber.py index bb309fc5..8e384152 100644 --- a/tools/analysis/transcriber.py +++ b/tools/analysis/transcriber.py @@ -135,50 +135,76 @@ class Transcriber(BaseTool): start = time.time() - # Load model (CPU by default, CUDA if available) + # faster-whisper executes through CTranslate2, so that runtime—not + # PyTorch—is authoritative for CUDA availability and compute types. + device = "cpu" + compute_type = "int8" try: - import torch - device = "cuda" if torch.cuda.is_available() else "cpu" - compute_type = "float16" if device == "cuda" else "int8" - except ImportError: + import ctranslate2 + + if ctranslate2.get_cuda_device_count() > 0: + supported = ctranslate2.get_supported_compute_types("cuda") + for candidate in ("float16", "int8_float16", "float32"): + if candidate in supported: + device = "cuda" + compute_type = candidate + break + except Exception: + # Probing is advisory. CPU remains a safe deterministic baseline. + pass + + def _transcribe_on(selected_device: str, selected_compute_type: str): + model = WhisperModel( + model_size, + device=selected_device, + compute_type=selected_compute_type, + ) + segments_iter, transcription_info = model.transcribe( + str(input_path), + language=language, + word_timestamps=True, + vad_filter=True, + ) + + parsed_segments = [] + parsed_words = [] + # faster-whisper evaluates lazily. Draining the iterator here keeps + # missing CUDA runtime libraries inside the fallback boundary. + for seg in segments_iter: + seg_data = { + "id": seg.id, + "start": round(seg.start, 3), + "end": round(seg.end, 3), + "text": seg.text.strip(), + } + + if seg.words: + words = [] + for word in seg.words: + word_entry = { + "word": word.word, + "start": round(word.start, 3), + "end": round(word.end, 3), + "probability": round(word.probability, 3), + } + words.append(word_entry) + parsed_words.append(word_entry) + seg_data["words"] = words + + parsed_segments.append(seg_data) + + return parsed_segments, parsed_words, transcription_info + + gpu_fallback_reason = None + try: + segments, word_timestamps, info = _transcribe_on(device, compute_type) + except Exception as exc: + if device == "cpu": + raise + gpu_fallback_reason = f"{type(exc).__name__}: {exc}" device = "cpu" compute_type = "int8" - - model = WhisperModel(model_size, device=device, compute_type=compute_type) - - # Transcribe - segments_iter, info = model.transcribe( - str(input_path), - language=language, - word_timestamps=True, - vad_filter=True, - ) - - segments = [] - word_timestamps = [] - - for seg in segments_iter: - seg_data = { - "id": seg.id, - "start": round(seg.start, 3), - "end": round(seg.end, 3), - "text": seg.text.strip(), - } - - if seg.words: - words = [] - for w in seg.words: - word_entry = { - "word": w.word, - "start": round(w.start, 3), - "end": round(w.end, 3), - "probability": round(w.probability, 3), - } - words.append(word_entry) - word_timestamps.append(word_entry) - seg_data["words"] = words - - segments.append(seg_data) + segments, word_timestamps, info = _transcribe_on(device, compute_type) detected_language = language or info.language duration = info.duration @@ -198,6 +224,8 @@ class Transcriber(BaseTool): "duration_seconds": round(duration, 3), "model_size": model_size, "device": device, + "compute_type": compute_type, + "gpu_fallback_reason": gpu_fallback_reason, } # Write transcript JSON diff --git a/tools/audio/audio_mixer.py b/tools/audio/audio_mixer.py index 189a153a..48dfcb6a 100644 --- a/tools/audio/audio_mixer.py +++ b/tools/audio/audio_mixer.py @@ -184,6 +184,14 @@ class AudioMixer(BaseTool): "default": 0.5, "description": "Duration of fade in/out at segment boundaries (seconds).", }, + "target_duration": { + "type": "number", + "exclusiveMinimum": 0, + "description": ( + "full_mix only. Exact output length in seconds. Pads a short " + "mix and trims a long mix so audio matches the composition." + ), + }, }, } @@ -500,6 +508,15 @@ class AudioMixer(BaseTool): output_path.parent.mkdir(parents=True, exist_ok=True) normalize = inputs.get("normalize", True) ducking = inputs.get("ducking", {"enabled": True}) + target_duration = inputs.get("target_duration") + target: float | None = None + if target_duration is not None: + try: + target = float(target_duration) + except (TypeError, ValueError): + return ToolResult(success=False, error="target_duration must be a positive number") + if target <= 0: + return ToolResult(success=False, error="target_duration must be greater than zero") speech_tracks = [t for t in tracks if t.get("role") in ("speech", "primary")] music_tracks = [t for t in tracks if t.get("role") in ("music", "secondary")] @@ -547,7 +564,14 @@ class AudioMixer(BaseTool): ) else: filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_all]") - filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]") + if target is not None: + filter_parts.append("[speech_all]asplit=2[speech_key_raw][speech_out]") + filter_parts.append( + f"[speech_key_raw]apad=whole_dur={target}," + f"atrim=duration={target},asetpts=PTS-STARTPTS[speech_key]" + ) + else: + filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]") # Mix music tracks together music_start = len(speech_tracks) @@ -596,19 +620,34 @@ class AudioMixer(BaseTool): f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]" ) + # A ducked music stream is gated by the speech sidechain, so its tail + # can disappear when narration ends. If the caller knows the video + # duration, make that the authoritative mix length before loudness + # normalization: apad extends short audio and atrim caps long audio. + premix_label = "premix" + if target is not None: + filter_parts.append( + f"[premix]apad=whole_dur={target},atrim=duration={target}," + "asetpts=PTS-STARTPTS[premix_duration]" + ) + premix_label = "premix_duration" + # Normalize if normalize: - filter_parts.append(self._loudnorm_filter(inputs, "premix", "out")) + filter_parts.append(self._loudnorm_filter(inputs, premix_label, "out")) out_label = "[out]" else: - out_label = "[premix]" + out_label = f"[{premix_label}]" filter_complex = ";".join(p for p in filter_parts if p) cmd = ["ffmpeg", "-y"] cmd.extend(input_args) cmd.extend(["-filter_complex", filter_complex]) - cmd.extend(["-map", out_label, str(output_path)]) + cmd.extend(["-map", out_label]) + if target is not None: + cmd.extend(["-t", str(target)]) + cmd.append(str(output_path)) self.run_command(cmd) @@ -621,6 +660,7 @@ class AudioMixer(BaseTool): "sfx_tracks": len(sfx_tracks), "ducking_enabled": duck_enabled, "normalized": normalize, + "target_duration": target_duration, "output": str(output_path), }, artifacts=[str(output_path)], diff --git a/tools/audio/google_music.py b/tools/audio/google_music.py index 891550c1..67f25d87 100644 --- a/tools/audio/google_music.py +++ b/tools/audio/google_music.py @@ -146,7 +146,8 @@ class GoogleMusic(BaseTool): from tools.google_credentials import get_genai_client, GOOGLE_API_TIMEOUT_MS http_options = types.HttpOptions(timeout=GOOGLE_API_TIMEOUT_MS) - client = get_genai_client(http_options=http_options) + # Lyria 3 is served only from Vertex's global location. + client = get_genai_client(http_options=http_options, location="global") except ImportError as e: return ToolResult( success=False, diff --git a/tools/enhancement/bg_remove.py b/tools/enhancement/bg_remove.py index 7802f45a..7d1c40e8 100644 --- a/tools/enhancement/bg_remove.py +++ b/tools/enhancement/bg_remove.py @@ -130,7 +130,7 @@ class BgRemove(BaseTool): result_image = rembg.remove( input_image, - model_name=model_name, + session=rembg.new_session(model_name), alpha_matting=alpha_matting, ) diff --git a/tools/enhancement/upscale.py b/tools/enhancement/upscale.py index 03648405..1fa18ccd 100644 --- a/tools/enhancement/upscale.py +++ b/tools/enhancement/upscale.py @@ -297,6 +297,11 @@ class Upscale(BaseTool): "model": model, "dni_weight": denoise_strength, "half": half, + # Full-frame x4 inference can terminate the process on low-memory + # CPU/MPS hosts before Python can raise an exception. Bound the + # working set there; keep CUDA on the faster single-pass path. + "tile": 0 if _device == "cuda" else 256, + "tile_pad": 10, } # Guard: only pass device= if the installed version accepts it if "device" in inspect.signature(RealESRGANer.__init__).parameters: diff --git a/tools/google_credentials.py b/tools/google_credentials.py index 71158c73..77565c2e 100644 --- a/tools/google_credentials.py +++ b/tools/google_credentials.py @@ -18,6 +18,12 @@ from typing import Any # Broad scope that covers Cloud Text-to-Speech and Vertex AI prediction. CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform" + +def resolve_google_location(location: str | None = None) -> str: + """Return a Vertex location, treating blank env values as unset.""" + + return location or os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1" + # Shared constants for long-running Google/Vertex AI generation calls (e.g. music, video) GOOGLE_API_TIMEOUT_SECONDS = 600 GOOGLE_API_TIMEOUT_MS = GOOGLE_API_TIMEOUT_SECONDS * 1000 @@ -38,8 +44,15 @@ def has_google_credentials() -> bool: ) -def get_genai_client(http_options: Any | None = None) -> Any: - """Lazily import and initialize the Google GenAI Client based on configured credentials.""" +def get_genai_client( + http_options: Any | None = None, + location: str | None = None, +) -> Any: + """Initialize Google GenAI using the configured credential mode. + + ``location`` overrides the Vertex region for globally hosted models. It is + deliberately ignored by the API-key backend, which has no region setting. + """ from google import genai api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") @@ -51,7 +64,7 @@ def get_genai_client(http_options: Any | None = None) -> Any: if use_vertex or (not api_key and service_account_configured()): kwargs = { "vertexai": True, - "location": os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1"), + "location": resolve_google_location(location), "http_options": http_options, } project_id = resolve_project_id() diff --git a/tools/graphics/google_imagen.py b/tools/graphics/google_imagen.py index ac2717ba..2850a8e3 100644 --- a/tools/graphics/google_imagen.py +++ b/tools/graphics/google_imagen.py @@ -22,6 +22,7 @@ from tools.base_tool import ( ) from tools.google_credentials import ( get_access_token, + resolve_google_location, resolve_project_id, service_account_configured, has_google_credentials, @@ -241,7 +242,7 @@ class GoogleImagen(BaseTool): } if bearer_token: - location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1") + location = resolve_google_location() url = ( f"https://{location}-aiplatform.googleapis.com/v1/projects/" f"{project_id}/locations/{location}/publishers/google/models/" diff --git a/tools/video/corpus_builder.py b/tools/video/corpus_builder.py index f426b40c..cb52ff6a 100644 --- a/tools/video/corpus_builder.py +++ b/tools/video/corpus_builder.py @@ -392,6 +392,38 @@ class CorpusBuilder(BaseTool): except Exception as e: cache_snapshot = {"error": f"{type(e).__name__}: {e}"} + # Per-candidate tolerance is useful only while at least one item + # survives. If every discovered candidate fails, reporting success + # persists an empty index and hides a systemic codec/CLIP failure + # until retrieval. A no-result or skip-only run remains valid. + total_failure = bool(candidates_seen) and not added_ids and not skipped + if total_failure: + first_errors = "; ".join( + item["error"] + for item in errors + if item.get("phase") == "process" + )[:400] + return ToolResult( + success=False, + error=( + f"All {failed} of {candidates_seen} candidates failed to " + "process; corpus index is empty. Check the media decoder " + "and the CLIP `transformers`/`torch` compatibility. " + f"First errors: {first_errors or '(none recorded)'}" + ), + data={ + "corpus_dir": str(corpus_dir), + "queries_run": len(queries), + "candidates_seen": candidates_seen, + "clips_added": 0, + "clips_skipped_existing": skipped, + "clips_failed": failed, + "total_corpus_size": len(corp), + "errors": errors[:25], + }, + duration_seconds=round(elapsed, 2), + ) + return ToolResult( success=True, data={ diff --git a/tools/video/hyperframes_compose.py b/tools/video/hyperframes_compose.py index e6c798c8..7511b686 100644 --- a/tools/video/hyperframes_compose.py +++ b/tools/video/hyperframes_compose.py @@ -226,6 +226,7 @@ class HyperFramesCompose(BaseTool): # We cache per-process so the first call pays ~2-5s and subsequent calls # (get_info spam from the registry) are free. _npm_resolve_cache: Optional[dict[str, str]] = None + _cli_probe_cache: Optional[dict[str, str]] = None @classmethod def _node_major_version(cls) -> Optional[int]: @@ -301,6 +302,45 @@ class HyperFramesCompose(BaseTool): cls._npm_resolve_cache = {"version": version} return cls._npm_resolve_cache + @classmethod + def _probe_cli(cls) -> dict[str, str]: + """Run the published CLI's doctor command once per process. + + Package resolution alone does not prove that the executable can start: + an upstream packaging regression can publish successfully while every + CLI command crashes during bootstrap. Provider preflight must not call + that state available. + """ + if cls._cli_probe_cache is not None: + return cls._cli_probe_cache + + npx = shutil.which("npx") + if not npx: + cls._cli_probe_cache = {"error": "npx not on PATH"} + return cls._cli_probe_cache + + try: + proc = subprocess.run( + [npx, "--yes", cls._NPM_PACKAGE, "doctor", "--json"], + capture_output=True, + text=True, + timeout=20, + ) + except subprocess.TimeoutExpired: + cls._cli_probe_cache = {"error": "doctor timed out after 20s"} + return cls._cli_probe_cache + except (OSError, subprocess.SubprocessError) as exc: + cls._cli_probe_cache = {"error": f"doctor failed: {type(exc).__name__}"} + return cls._cli_probe_cache + + if proc.returncode != 0: + output = "\n".join(filter(None, [proc.stderr, proc.stdout])).strip() + tail = output.splitlines()[-1][:200] if output else f"exit {proc.returncode}" + cls._cli_probe_cache = {"error": f"doctor failed: {tail}"} + else: + cls._cli_probe_cache = {"status": "ok"} + return cls._cli_probe_cache + def _runtime_check(self) -> dict[str, Any]: """Return availability state for the HyperFrames runtime. @@ -336,6 +376,12 @@ class HyperFramesCompose(BaseTool): f"{npm_resolve['error']}" ) + cli_probe: dict[str, str] = {} + if not reasons: + cli_probe = self._probe_cli() + if "error" in cli_probe: + reasons.append(f"published CLI is not executable: {cli_probe['error']}") + return { "runtime_available": not reasons, "node_major": node_major, @@ -344,6 +390,8 @@ class HyperFramesCompose(BaseTool): "npm_package": self._NPM_PACKAGE, "npm_package_version": npm_resolve.get("version"), "npm_resolve_error": npm_resolve.get("error"), + "cli_probe_status": cli_probe.get("status"), + "cli_probe_error": cli_probe.get("error"), "reasons": reasons, } diff --git a/tools/video/veo_video.py b/tools/video/veo_video.py index de4efea2..11606d7b 100644 --- a/tools/video/veo_video.py +++ b/tools/video/veo_video.py @@ -308,13 +308,6 @@ class VeoVideo(BaseTool): client._api_client, "vertexai", False ) - if is_vertex: - return ToolResult( - success=False, - error="Google Veo video generation via google-genai is only supported using the Gemini Developer API (API key) backend. " - "Please configure GEMINI_API_KEY/GOOGLE_API_KEY or use the FAL.ai backend.", - ) - prompt = inputs["prompt"] operation = inputs.get("operation", "text_to_video") model_variant = inputs.get("model_variant", "veo3.1") @@ -501,7 +494,19 @@ class VeoVideo(BaseTool): success=False, error="No video asset returned in the response.", ) - client.files.download(file=video_asset) + if not is_vertex: + # The Files service is a Gemini Developer API feature. Vertex + # returns bytes inline when no GCS output URI is requested. + client.files.download(file=video_asset) + elif not getattr(video_asset, "video_bytes", None): + return ToolResult( + success=False, + error=( + "Vertex AI returned a video without inline bytes " + f"(uri={getattr(video_asset, 'uri', None)!r}). Configure " + "the request without an output GCS URI so bytes are returned inline." + ), + ) output_path = Path(inputs.get("output_path", "veo_output.mp4")) output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/tools/video/video_compose.py b/tools/video/video_compose.py index 976a2e55..6fd69ef4 100644 --- a/tools/video/video_compose.py +++ b/tools/video/video_compose.py @@ -31,11 +31,14 @@ the agent to re-ask the user rather than substituting a different engine. from __future__ import annotations import json +import hashlib import logging +import shutil import subprocess import time from pathlib import Path from typing import Any, Optional +from urllib.parse import unquote, urlsplit from tools.base_tool import ( BaseTool, @@ -387,6 +390,49 @@ class VideoCompose(BaseTool): except Exception: return False + def _mux_external_audio(self, video_path: Path, audio_path: str | Path) -> ToolResult: + """Atomically replace a rendered video's audio with the approved mix.""" + + audio = Path(audio_path).resolve() + if not audio.is_file(): + return ToolResult(success=False, error=f"Mixed audio not found: {audio}") + + temp_output = video_path.with_name( + f".{video_path.stem}.audio-mux-{time.time_ns()}{video_path.suffix}" + ) + try: + self.run_command([ + "ffmpeg", "-y", + "-i", str(video_path), + "-i", str(audio), + "-map", "0:v:0", + "-map", "1:a:0", + "-c:v", "copy", + "-c:a", "aac", + "-b:a", "192k", + "-af", "apad", + "-shortest", + "-movflags", "+faststart", + str(temp_output), + ]) + if not temp_output.is_file(): + return ToolResult( + success=False, + error=f"Audio mux completed but output file is missing: {temp_output}", + ) + temp_output.replace(video_path) + except Exception as exc: + return ToolResult(success=False, error=f"Could not mux mixed audio: {exc}") + finally: + if temp_output.exists(): + temp_output.unlink() + + return ToolResult( + success=True, + data={"output": str(video_path), "has_mixed_audio": True}, + artifacts=[str(video_path)], + ) + def _compose(self, inputs: dict[str, Any]) -> ToolResult: """FFmpeg composition: concat video cuts, add audio, burn subtitles. @@ -714,6 +760,124 @@ class VideoCompose(BaseTool): ) return comp + @staticmethod + def _cuts_to_cinematic_scenes(cuts: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Adapt canonical sequential cuts to CinematicRenderer's scene contract.""" + + scenes: list[dict[str, Any]] = [] + timeline_cursor = 0.0 + hard_transitions = {"cut", "none"} + title_types = {"hero_title", "text_card", "title"} + + for index, cut in enumerate(cuts): + try: + source_in = float(cut.get("in_seconds", 0)) + source_out = float(cut.get("out_seconds", source_in)) + speed = max(float(cut.get("speed", 1.0)), 0.1) + except (TypeError, ValueError): + continue + duration = max(0.0, (source_out - source_in) / speed) + if duration <= 0: + continue + + scene_id = str(cut.get("id") or f"cut-{index + 1}") + source = str(cut.get("source") or "") + cut_type = str(cut.get("type") or "").lower() + common = { + "id": scene_id, + "startSeconds": timeline_cursor, + "durationSeconds": duration, + } + + if cut_type in title_types or not source: + scene: dict[str, Any] = { + **common, + "kind": "title", + "text": str( + cut.get("text") + or cut.get("title") + or cut.get("reason") + or scene_id + ), + } + if source: + scene["backgroundSrc"] = source + scene["backgroundTrimBeforeSeconds"] = source_in + scene["backgroundTrimAfterSeconds"] = source_out + else: + scene = { + **common, + "kind": "video", + "src": source, + "trimBeforeSeconds": source_in, + "trimAfterSeconds": source_out, + "playbackRate": speed, + } + if str(cut.get("transition_in") or "").lower() in hard_transitions: + scene["fadeInFrames"] = 0 + if str(cut.get("transition_out") or "").lower() in hard_transitions: + scene["fadeOutFrames"] = 0 + + scenes.append(scene) + timeline_cursor += duration + + return scenes + + @staticmethod + def _stage_remotion_media(value: Any, public_dir: Path) -> int: + """Copy local media references into a Remotion public dir in-place. + + OffthreadVideo's compositor rejects ``file://`` sources. Rewriting + staged files to relative ``staticFile()`` paths works for video and + image components on every platform. + """ + + staged_by_source: dict[Path, str] = {} + media_keys = {"source", "src", "backgroundSrc"} + + def visit(node: Any, parent_key: str | None = None) -> Any: + if isinstance(node, dict): + for key, child in list(node.items()): + node[key] = visit(child, key) + return node + if isinstance(node, list): + for index, child in enumerate(node): + node[index] = visit(child, parent_key) + return node + if not isinstance(node, str) or parent_key not in media_keys: + return node + if node.startswith(("http://", "https://", "data:")): + return node + + if node.lower().startswith("file://"): + parsed = urlsplit(node) + decoded_path = unquote(parsed.path) + if len(parsed.netloc) == 2 and parsed.netloc[1] == ":": + raw_path = f"{parsed.netloc}{decoded_path}" + elif parsed.netloc and parsed.netloc.lower() != "localhost": + raw_path = f"//{parsed.netloc}{decoded_path}" + else: + raw_path = decoded_path + # Standard Windows file URIs use file:///C:/...; pathlib on + # Windows needs the drive path without the URI's leading slash. + if len(raw_path) >= 3 and raw_path[0] == "/" and raw_path[2] == ":": + raw_path = raw_path[1:] + else: + raw_path = node + source = Path(raw_path).resolve() + if not source.is_file(): + return node + if source not in staged_by_source: + digest = hashlib.sha256(str(source).encode("utf-8")).hexdigest()[:12] + name = f"{digest}-{source.name}" + public_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, public_dir / name) + staged_by_source[source] = name + return staged_by_source[source] + + visit(value) + return len(staged_by_source) + def _render_via_atelier( self, inputs: dict[str, Any], @@ -840,6 +1004,11 @@ class VideoCompose(BaseTool): error=f"Atelier render completed but output file missing: {output_path}", ) + if inputs.get("audio_path"): + mux_result = self._mux_external_audio(output_path, inputs["audio_path"]) + if not mux_result.success: + return mux_result + # --- Atelier post-render review ------------------------------------- # The cut-schema paths run _run_final_review (technical/visual/audio # probes + transcript-vs-script). Atelier MUST do the same so hero @@ -1063,8 +1232,12 @@ class VideoCompose(BaseTool): try: from styles.playbook_loader import load_playbook playbook = load_playbook(playbook_name) - except Exception: - pass + except Exception as exc: + logging.getLogger(__name__).warning( + "Could not load style playbook %r for Remotion theme: %s", + playbook_name, + exc, + ) if playbook: vl = playbook.get("visual_language", {}) @@ -1418,6 +1591,8 @@ class VideoCompose(BaseTool): # would only take effect on a direct _remotion_render() call. if inputs.get("remotion_timeout_ms") is not None: remotion_inputs["remotion_timeout_ms"] = inputs["remotion_timeout_ms"] + if inputs.get("public_dir") is not None: + remotion_inputs["public_dir"] = inputs["public_dir"] render_result = self._remotion_render(remotion_inputs) # Governance: NEVER silently fall back to FFmpeg when Remotion fails. @@ -1437,6 +1612,11 @@ class VideoCompose(BaseTool): f"Per governance: renderer downgrade requires user approval." ), ) + if inputs.get("audio_path"): + mux_result = self._mux_external_audio(output_path, inputs["audio_path"]) + if not mux_result.success: + return mux_result + render_result.data["has_mixed_audio"] = True else: # --- FFmpeg fallback: only when Remotion is unavailable --- options = inputs.get("options", {}) @@ -1546,7 +1726,12 @@ class VideoCompose(BaseTool): try: from styles.playbook_loader import load_playbook # type: ignore playbook_data = load_playbook(playbook_name) - except Exception: + except Exception as exc: + logging.getLogger(__name__).warning( + "Could not load style playbook %r for HyperFrames bridge: %s", + playbook_name, + exc, + ) playbook_data = None hf_inputs: dict[str, Any] = { @@ -1677,8 +1862,6 @@ class VideoCompose(BaseTool): types, and transitions using React-based frame-accurate rendering. Accepts edit_decisions (with resolved file paths) or raw composition_data. """ - import shutil - if not shutil.which("npx"): return ToolResult( success=False, @@ -1700,16 +1883,6 @@ class VideoCompose(BaseTool): # Deep-copy props so we don't mutate the original props = json.loads(json.dumps(composition_data)) - # Convert absolute file paths to file:// URIs for Remotion's - # Img and OffthreadVideo components - for cut in props.get("cuts", []): - source = cut.get("source", "") - if source and not source.startswith(("http://", "https://", "file://")): - resolved = Path(source).resolve() - if resolved.exists(): - posix = resolved.as_posix() - cut["source"] = f"file:///{posix}" if not posix.startswith("/") else f"file://{posix}" - # Build a custom themeConfig from the playbook's actual colors. # This ensures every video gets a unique visual identity derived # from its production decisions — not picked from a preset menu. @@ -1723,11 +1896,6 @@ class VideoCompose(BaseTool): if theme_config: props["themeConfig"] = theme_config - # Write props to temp file for Remotion CLI - props_path = output_path.parent / ".remotion_props.json" - with open(props_path, "w", encoding="utf-8") as f: - json.dump(props, f) - # remotion-composer lives at project root composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer" if not composer_dir.exists(): @@ -1741,6 +1909,39 @@ class VideoCompose(BaseTool): renderer_family = (composition_data or {}).get("renderer_family", "explainer-data") composition_id = self._get_composition_id(renderer_family) + if composition_id == "CinematicRenderer": + if not props.get("scenes") and props.get("cuts"): + props["scenes"] = self._cuts_to_cinematic_scenes(props["cuts"]) + props.pop("cuts", None) + if not props.get("scenes"): + return ToolResult( + success=False, + error="CinematicRenderer received cuts but none could be adapted into scenes.", + ) + + requested_public_dir = inputs.get("public_dir") + cleanup_public_dir = False + public_dir: Path | None = None + if requested_public_dir: + public_dir = Path(requested_public_dir).resolve() + if not public_dir.is_dir(): + return ToolResult( + success=False, + error=f"Remotion public_dir does not exist or is not a directory: {public_dir}", + ) + else: + public_dir = output_path.parent / f".remotion-public-{output_path.stem}" + cleanup_public_dir = True + + staged_count = self._stage_remotion_media(props, public_dir) + if not staged_count and cleanup_public_dir: + public_dir = None + + # Write the fully adapted/staged props, never the original cut payload. + props_path = output_path.parent / ".remotion_props.json" + with open(props_path, "w", encoding="utf-8") as f: + json.dump(props, f) + cmd = [ "npx", "remotion", "render", str(composer_dir / "src" / "index.tsx"), @@ -1753,6 +1954,8 @@ class VideoCompose(BaseTool): # API Remotion recommends for file paths and is cross-platform safe. f"--props={props_path}", ] + if public_dir is not None: + cmd.append(f"--public-dir={public_dir}") # Apply media profile dimensions profile_name = inputs.get("profile") @@ -1770,7 +1973,8 @@ class VideoCompose(BaseTool): # opaque failure. Pass it through and give the subprocess enough headroom # so run_command() does not kill Remotion before its own timeout fires. remotion_timeout_ms = inputs.get("remotion_timeout_ms") - subprocess_timeout = 600 + scene_count = len(props.get("scenes") or props.get("cuts") or []) + subprocess_timeout = max(600, scene_count * 15) if remotion_timeout_ms: try: ms = int(remotion_timeout_ms) @@ -1808,6 +2012,8 @@ class VideoCompose(BaseTool): finally: if props_path.exists(): props_path.unlink() + if cleanup_public_dir and public_dir is not None and public_dir.exists(): + shutil.rmtree(public_dir, ignore_errors=True) if not output_path.exists(): return ToolResult( @@ -1821,6 +2027,7 @@ class VideoCompose(BaseTool): "operation": "remotion_render", "output": str(output_path), "profile": profile_name, + "staged_media_count": staged_count, }, artifacts=[str(output_path)], )