mirror of
https://github.com/calesthio/OpenMontage.git
synced 2026-08-05 15:20:40 +08:00
fix: recover bounded defects from PR backlog
This commit is contained in:
@@ -135,50 +135,76 @@ class Transcriber(BaseTool):
|
||||
|
||||
start = time.time()
|
||||
|
||||
# Load model (CPU by default, CUDA if available)
|
||||
# faster-whisper executes through CTranslate2, so that runtime—not
|
||||
# PyTorch—is authoritative for CUDA availability and compute types.
|
||||
device = "cpu"
|
||||
compute_type = "int8"
|
||||
try:
|
||||
import torch
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
except ImportError:
|
||||
import ctranslate2
|
||||
|
||||
if ctranslate2.get_cuda_device_count() > 0:
|
||||
supported = ctranslate2.get_supported_compute_types("cuda")
|
||||
for candidate in ("float16", "int8_float16", "float32"):
|
||||
if candidate in supported:
|
||||
device = "cuda"
|
||||
compute_type = candidate
|
||||
break
|
||||
except Exception:
|
||||
# Probing is advisory. CPU remains a safe deterministic baseline.
|
||||
pass
|
||||
|
||||
def _transcribe_on(selected_device: str, selected_compute_type: str):
|
||||
model = WhisperModel(
|
||||
model_size,
|
||||
device=selected_device,
|
||||
compute_type=selected_compute_type,
|
||||
)
|
||||
segments_iter, transcription_info = model.transcribe(
|
||||
str(input_path),
|
||||
language=language,
|
||||
word_timestamps=True,
|
||||
vad_filter=True,
|
||||
)
|
||||
|
||||
parsed_segments = []
|
||||
parsed_words = []
|
||||
# faster-whisper evaluates lazily. Draining the iterator here keeps
|
||||
# missing CUDA runtime libraries inside the fallback boundary.
|
||||
for seg in segments_iter:
|
||||
seg_data = {
|
||||
"id": seg.id,
|
||||
"start": round(seg.start, 3),
|
||||
"end": round(seg.end, 3),
|
||||
"text": seg.text.strip(),
|
||||
}
|
||||
|
||||
if seg.words:
|
||||
words = []
|
||||
for word in seg.words:
|
||||
word_entry = {
|
||||
"word": word.word,
|
||||
"start": round(word.start, 3),
|
||||
"end": round(word.end, 3),
|
||||
"probability": round(word.probability, 3),
|
||||
}
|
||||
words.append(word_entry)
|
||||
parsed_words.append(word_entry)
|
||||
seg_data["words"] = words
|
||||
|
||||
parsed_segments.append(seg_data)
|
||||
|
||||
return parsed_segments, parsed_words, transcription_info
|
||||
|
||||
gpu_fallback_reason = None
|
||||
try:
|
||||
segments, word_timestamps, info = _transcribe_on(device, compute_type)
|
||||
except Exception as exc:
|
||||
if device == "cpu":
|
||||
raise
|
||||
gpu_fallback_reason = f"{type(exc).__name__}: {exc}"
|
||||
device = "cpu"
|
||||
compute_type = "int8"
|
||||
|
||||
model = WhisperModel(model_size, device=device, compute_type=compute_type)
|
||||
|
||||
# Transcribe
|
||||
segments_iter, info = model.transcribe(
|
||||
str(input_path),
|
||||
language=language,
|
||||
word_timestamps=True,
|
||||
vad_filter=True,
|
||||
)
|
||||
|
||||
segments = []
|
||||
word_timestamps = []
|
||||
|
||||
for seg in segments_iter:
|
||||
seg_data = {
|
||||
"id": seg.id,
|
||||
"start": round(seg.start, 3),
|
||||
"end": round(seg.end, 3),
|
||||
"text": seg.text.strip(),
|
||||
}
|
||||
|
||||
if seg.words:
|
||||
words = []
|
||||
for w in seg.words:
|
||||
word_entry = {
|
||||
"word": w.word,
|
||||
"start": round(w.start, 3),
|
||||
"end": round(w.end, 3),
|
||||
"probability": round(w.probability, 3),
|
||||
}
|
||||
words.append(word_entry)
|
||||
word_timestamps.append(word_entry)
|
||||
seg_data["words"] = words
|
||||
|
||||
segments.append(seg_data)
|
||||
segments, word_timestamps, info = _transcribe_on(device, compute_type)
|
||||
|
||||
detected_language = language or info.language
|
||||
duration = info.duration
|
||||
@@ -198,6 +224,8 @@ class Transcriber(BaseTool):
|
||||
"duration_seconds": round(duration, 3),
|
||||
"model_size": model_size,
|
||||
"device": device,
|
||||
"compute_type": compute_type,
|
||||
"gpu_fallback_reason": gpu_fallback_reason,
|
||||
}
|
||||
|
||||
# Write transcript JSON
|
||||
|
||||
@@ -184,6 +184,14 @@ class AudioMixer(BaseTool):
|
||||
"default": 0.5,
|
||||
"description": "Duration of fade in/out at segment boundaries (seconds).",
|
||||
},
|
||||
"target_duration": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"description": (
|
||||
"full_mix only. Exact output length in seconds. Pads a short "
|
||||
"mix and trims a long mix so audio matches the composition."
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -500,6 +508,15 @@ class AudioMixer(BaseTool):
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
normalize = inputs.get("normalize", True)
|
||||
ducking = inputs.get("ducking", {"enabled": True})
|
||||
target_duration = inputs.get("target_duration")
|
||||
target: float | None = None
|
||||
if target_duration is not None:
|
||||
try:
|
||||
target = float(target_duration)
|
||||
except (TypeError, ValueError):
|
||||
return ToolResult(success=False, error="target_duration must be a positive number")
|
||||
if target <= 0:
|
||||
return ToolResult(success=False, error="target_duration must be greater than zero")
|
||||
|
||||
speech_tracks = [t for t in tracks if t.get("role") in ("speech", "primary")]
|
||||
music_tracks = [t for t in tracks if t.get("role") in ("music", "secondary")]
|
||||
@@ -547,7 +564,14 @@ class AudioMixer(BaseTool):
|
||||
)
|
||||
else:
|
||||
filter_parts.append(f"[a{speech_indices[0]}]acopy[speech_all]")
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
|
||||
if target is not None:
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key_raw][speech_out]")
|
||||
filter_parts.append(
|
||||
f"[speech_key_raw]apad=whole_dur={target},"
|
||||
f"atrim=duration={target},asetpts=PTS-STARTPTS[speech_key]"
|
||||
)
|
||||
else:
|
||||
filter_parts.append("[speech_all]asplit=2[speech_key][speech_out]")
|
||||
|
||||
# Mix music tracks together
|
||||
music_start = len(speech_tracks)
|
||||
@@ -596,19 +620,34 @@ class AudioMixer(BaseTool):
|
||||
f"{all_labels}amix=inputs={len(all_tracks)}:duration=longest:dropout_transition=2[premix]"
|
||||
)
|
||||
|
||||
# A ducked music stream is gated by the speech sidechain, so its tail
|
||||
# can disappear when narration ends. If the caller knows the video
|
||||
# duration, make that the authoritative mix length before loudness
|
||||
# normalization: apad extends short audio and atrim caps long audio.
|
||||
premix_label = "premix"
|
||||
if target is not None:
|
||||
filter_parts.append(
|
||||
f"[premix]apad=whole_dur={target},atrim=duration={target},"
|
||||
"asetpts=PTS-STARTPTS[premix_duration]"
|
||||
)
|
||||
premix_label = "premix_duration"
|
||||
|
||||
# Normalize
|
||||
if normalize:
|
||||
filter_parts.append(self._loudnorm_filter(inputs, "premix", "out"))
|
||||
filter_parts.append(self._loudnorm_filter(inputs, premix_label, "out"))
|
||||
out_label = "[out]"
|
||||
else:
|
||||
out_label = "[premix]"
|
||||
out_label = f"[{premix_label}]"
|
||||
|
||||
filter_complex = ";".join(p for p in filter_parts if p)
|
||||
|
||||
cmd = ["ffmpeg", "-y"]
|
||||
cmd.extend(input_args)
|
||||
cmd.extend(["-filter_complex", filter_complex])
|
||||
cmd.extend(["-map", out_label, str(output_path)])
|
||||
cmd.extend(["-map", out_label])
|
||||
if target is not None:
|
||||
cmd.extend(["-t", str(target)])
|
||||
cmd.append(str(output_path))
|
||||
|
||||
self.run_command(cmd)
|
||||
|
||||
@@ -621,6 +660,7 @@ class AudioMixer(BaseTool):
|
||||
"sfx_tracks": len(sfx_tracks),
|
||||
"ducking_enabled": duck_enabled,
|
||||
"normalized": normalize,
|
||||
"target_duration": target_duration,
|
||||
"output": str(output_path),
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
|
||||
@@ -146,7 +146,8 @@ class GoogleMusic(BaseTool):
|
||||
from tools.google_credentials import get_genai_client, GOOGLE_API_TIMEOUT_MS
|
||||
|
||||
http_options = types.HttpOptions(timeout=GOOGLE_API_TIMEOUT_MS)
|
||||
client = get_genai_client(http_options=http_options)
|
||||
# Lyria 3 is served only from Vertex's global location.
|
||||
client = get_genai_client(http_options=http_options, location="global")
|
||||
except ImportError as e:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
|
||||
@@ -130,7 +130,7 @@ class BgRemove(BaseTool):
|
||||
|
||||
result_image = rembg.remove(
|
||||
input_image,
|
||||
model_name=model_name,
|
||||
session=rembg.new_session(model_name),
|
||||
alpha_matting=alpha_matting,
|
||||
)
|
||||
|
||||
|
||||
@@ -297,6 +297,11 @@ class Upscale(BaseTool):
|
||||
"model": model,
|
||||
"dni_weight": denoise_strength,
|
||||
"half": half,
|
||||
# Full-frame x4 inference can terminate the process on low-memory
|
||||
# CPU/MPS hosts before Python can raise an exception. Bound the
|
||||
# working set there; keep CUDA on the faster single-pass path.
|
||||
"tile": 0 if _device == "cuda" else 256,
|
||||
"tile_pad": 10,
|
||||
}
|
||||
# Guard: only pass device= if the installed version accepts it
|
||||
if "device" in inspect.signature(RealESRGANer.__init__).parameters:
|
||||
|
||||
@@ -18,6 +18,12 @@ from typing import Any
|
||||
# Broad scope that covers Cloud Text-to-Speech and Vertex AI prediction.
|
||||
CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"
|
||||
|
||||
|
||||
def resolve_google_location(location: str | None = None) -> str:
|
||||
"""Return a Vertex location, treating blank env values as unset."""
|
||||
|
||||
return location or os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1"
|
||||
|
||||
# Shared constants for long-running Google/Vertex AI generation calls (e.g. music, video)
|
||||
GOOGLE_API_TIMEOUT_SECONDS = 600
|
||||
GOOGLE_API_TIMEOUT_MS = GOOGLE_API_TIMEOUT_SECONDS * 1000
|
||||
@@ -38,8 +44,15 @@ def has_google_credentials() -> bool:
|
||||
)
|
||||
|
||||
|
||||
def get_genai_client(http_options: Any | None = None) -> Any:
|
||||
"""Lazily import and initialize the Google GenAI Client based on configured credentials."""
|
||||
def get_genai_client(
|
||||
http_options: Any | None = None,
|
||||
location: str | None = None,
|
||||
) -> Any:
|
||||
"""Initialize Google GenAI using the configured credential mode.
|
||||
|
||||
``location`` overrides the Vertex region for globally hosted models. It is
|
||||
deliberately ignored by the API-key backend, which has no region setting.
|
||||
"""
|
||||
from google import genai
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY")
|
||||
@@ -51,7 +64,7 @@ def get_genai_client(http_options: Any | None = None) -> Any:
|
||||
if use_vertex or (not api_key and service_account_configured()):
|
||||
kwargs = {
|
||||
"vertexai": True,
|
||||
"location": os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1"),
|
||||
"location": resolve_google_location(location),
|
||||
"http_options": http_options,
|
||||
}
|
||||
project_id = resolve_project_id()
|
||||
|
||||
@@ -22,6 +22,7 @@ from tools.base_tool import (
|
||||
)
|
||||
from tools.google_credentials import (
|
||||
get_access_token,
|
||||
resolve_google_location,
|
||||
resolve_project_id,
|
||||
service_account_configured,
|
||||
has_google_credentials,
|
||||
@@ -241,7 +242,7 @@ class GoogleImagen(BaseTool):
|
||||
}
|
||||
|
||||
if bearer_token:
|
||||
location = os.environ.get("GOOGLE_CLOUD_LOCATION", "us-central1")
|
||||
location = resolve_google_location()
|
||||
url = (
|
||||
f"https://{location}-aiplatform.googleapis.com/v1/projects/"
|
||||
f"{project_id}/locations/{location}/publishers/google/models/"
|
||||
|
||||
@@ -392,6 +392,38 @@ class CorpusBuilder(BaseTool):
|
||||
except Exception as e:
|
||||
cache_snapshot = {"error": f"{type(e).__name__}: {e}"}
|
||||
|
||||
# Per-candidate tolerance is useful only while at least one item
|
||||
# survives. If every discovered candidate fails, reporting success
|
||||
# persists an empty index and hides a systemic codec/CLIP failure
|
||||
# until retrieval. A no-result or skip-only run remains valid.
|
||||
total_failure = bool(candidates_seen) and not added_ids and not skipped
|
||||
if total_failure:
|
||||
first_errors = "; ".join(
|
||||
item["error"]
|
||||
for item in errors
|
||||
if item.get("phase") == "process"
|
||||
)[:400]
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
f"All {failed} of {candidates_seen} candidates failed to "
|
||||
"process; corpus index is empty. Check the media decoder "
|
||||
"and the CLIP `transformers`/`torch` compatibility. "
|
||||
f"First errors: {first_errors or '(none recorded)'}"
|
||||
),
|
||||
data={
|
||||
"corpus_dir": str(corpus_dir),
|
||||
"queries_run": len(queries),
|
||||
"candidates_seen": candidates_seen,
|
||||
"clips_added": 0,
|
||||
"clips_skipped_existing": skipped,
|
||||
"clips_failed": failed,
|
||||
"total_corpus_size": len(corp),
|
||||
"errors": errors[:25],
|
||||
},
|
||||
duration_seconds=round(elapsed, 2),
|
||||
)
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
|
||||
@@ -226,6 +226,7 @@ class HyperFramesCompose(BaseTool):
|
||||
# We cache per-process so the first call pays ~2-5s and subsequent calls
|
||||
# (get_info spam from the registry) are free.
|
||||
_npm_resolve_cache: Optional[dict[str, str]] = None
|
||||
_cli_probe_cache: Optional[dict[str, str]] = None
|
||||
|
||||
@classmethod
|
||||
def _node_major_version(cls) -> Optional[int]:
|
||||
@@ -301,6 +302,45 @@ class HyperFramesCompose(BaseTool):
|
||||
cls._npm_resolve_cache = {"version": version}
|
||||
return cls._npm_resolve_cache
|
||||
|
||||
@classmethod
|
||||
def _probe_cli(cls) -> dict[str, str]:
|
||||
"""Run the published CLI's doctor command once per process.
|
||||
|
||||
Package resolution alone does not prove that the executable can start:
|
||||
an upstream packaging regression can publish successfully while every
|
||||
CLI command crashes during bootstrap. Provider preflight must not call
|
||||
that state available.
|
||||
"""
|
||||
if cls._cli_probe_cache is not None:
|
||||
return cls._cli_probe_cache
|
||||
|
||||
npx = shutil.which("npx")
|
||||
if not npx:
|
||||
cls._cli_probe_cache = {"error": "npx not on PATH"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[npx, "--yes", cls._NPM_PACKAGE, "doctor", "--json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls._cli_probe_cache = {"error": "doctor timed out after 20s"}
|
||||
return cls._cli_probe_cache
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
cls._cli_probe_cache = {"error": f"doctor failed: {type(exc).__name__}"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
if proc.returncode != 0:
|
||||
output = "\n".join(filter(None, [proc.stderr, proc.stdout])).strip()
|
||||
tail = output.splitlines()[-1][:200] if output else f"exit {proc.returncode}"
|
||||
cls._cli_probe_cache = {"error": f"doctor failed: {tail}"}
|
||||
else:
|
||||
cls._cli_probe_cache = {"status": "ok"}
|
||||
return cls._cli_probe_cache
|
||||
|
||||
def _runtime_check(self) -> dict[str, Any]:
|
||||
"""Return availability state for the HyperFrames runtime.
|
||||
|
||||
@@ -336,6 +376,12 @@ class HyperFramesCompose(BaseTool):
|
||||
f"{npm_resolve['error']}"
|
||||
)
|
||||
|
||||
cli_probe: dict[str, str] = {}
|
||||
if not reasons:
|
||||
cli_probe = self._probe_cli()
|
||||
if "error" in cli_probe:
|
||||
reasons.append(f"published CLI is not executable: {cli_probe['error']}")
|
||||
|
||||
return {
|
||||
"runtime_available": not reasons,
|
||||
"node_major": node_major,
|
||||
@@ -344,6 +390,8 @@ class HyperFramesCompose(BaseTool):
|
||||
"npm_package": self._NPM_PACKAGE,
|
||||
"npm_package_version": npm_resolve.get("version"),
|
||||
"npm_resolve_error": npm_resolve.get("error"),
|
||||
"cli_probe_status": cli_probe.get("status"),
|
||||
"cli_probe_error": cli_probe.get("error"),
|
||||
"reasons": reasons,
|
||||
}
|
||||
|
||||
|
||||
@@ -308,13 +308,6 @@ class VeoVideo(BaseTool):
|
||||
client._api_client, "vertexai", False
|
||||
)
|
||||
|
||||
if is_vertex:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="Google Veo video generation via google-genai is only supported using the Gemini Developer API (API key) backend. "
|
||||
"Please configure GEMINI_API_KEY/GOOGLE_API_KEY or use the FAL.ai backend.",
|
||||
)
|
||||
|
||||
prompt = inputs["prompt"]
|
||||
operation = inputs.get("operation", "text_to_video")
|
||||
model_variant = inputs.get("model_variant", "veo3.1")
|
||||
@@ -501,7 +494,19 @@ class VeoVideo(BaseTool):
|
||||
success=False,
|
||||
error="No video asset returned in the response.",
|
||||
)
|
||||
client.files.download(file=video_asset)
|
||||
if not is_vertex:
|
||||
# The Files service is a Gemini Developer API feature. Vertex
|
||||
# returns bytes inline when no GCS output URI is requested.
|
||||
client.files.download(file=video_asset)
|
||||
elif not getattr(video_asset, "video_bytes", None):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=(
|
||||
"Vertex AI returned a video without inline bytes "
|
||||
f"(uri={getattr(video_asset, 'uri', None)!r}). Configure "
|
||||
"the request without an output GCS URI so bytes are returned inline."
|
||||
),
|
||||
)
|
||||
|
||||
output_path = Path(inputs.get("output_path", "veo_output.mp4"))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -31,11 +31,14 @@ the agent to re-ask the user rather than substituting a different engine.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
@@ -387,6 +390,49 @@ class VideoCompose(BaseTool):
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _mux_external_audio(self, video_path: Path, audio_path: str | Path) -> ToolResult:
|
||||
"""Atomically replace a rendered video's audio with the approved mix."""
|
||||
|
||||
audio = Path(audio_path).resolve()
|
||||
if not audio.is_file():
|
||||
return ToolResult(success=False, error=f"Mixed audio not found: {audio}")
|
||||
|
||||
temp_output = video_path.with_name(
|
||||
f".{video_path.stem}.audio-mux-{time.time_ns()}{video_path.suffix}"
|
||||
)
|
||||
try:
|
||||
self.run_command([
|
||||
"ffmpeg", "-y",
|
||||
"-i", str(video_path),
|
||||
"-i", str(audio),
|
||||
"-map", "0:v:0",
|
||||
"-map", "1:a:0",
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac",
|
||||
"-b:a", "192k",
|
||||
"-af", "apad",
|
||||
"-shortest",
|
||||
"-movflags", "+faststart",
|
||||
str(temp_output),
|
||||
])
|
||||
if not temp_output.is_file():
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Audio mux completed but output file is missing: {temp_output}",
|
||||
)
|
||||
temp_output.replace(video_path)
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, error=f"Could not mux mixed audio: {exc}")
|
||||
finally:
|
||||
if temp_output.exists():
|
||||
temp_output.unlink()
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={"output": str(video_path), "has_mixed_audio": True},
|
||||
artifacts=[str(video_path)],
|
||||
)
|
||||
|
||||
def _compose(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
"""FFmpeg composition: concat video cuts, add audio, burn subtitles.
|
||||
|
||||
@@ -714,6 +760,124 @@ class VideoCompose(BaseTool):
|
||||
)
|
||||
return comp
|
||||
|
||||
@staticmethod
|
||||
def _cuts_to_cinematic_scenes(cuts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Adapt canonical sequential cuts to CinematicRenderer's scene contract."""
|
||||
|
||||
scenes: list[dict[str, Any]] = []
|
||||
timeline_cursor = 0.0
|
||||
hard_transitions = {"cut", "none"}
|
||||
title_types = {"hero_title", "text_card", "title"}
|
||||
|
||||
for index, cut in enumerate(cuts):
|
||||
try:
|
||||
source_in = float(cut.get("in_seconds", 0))
|
||||
source_out = float(cut.get("out_seconds", source_in))
|
||||
speed = max(float(cut.get("speed", 1.0)), 0.1)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
duration = max(0.0, (source_out - source_in) / speed)
|
||||
if duration <= 0:
|
||||
continue
|
||||
|
||||
scene_id = str(cut.get("id") or f"cut-{index + 1}")
|
||||
source = str(cut.get("source") or "")
|
||||
cut_type = str(cut.get("type") or "").lower()
|
||||
common = {
|
||||
"id": scene_id,
|
||||
"startSeconds": timeline_cursor,
|
||||
"durationSeconds": duration,
|
||||
}
|
||||
|
||||
if cut_type in title_types or not source:
|
||||
scene: dict[str, Any] = {
|
||||
**common,
|
||||
"kind": "title",
|
||||
"text": str(
|
||||
cut.get("text")
|
||||
or cut.get("title")
|
||||
or cut.get("reason")
|
||||
or scene_id
|
||||
),
|
||||
}
|
||||
if source:
|
||||
scene["backgroundSrc"] = source
|
||||
scene["backgroundTrimBeforeSeconds"] = source_in
|
||||
scene["backgroundTrimAfterSeconds"] = source_out
|
||||
else:
|
||||
scene = {
|
||||
**common,
|
||||
"kind": "video",
|
||||
"src": source,
|
||||
"trimBeforeSeconds": source_in,
|
||||
"trimAfterSeconds": source_out,
|
||||
"playbackRate": speed,
|
||||
}
|
||||
if str(cut.get("transition_in") or "").lower() in hard_transitions:
|
||||
scene["fadeInFrames"] = 0
|
||||
if str(cut.get("transition_out") or "").lower() in hard_transitions:
|
||||
scene["fadeOutFrames"] = 0
|
||||
|
||||
scenes.append(scene)
|
||||
timeline_cursor += duration
|
||||
|
||||
return scenes
|
||||
|
||||
@staticmethod
|
||||
def _stage_remotion_media(value: Any, public_dir: Path) -> int:
|
||||
"""Copy local media references into a Remotion public dir in-place.
|
||||
|
||||
OffthreadVideo's compositor rejects ``file://`` sources. Rewriting
|
||||
staged files to relative ``staticFile()`` paths works for video and
|
||||
image components on every platform.
|
||||
"""
|
||||
|
||||
staged_by_source: dict[Path, str] = {}
|
||||
media_keys = {"source", "src", "backgroundSrc"}
|
||||
|
||||
def visit(node: Any, parent_key: str | None = None) -> Any:
|
||||
if isinstance(node, dict):
|
||||
for key, child in list(node.items()):
|
||||
node[key] = visit(child, key)
|
||||
return node
|
||||
if isinstance(node, list):
|
||||
for index, child in enumerate(node):
|
||||
node[index] = visit(child, parent_key)
|
||||
return node
|
||||
if not isinstance(node, str) or parent_key not in media_keys:
|
||||
return node
|
||||
if node.startswith(("http://", "https://", "data:")):
|
||||
return node
|
||||
|
||||
if node.lower().startswith("file://"):
|
||||
parsed = urlsplit(node)
|
||||
decoded_path = unquote(parsed.path)
|
||||
if len(parsed.netloc) == 2 and parsed.netloc[1] == ":":
|
||||
raw_path = f"{parsed.netloc}{decoded_path}"
|
||||
elif parsed.netloc and parsed.netloc.lower() != "localhost":
|
||||
raw_path = f"//{parsed.netloc}{decoded_path}"
|
||||
else:
|
||||
raw_path = decoded_path
|
||||
# Standard Windows file URIs use file:///C:/...; pathlib on
|
||||
# Windows needs the drive path without the URI's leading slash.
|
||||
if len(raw_path) >= 3 and raw_path[0] == "/" and raw_path[2] == ":":
|
||||
raw_path = raw_path[1:]
|
||||
else:
|
||||
raw_path = node
|
||||
source = Path(raw_path).resolve()
|
||||
if not source.is_file():
|
||||
return node
|
||||
if source not in staged_by_source:
|
||||
digest = hashlib.sha256(str(source).encode("utf-8")).hexdigest()[:12]
|
||||
name = f"{digest}-{source.name}"
|
||||
public_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, public_dir / name)
|
||||
staged_by_source[source] = name
|
||||
return staged_by_source[source]
|
||||
|
||||
visit(value)
|
||||
return len(staged_by_source)
|
||||
|
||||
def _render_via_atelier(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
@@ -840,6 +1004,11 @@ class VideoCompose(BaseTool):
|
||||
error=f"Atelier render completed but output file missing: {output_path}",
|
||||
)
|
||||
|
||||
if inputs.get("audio_path"):
|
||||
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
|
||||
if not mux_result.success:
|
||||
return mux_result
|
||||
|
||||
# --- Atelier post-render review -------------------------------------
|
||||
# The cut-schema paths run _run_final_review (technical/visual/audio
|
||||
# probes + transcript-vs-script). Atelier MUST do the same so hero
|
||||
@@ -1063,8 +1232,12 @@ class VideoCompose(BaseTool):
|
||||
try:
|
||||
from styles.playbook_loader import load_playbook
|
||||
playbook = load_playbook(playbook_name)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(
|
||||
"Could not load style playbook %r for Remotion theme: %s",
|
||||
playbook_name,
|
||||
exc,
|
||||
)
|
||||
|
||||
if playbook:
|
||||
vl = playbook.get("visual_language", {})
|
||||
@@ -1418,6 +1591,8 @@ class VideoCompose(BaseTool):
|
||||
# would only take effect on a direct _remotion_render() call.
|
||||
if inputs.get("remotion_timeout_ms") is not None:
|
||||
remotion_inputs["remotion_timeout_ms"] = inputs["remotion_timeout_ms"]
|
||||
if inputs.get("public_dir") is not None:
|
||||
remotion_inputs["public_dir"] = inputs["public_dir"]
|
||||
render_result = self._remotion_render(remotion_inputs)
|
||||
|
||||
# Governance: NEVER silently fall back to FFmpeg when Remotion fails.
|
||||
@@ -1437,6 +1612,11 @@ class VideoCompose(BaseTool):
|
||||
f"Per governance: renderer downgrade requires user approval."
|
||||
),
|
||||
)
|
||||
if inputs.get("audio_path"):
|
||||
mux_result = self._mux_external_audio(output_path, inputs["audio_path"])
|
||||
if not mux_result.success:
|
||||
return mux_result
|
||||
render_result.data["has_mixed_audio"] = True
|
||||
else:
|
||||
# --- FFmpeg fallback: only when Remotion is unavailable ---
|
||||
options = inputs.get("options", {})
|
||||
@@ -1546,7 +1726,12 @@ class VideoCompose(BaseTool):
|
||||
try:
|
||||
from styles.playbook_loader import load_playbook # type: ignore
|
||||
playbook_data = load_playbook(playbook_name)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
logging.getLogger(__name__).warning(
|
||||
"Could not load style playbook %r for HyperFrames bridge: %s",
|
||||
playbook_name,
|
||||
exc,
|
||||
)
|
||||
playbook_data = None
|
||||
|
||||
hf_inputs: dict[str, Any] = {
|
||||
@@ -1677,8 +1862,6 @@ class VideoCompose(BaseTool):
|
||||
types, and transitions using React-based frame-accurate rendering.
|
||||
Accepts edit_decisions (with resolved file paths) or raw composition_data.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
if not shutil.which("npx"):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
@@ -1700,16 +1883,6 @@ class VideoCompose(BaseTool):
|
||||
# Deep-copy props so we don't mutate the original
|
||||
props = json.loads(json.dumps(composition_data))
|
||||
|
||||
# Convert absolute file paths to file:// URIs for Remotion's
|
||||
# Img and OffthreadVideo components
|
||||
for cut in props.get("cuts", []):
|
||||
source = cut.get("source", "")
|
||||
if source and not source.startswith(("http://", "https://", "file://")):
|
||||
resolved = Path(source).resolve()
|
||||
if resolved.exists():
|
||||
posix = resolved.as_posix()
|
||||
cut["source"] = f"file:///{posix}" if not posix.startswith("/") else f"file://{posix}"
|
||||
|
||||
# Build a custom themeConfig from the playbook's actual colors.
|
||||
# This ensures every video gets a unique visual identity derived
|
||||
# from its production decisions — not picked from a preset menu.
|
||||
@@ -1723,11 +1896,6 @@ class VideoCompose(BaseTool):
|
||||
if theme_config:
|
||||
props["themeConfig"] = theme_config
|
||||
|
||||
# Write props to temp file for Remotion CLI
|
||||
props_path = output_path.parent / ".remotion_props.json"
|
||||
with open(props_path, "w", encoding="utf-8") as f:
|
||||
json.dump(props, f)
|
||||
|
||||
# remotion-composer lives at project root
|
||||
composer_dir = Path(__file__).resolve().parent.parent.parent / "remotion-composer"
|
||||
if not composer_dir.exists():
|
||||
@@ -1741,6 +1909,39 @@ class VideoCompose(BaseTool):
|
||||
renderer_family = (composition_data or {}).get("renderer_family", "explainer-data")
|
||||
composition_id = self._get_composition_id(renderer_family)
|
||||
|
||||
if composition_id == "CinematicRenderer":
|
||||
if not props.get("scenes") and props.get("cuts"):
|
||||
props["scenes"] = self._cuts_to_cinematic_scenes(props["cuts"])
|
||||
props.pop("cuts", None)
|
||||
if not props.get("scenes"):
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="CinematicRenderer received cuts but none could be adapted into scenes.",
|
||||
)
|
||||
|
||||
requested_public_dir = inputs.get("public_dir")
|
||||
cleanup_public_dir = False
|
||||
public_dir: Path | None = None
|
||||
if requested_public_dir:
|
||||
public_dir = Path(requested_public_dir).resolve()
|
||||
if not public_dir.is_dir():
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"Remotion public_dir does not exist or is not a directory: {public_dir}",
|
||||
)
|
||||
else:
|
||||
public_dir = output_path.parent / f".remotion-public-{output_path.stem}"
|
||||
cleanup_public_dir = True
|
||||
|
||||
staged_count = self._stage_remotion_media(props, public_dir)
|
||||
if not staged_count and cleanup_public_dir:
|
||||
public_dir = None
|
||||
|
||||
# Write the fully adapted/staged props, never the original cut payload.
|
||||
props_path = output_path.parent / ".remotion_props.json"
|
||||
with open(props_path, "w", encoding="utf-8") as f:
|
||||
json.dump(props, f)
|
||||
|
||||
cmd = [
|
||||
"npx", "remotion", "render",
|
||||
str(composer_dir / "src" / "index.tsx"),
|
||||
@@ -1753,6 +1954,8 @@ class VideoCompose(BaseTool):
|
||||
# API Remotion recommends for file paths and is cross-platform safe.
|
||||
f"--props={props_path}",
|
||||
]
|
||||
if public_dir is not None:
|
||||
cmd.append(f"--public-dir={public_dir}")
|
||||
|
||||
# Apply media profile dimensions
|
||||
profile_name = inputs.get("profile")
|
||||
@@ -1770,7 +1973,8 @@ class VideoCompose(BaseTool):
|
||||
# opaque failure. Pass it through and give the subprocess enough headroom
|
||||
# so run_command() does not kill Remotion before its own timeout fires.
|
||||
remotion_timeout_ms = inputs.get("remotion_timeout_ms")
|
||||
subprocess_timeout = 600
|
||||
scene_count = len(props.get("scenes") or props.get("cuts") or [])
|
||||
subprocess_timeout = max(600, scene_count * 15)
|
||||
if remotion_timeout_ms:
|
||||
try:
|
||||
ms = int(remotion_timeout_ms)
|
||||
@@ -1808,6 +2012,8 @@ class VideoCompose(BaseTool):
|
||||
finally:
|
||||
if props_path.exists():
|
||||
props_path.unlink()
|
||||
if cleanup_public_dir and public_dir is not None and public_dir.exists():
|
||||
shutil.rmtree(public_dir, ignore_errors=True)
|
||||
|
||||
if not output_path.exists():
|
||||
return ToolResult(
|
||||
@@ -1821,6 +2027,7 @@ class VideoCompose(BaseTool):
|
||||
"operation": "remotion_render",
|
||||
"output": str(output_path),
|
||||
"profile": profile_name,
|
||||
"staged_media_count": staged_count,
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user