diff --git a/.agents/skills/kling-official/SKILL.md b/.agents/skills/kling-official/SKILL.md index a1939e95..206c4909 100644 --- a/.agents/skills/kling-official/SKILL.md +++ b/.agents/skills/kling-official/SKILL.md @@ -141,8 +141,8 @@ Keep it separate from local `talking_head`. Pipelines that want Kling avatar out `kling_lip_sync` has two steps: -1. `POST /v1/videos/identify-face` with `video_id` or `video_url` -2. `POST /v1/videos/advanced-lip-sync` with `session_id`, `face_choose[]`, and `audio_id` or `sound_file` +1. `POST /v1/videos/identify-face` with `video_id` or `video_url`; read faces from `data.face_data[]` +2. `POST /v1/videos/advanced-lip-sync` with `session_id` and one `face_choose[]` item containing `face_id`, `audio_id` or `sound_file`, and the sound start/end/insert times Local video paths must not be silently uploaded through fal.ai or any other provider. If multiple faces are returned and the user did not pass `face_id` or `face_choose`, stop and return the face list for confirmation unless `auto_select_face=True` was explicitly set. If auto-selecting, record the selection reason and selected face in the result/artifact. diff --git a/scripts/kling_official_animated_explainer_e2e.py b/scripts/kling_official_animated_explainer_e2e.py index c13e5e95..a9db4d8f 100644 --- a/scripts/kling_official_animated_explainer_e2e.py +++ b/scripts/kling_official_animated_explainer_e2e.py @@ -4,8 +4,9 @@ This script validates the official Kling provider path through OpenMontage selectors and the animated-explainer asset/compose surface. -Default mode is a no-cost dry run. Use --live-tts for one paid TTS sample, or ---live-full for TTS + image + image-to-video + local FFmpeg compose. +Default mode is a no-cost dry run. Use --live-tts for one paid TTS sample, +--live-full for TTS + image + image-to-video + local FFmpeg compose, or +--live-all to add avatar and lip-sync provider smokes. """ from __future__ import annotations @@ -113,6 +114,8 @@ def _tool_statuses() -> dict[str, str]: "kling_tts", "kling_official_image", "kling_official_video", + "kling_avatar", + "kling_lip_sync", ] statuses: dict[str, str] = {} for name in names: @@ -123,7 +126,7 @@ def _tool_statuses() -> dict[str, str]: def _capability_summary() -> dict[str, Any]: summary = registry.provider_menu_summary() - wanted = {"tts", "image_generation", "video_generation", "video_post"} + wanted = {"tts", "image_generation", "video_generation", "avatar", "video_post"} return { "composition_runtimes": summary.get("composition_runtimes", {}), "capabilities": [ @@ -454,6 +457,198 @@ def _run_live_full( } +def _first_remote_url(result_data: dict[str, Any]) -> str: + direct = result_data.get("remote_url") + if direct: + return str(direct) + for item in result_data.get("remote_outputs") or []: + if not isinstance(item, dict): + continue + url = item.get("url") or item.get("video_url") or item.get("resource_url") + if url: + return str(url) + raise RuntimeError("Kling result did not include a remote video URL for lip-sync input.") + + +def _run_live_avatar_suite( + project_dir: Path, + *, + timeout_seconds: int, + poll_interval: float, +) -> dict[str, Any]: + image = registry.get("image_selector") + avatar = registry.get("kling_avatar") + lip_sync = registry.get("kling_lip_sync") + assert image and avatar and lip_sync + + assets_dir = project_dir / "assets" + image_dir = assets_dir / "images" + video_dir = assets_dir / "video" + artifacts_dir = project_dir / "artifacts" + narration_path = assets_dir / "audio" / "narration.mp3" + if not narration_path.is_file(): + raise RuntimeError( + f"Avatar smoke requires an existing narration file: {narration_path}. " + "Run --live-full or --live-all first." + ) + + _announce_paid_call( + "image_selector -> kling_official_image", + "kling_official", + "kling-v3 generation", + "Create one synthetic single-face portrait for the avatar provider smoke.", + "sample", + ) + portrait_result = image.execute( + { + "preferred_provider": "kling_official", + "allowed_providers": ["kling_official"], + "prompt": ( + "Photorealistic studio portrait of one fictional adult presenter, front-facing, " + "head and shoulders centered, neutral expression, mouth closed, even soft lighting, " + "plain background, no text, no watermark." + ), + "negative_prompt": "multiple people, profile view, open mouth, obscured face, text, watermark", + "api_family": "generation", + "model_name": "kling-v3", + "resolution": "1k", + "aspect_ratio": "1:1", + "n": 1, + "output_path": str(image_dir / "avatar_portrait.png"), + } + ) + _require_success("avatar portrait", portrait_result) + portrait_path = Path(portrait_result.data["output_path"]) + + _announce_paid_call( + "kling_avatar", + "kling_official", + "kling-official-avatar std", + "Validate photo-and-audio to avatar video through the official provider.", + "sample", + ) + avatar_result = avatar.execute( + { + "image_path": str(portrait_path), + "audio_path": str(narration_path), + "prompt": "Natural presenter delivery with subtle head movement and stable identity.", + "mode": "std", + "timeout_seconds": max(timeout_seconds, 900), + "poll_interval": poll_interval, + "output_path": str(video_dir / "kling_avatar_smoke.mp4"), + } + ) + _require_success("kling_avatar", avatar_result) + avatar_path = Path(avatar_result.data["output_path"]) + partial_report_path = artifacts_dir / "kling_avatar_live_partial.json" + _write_json( + partial_report_path, + { + "portrait": portrait_result.data, + "avatar": avatar_result.data, + "artifacts": { + "narration": str(narration_path), + "portrait": str(portrait_path), + "avatar": str(avatar_path), + }, + "ffprobe": { + "portrait": _probe_media(portrait_path), + "avatar": _probe_media(avatar_path), + }, + }, + ) + + _announce_paid_call( + "kling_lip_sync", + "kling_official", + "kling-official-lip-sync full_lip_sync", + "Validate identify-face, explicit auto-selection, and advanced lip-sync as one smoke.", + "sample", + ) + lip_sync_result = lip_sync.execute( + { + "operation": "full_lip_sync", + "video_url": _first_remote_url(avatar_result.data), + "audio_path": str(narration_path), + "auto_select_face": True, + "faces_artifact_path": str(artifacts_dir / "kling_lip_sync_faces.json"), + "timeout_seconds": max(timeout_seconds, 900), + "poll_interval": poll_interval, + "output_path": str(video_dir / "kling_lip_sync_smoke.mp4"), + } + ) + _require_success("kling_lip_sync", lip_sync_result) + lip_sync_path = Path(lip_sync_result.data["output_path"]) + + estimated_cost = sum( + float(getattr(result, "cost_usd", 0) or 0) + for result in (portrait_result, avatar_result, lip_sync_result) + ) + return { + "portrait": portrait_result.data, + "avatar": avatar_result.data, + "lip_sync": lip_sync_result.data, + "artifacts": { + "narration": str(narration_path), + "portrait": str(portrait_path), + "avatar": str(avatar_path), + "lip_sync": str(lip_sync_path), + "faces": str(artifacts_dir / "kling_lip_sync_faces.json"), + "avatar_partial_report": str(partial_report_path), + }, + "ffprobe": { + "portrait": _probe_media(portrait_path), + "avatar": _probe_media(avatar_path), + "lip_sync": _probe_media(lip_sync_path), + }, + "estimated_cost_usd": estimated_cost, + } + + +def _run_live_all( + project_dir: Path, + *, + voice_id: str, + voice_language: str, + voice_speed: float, + text: str, + timeout_seconds: int, + poll_interval: float, + include_account_usage: bool, + video_duration: str, +) -> dict[str, Any]: + core = _run_live_full( + project_dir, + voice_id=voice_id, + voice_language=voice_language, + voice_speed=voice_speed, + text=text, + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, + include_account_usage=include_account_usage, + video_duration=video_duration, + ) + avatar_suite = _run_live_avatar_suite( + project_dir, + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, + ) + return { + "core": core, + "avatar_suite": avatar_suite, + "artifacts": { + **core["artifacts"], + **avatar_suite["artifacts"], + }, + "ffprobe": { + "core": core["ffprobe"], + "avatar_suite": avatar_suite["ffprobe"], + }, + "estimated_cost_usd": float(core.get("estimated_cost_usd") or 0) + + float(avatar_suite.get("estimated_cost_usd") or 0), + } + + def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) live = parser.add_mutually_exclusive_group() @@ -465,6 +660,16 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: action="store_true", help="Run paid Kling TTS, image, video, and local compose.", ) + live.add_argument( + "--live-avatar", + action="store_true", + help="Use existing narration to run paid Kling portrait, avatar, and lip-sync samples.", + ) + live.add_argument( + "--live-all", + action="store_true", + help="Run the full smoke plus paid Kling avatar and lip-sync samples.", + ) parser.add_argument("--voice-id", default=DEFAULT_VOICE_ID) parser.add_argument("--voice-language", choices=["en", "zh"], default="en") parser.add_argument("--voice-speed", type=float, default=1.0) @@ -482,6 +687,10 @@ def _execution_mode(args: argparse.Namespace) -> str: return "live_tts" if getattr(args, "live_full", False): return "live_full" + if getattr(args, "live_avatar", False): + return "live_avatar" + if getattr(args, "live_all", False): + return "live_all" return "dry_run" @@ -545,10 +754,30 @@ def main(argv: Sequence[str] | None = None) -> int: include_account_usage=args.include_account_usage, video_duration=args.video_duration, ) + elif mode == "live_avatar": + report["live_avatar_result"] = _run_live_avatar_suite( + project_dir, + timeout_seconds=args.timeout_seconds, + poll_interval=args.poll_interval, + ) + elif mode == "live_all": + report["live_all_result"] = _run_live_all( + project_dir, + voice_id=args.voice_id, + voice_language=args.voice_language, + voice_speed=args.voice_speed, + text=args.text, + timeout_seconds=args.timeout_seconds, + poll_interval=args.poll_interval, + include_account_usage=args.include_account_usage, + video_duration=args.video_duration, + ) else: report["next_steps"] = [ "Run with --live-tts to make one paid Kling TTS sample call.", "Run with --live-full to make paid Kling TTS/image/video calls and compose final_kling_e2e_smoke.mp4.", + "Run with --live-avatar to reuse narration for paid Kling avatar and lip-sync provider smokes.", + "Run with --live-all to add paid Kling avatar and lip-sync provider smokes.", ] except Exception as exc: report["failed"] = {"error": str(exc)} @@ -563,6 +792,12 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"narration: {report['live_tts_result']['artifacts']['narration']}") elif mode == "live_full": print(f"final: {report['live_full_result']['artifacts']['final']}") + elif mode == "live_avatar": + print(f"avatar: {report['live_avatar_result']['artifacts']['avatar']}") + print(f"lip_sync: {report['live_avatar_result']['artifacts']['lip_sync']}") + elif mode == "live_all": + print(f"avatar: {report['live_all_result']['artifacts']['avatar']}") + print(f"lip_sync: {report['live_all_result']['artifacts']['lip_sync']}") return 0 diff --git a/tests/contracts/conftest.py b/tests/contracts/conftest.py new file mode 100644 index 00000000..c838f7a0 --- /dev/null +++ b/tests/contracts/conftest.py @@ -0,0 +1,15 @@ +"""Shared fixtures for contract tests.""" + +from __future__ import annotations + +import pytest + +from tools.tool_registry import ToolRegistry + + +@pytest.fixture() +def isolated_tool_registry(monkeypatch) -> ToolRegistry: + """Provide a registry singleton replacement scoped to one test.""" + test_registry = ToolRegistry() + monkeypatch.setattr("tools.tool_registry.registry", test_registry) + return test_registry diff --git a/tests/contracts/test_kling_official_avatar.py b/tests/contracts/test_kling_official_avatar.py index 2af63f11..eb6806ba 100644 --- a/tests/contracts/test_kling_official_avatar.py +++ b/tests/contracts/test_kling_official_avatar.py @@ -7,6 +7,8 @@ import json import sys from pathlib import Path +import pytest + PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) @@ -14,14 +16,12 @@ from tools.avatar.kling_avatar import KlingAvatar from tools.avatar.kling_lip_sync import KlingLipSync from tools.avatar.lip_sync import LipSync from tools.avatar.talking_head import TalkingHead -from tools.tool_registry import registry -def test_registry_discovers_kling_avatar(monkeypatch): +def test_registry_discovers_kling_avatar(monkeypatch, isolated_tool_registry): monkeypatch.delenv("KLING_API_KEY", raising=False) - registry.clear() - registry.discover("tools") - tool = registry.get("kling_avatar") + isolated_tool_registry.discover("tools") + tool = isolated_tool_registry.get("kling_avatar") assert tool is not None assert tool.capability == "avatar" assert tool.provider == "kling_official" @@ -123,11 +123,10 @@ def test_avatar_cost_estimate_is_not_zero(): assert tool.dry_run({"image_url": "x", "audio_id": "a"})["cost_estimate_confidence"] == "low" -def test_registry_discovers_kling_lip_sync(monkeypatch): +def test_registry_discovers_kling_lip_sync(monkeypatch, isolated_tool_registry): monkeypatch.delenv("KLING_API_KEY", raising=False) - registry.clear() - registry.discover("tools") - tool = registry.get("kling_lip_sync") + isolated_tool_registry.discover("tools") + tool = isolated_tool_registry.get("kling_lip_sync") assert tool is not None assert tool.capability == "avatar" assert tool.provider == "kling_official" @@ -135,8 +134,12 @@ def test_registry_discovers_kling_lip_sync(monkeypatch): def test_lip_sync_schema_and_local_tool_are_distinct(): tool = KlingLipSync() + props = tool.input_schema["properties"] assert "kling-official" in tool.agent_skills assert "avatar-video" in tool.agent_skills + assert "sound_start_time" in props + assert "sound_end_time" in props + assert "sound_insert_time" in props assert tool.runtime.value == "api" assert LipSync().provider == "wav2lip" assert LipSync().runtime.value == "local_gpu" @@ -165,17 +168,99 @@ def test_advanced_lip_sync_payload_uses_face_and_audio_path(tmp_path): "session_id": "session-a", "face_id": "face-a", "audio_path": str(audio_path), + "sound_start_time": 0, + "sound_end_time": 2500, + "sound_insert_time": 500, "callback_url": "https://example.com/kling/callback", } ) assert request["path"] == "/v1/videos/advanced-lip-sync" assert request["payload"]["session_id"] == "session-a" - assert request["payload"]["face_choose"] == [{"face_id": "face-a"}] - assert request["payload"]["sound_file"] == base64.b64encode(b"audio").decode("ascii") + assert request["payload"]["face_choose"] == [ + { + "face_id": "face-a", + "sound_file": base64.b64encode(b"audio").decode("ascii"), + "sound_start_time": 0, + "sound_end_time": 2500, + "sound_insert_time": 500, + } + ] + assert "sound_file" not in request["payload"] assert request["payload"]["callback_url"] == "https://example.com/kling/callback" +def test_advanced_lip_sync_accepts_official_nested_face_choose(): + face_choose = { + "face_id": "face-a", + "audio_id": "audio-a", + "sound_start_time": 0, + "sound_end_time": 4000, + "sound_insert_time": 500, + } + + request = KlingLipSync()._build_advanced_request( + {"session_id": "session-a", "face_choose": [face_choose]} + ) + + assert request["payload"]["face_choose"] == [face_choose] + assert request["audio_source"] == {"type": "audio_id", "value": "audio-a"} + + +def test_full_lip_sync_preserves_nested_face_timing_defaults(): + inputs = { + "session_id": "session-a", + "face_choose": [ + { + "face_id": "face-a", + "audio_id": "audio-a", + "sound_start_time": 250, + "sound_end_time": 4250, + "sound_insert_time": 750, + } + ], + } + tool = KlingLipSync() + + tool._apply_face_timing_defaults( + inputs, {"face_id": "face-a", "start_time": 0, "end_time": 0} + ) + request = tool._build_advanced_request(inputs) + + assert request["payload"]["face_choose"][0]["sound_start_time"] == 250 + assert request["payload"]["face_choose"][0]["sound_end_time"] == 4250 + assert request["payload"]["face_choose"][0]["sound_insert_time"] == 750 + + +@pytest.mark.parametrize( + ("top_level", "nested", "message"), + [ + ({"audio_id": "audio-top"}, {"audio_id": "audio-nested"}, "audio input"), + ({"sound_end_time": 5000}, {"sound_end_time": 4000}, "sound_end_time"), + ], +) +def test_advanced_lip_sync_rejects_conflicting_top_level_and_nested_values( + top_level, nested, message +): + inputs = { + "session_id": "session-a", + "face_choose": [ + { + "face_id": "face-a", + "audio_id": "audio-a", + "sound_start_time": 0, + "sound_end_time": 4000, + "sound_insert_time": 0, + **nested, + } + ], + **top_level, + } + + with pytest.raises(ValueError, match=message): + KlingLipSync()._build_advanced_request(inputs) + + def test_identify_face_execute_writes_faces_artifact(monkeypatch, tmp_path): class FakeClient: def post(self, path, payload): @@ -184,7 +269,14 @@ def test_identify_face_execute_writes_faces_artifact(monkeypatch, tmp_path): "code": 0, "data": { "session_id": "session-a", - "faces": [{"face_id": "face-a", "bbox": [0, 0, 100, 100]}], + "face_data": [ + { + "face_id": "face-a", + "face_image": "https://example.com/face.png", + "start_time": 0, + "end_time": 5200, + } + ], }, } @@ -214,9 +306,9 @@ def test_full_lip_sync_requires_confirmation_for_multiple_faces(monkeypatch, tmp "code": 0, "data": { "session_id": "session-a", - "faces": [ - {"face_id": "face-small", "bbox": [0, 0, 50, 50]}, - {"face_id": "face-large", "bbox": [0, 0, 200, 200]}, + "face_data": [ + {"face_id": "face-small", "bbox": [0, 0, 50, 50], "start_time": 0, "end_time": 4000}, + {"face_id": "face-large", "bbox": [0, 0, 200, 200], "start_time": 500, "end_time": 5500}, ], }, } @@ -247,9 +339,9 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_ "code": 0, "data": { "session_id": "session-a", - "faces": [ - {"face_id": "face-small", "bbox": [0, 0, 50, 50]}, - {"face_id": "face-large", "bbox": [0, 0, 200, 200]}, + "face_data": [ + {"face_id": "face-small", "bbox": [0, 0, 50, 50], "start_time": 0, "end_time": 4000}, + {"face_id": "face-large", "bbox": [0, 0, 200, 200], "start_time": 500, "end_time": 5500}, ], }, } @@ -257,7 +349,15 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_ def create_classic_task(self, path, payload): self.path = path self.payload = payload - assert payload["face_choose"] == [{"face_id": "face-large"}] + assert payload["face_choose"] == [ + { + "face_id": "face-large", + "audio_id": "audio-a", + "sound_start_time": 0, + "sound_end_time": 5000, + "sound_insert_time": 500, + } + ] return "lip-task-1" def poll_classic(self, path, task_id, result_key, timeout_seconds, poll_interval): @@ -285,7 +385,15 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_ assert result.success assert result.data["task_id"] == "lip-task-1" assert result.data["face_selection"]["selection_method"] == "auto_selected" - assert result.data["face_choose"] == [{"face_id": "face-large"}] + assert result.data["face_choose"] == [ + { + "face_id": "face-large", + "audio_id": "audio-a", + "sound_start_time": 0, + "sound_end_time": 5000, + "sound_insert_time": 500, + } + ] assert Path(result.artifacts[0]).read_bytes() == b"video" faces_artifact = next(Path(path) for path in result.artifacts if Path(path).name == "kling_lip_sync_faces.json") assert json.loads(faces_artifact.read_text())["selection"]["selection_method"] == "auto_selected" @@ -300,10 +408,21 @@ def test_auto_select_face_area_avoids_position_inflation(): def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path): + audio_path = tmp_path / "voice.mp3" + audio_path.write_bytes(b"audio") + class FakeClient: def create_classic_task(self, path, payload): assert path == "/v1/videos/advanced-lip-sync" - assert payload["face_choose"] == [{"face_id": "face-a"}] + assert payload["face_choose"] == [ + { + "face_id": "face-a", + "sound_file": base64.b64encode(b"audio").decode("ascii"), + "sound_start_time": 0, + "sound_end_time": 4000, + "sound_insert_time": 0, + } + ] return "lip-task-1" def poll_classic(self, path, task_id, result_key, timeout_seconds, poll_interval): @@ -322,7 +441,10 @@ def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path): "operation": "advanced_lip_sync", "session_id": "session-a", "face_id": "face-a", - "audio_id": "audio-a", + "audio_path": str(audio_path), + "sound_start_time": 0, + "sound_end_time": 4000, + "sound_insert_time": 0, "output_path": str(tmp_path / "lip.mp4"), } ) @@ -331,6 +453,15 @@ def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path): assert result.data["provider"] == "kling_official" assert result.data["task_id"] == "lip-task-1" assert result.data["duration_seconds"] == 4.0 + assert result.data["face_choose"] == [ + { + "face_id": "face-a", + "sound_start_time": 0, + "sound_end_time": 4000, + "sound_insert_time": 0, + "sound_file_provided": True, + } + ] assert result.cost_usd > 0 diff --git a/tests/contracts/test_kling_official_core.py b/tests/contracts/test_kling_official_core.py index 7e38708b..2b3250ea 100644 --- a/tests/contracts/test_kling_official_core.py +++ b/tests/contracts/test_kling_official_core.py @@ -27,7 +27,6 @@ from tools.audio.kling_tts import KlingTTS from tools.avatar.kling_avatar import KlingAvatar from tools.avatar.kling_lip_sync import KlingLipSync from tools.graphics.kling_official_image import KlingOfficialImage -from tools.tool_registry import registry from tools.video.kling_official_video import KlingOfficialVideo @@ -194,9 +193,9 @@ def test_turbo_create_and_poll_parse_result_paths(monkeypatch): def test_schema_snapshot_contains_phase1_contract_fields(): fixture = PROJECT_ROOT / "tests/fixtures/kling_official/schema_snapshot.json" data = json.loads(fixture.read_text()) - assert data["build_id"] == "97344324" - assert "index-B9E4in0e.js" in data["chunk_names"] - assert "document-navigation-nxVgwiS5.js" in data["chunk_names"] + assert data["build_id"] == "97939672" + assert "index-0m3slU3p.js" in data["chunk_names"] + assert "document-navigation-Dk7H_V3n.js" in data["chunk_names"] assert data["api_base"]["auth_env"] == "KLING_API_KEY" assert data["task_statuses"]["classic"] == ["submitted", "processing", "succeed", "failed"] assert data["task_statuses"]["turbo"] == ["submitted", "processing", "succeeded", "failed"] @@ -211,6 +210,7 @@ def test_schema_snapshot_contains_phase1_contract_fields(): assert data["endpoints"]["video_effects"]["path"] == "/v1/videos/effects" assert data["result_paths"]["classic_audio_results"] == "data.task_result.audios[]" assert data["result_paths"]["identify_face_session"] == "data.session_id" + assert data["result_paths"]["identify_face_results"] == "data.face_data[]" assert data["core_field_enums"]["tts_voice_language"] == ["zh", "en"] assert data["core_field_enums"]["avatar_mode"] == ["std", "pro"] @@ -251,7 +251,7 @@ def test_elements_helper_normalizes_and_records_metadata(tmp_path): raise AssertionError("element_list items without element_id must be rejected") -def test_elements_helper_read_only_endpoints_do_not_enter_registry(): +def test_elements_helper_read_only_endpoints_do_not_enter_registry(isolated_tool_registry): fake = HelperFakeClient() assert get_custom_element(123, client=fake)["data"]["element_id"] == 123 assert list_custom_elements(client=fake)["data"][0]["element_id"] == 456 @@ -266,10 +266,9 @@ def test_elements_helper_read_only_endpoints_do_not_enter_registry(): assert not hasattr(elements_module, "create_element") assert not hasattr(elements_module, "delete_element") - registry.clear() - registry.discover("tools") - assert registry.get("kling_elements") is None - assert registry.get("kling_account_usage") is None + isolated_tool_registry.discover("tools") + assert isolated_tool_registry.get("kling_elements") is None + assert isolated_tool_registry.get("kling_account_usage") is None def test_account_usage_helper_uses_endpoint_cache_and_throttle(): @@ -402,8 +401,7 @@ def test_provider_agent_skills_reference_kling_official(): assert "kling-official" in KlingLipSync().agent_skills -def test_phase3_does_not_register_audio_or_video_effect_tools(): - registry.clear() - registry.discover("tools") - assert registry.get("kling_audio") is None - assert registry.get("kling_effects") is None +def test_phase3_does_not_register_audio_or_video_effect_tools(isolated_tool_registry): + isolated_tool_registry.discover("tools") + assert isolated_tool_registry.get("kling_audio") is None + assert isolated_tool_registry.get("kling_effects") is None diff --git a/tests/contracts/test_kling_official_e2e_script.py b/tests/contracts/test_kling_official_e2e_script.py index 33faddeb..094eca43 100644 --- a/tests/contracts/test_kling_official_e2e_script.py +++ b/tests/contracts/test_kling_official_e2e_script.py @@ -43,6 +43,8 @@ def test_cli_modes_are_explicit_and_non_paid_by_default(): assert script._execution_mode(script._parse_args(["--live-tts"])) == "live_tts" assert script._execution_mode(script._parse_args(["--live-full"])) == "live_full" assert script._execution_mode(script._parse_args(["--live"])) == "live_full" + assert script._execution_mode(script._parse_args(["--live-avatar"])) == "live_avatar" + assert script._execution_mode(script._parse_args(["--live-all"])) == "live_all" def test_video_duration_aligns_to_narration_within_kling_limits(): @@ -52,3 +54,39 @@ def test_video_duration_aligns_to_narration_within_kling_limits(): assert script._aligned_video_duration("10", 6.05) == "10" assert script._aligned_video_duration("3", None) == "3" assert script._aligned_video_duration("3", 30.0) == "15" + + +def test_live_all_combines_core_and_avatar_results(monkeypatch, tmp_path): + script = _load_script() + core = { + "artifacts": {"narration": "narration.mp3", "final": "final.mp4"}, + "ffprobe": {"final": {"duration": 6.0}}, + "estimated_cost_usd": 0.25, + } + avatar_suite = { + "artifacts": {"avatar": "avatar.mp4", "lip_sync": "lip.mp4"}, + "ffprobe": {"avatar": {"duration": 7.0}, "lip_sync": {"duration": 7.0}}, + "estimated_cost_usd": 0.75, + } + monkeypatch.setattr(script, "_run_live_full", lambda *args, **kwargs: core) + monkeypatch.setattr( + script, "_run_live_avatar_suite", lambda *args, **kwargs: avatar_suite + ) + + result = script._run_live_all( + tmp_path, + voice_id="voice-a", + voice_language="en", + voice_speed=1.0, + text="hello", + timeout_seconds=30, + poll_interval=1.0, + include_account_usage=False, + video_duration="3", + ) + + assert result["core"] is core + assert result["avatar_suite"] is avatar_suite + assert result["artifacts"]["final"] == "final.mp4" + assert result["artifacts"]["lip_sync"] == "lip.mp4" + assert result["estimated_cost_usd"] == 1.0 diff --git a/tests/contracts/test_kling_official_image.py b/tests/contracts/test_kling_official_image.py index 1ced976d..e8ea37ce 100644 --- a/tests/contracts/test_kling_official_image.py +++ b/tests/contracts/test_kling_official_image.py @@ -9,16 +9,13 @@ from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) -from tools.graphics.image_selector import ImageSelector from tools.graphics.kling_official_image import KlingOfficialImage -from tools.tool_registry import registry -def test_registry_discovers_kling_official_image(monkeypatch): +def test_registry_discovers_kling_official_image(monkeypatch, isolated_tool_registry): monkeypatch.delenv("KLING_API_KEY", raising=False) - registry.clear() - registry.discover("tools") - tool = registry.get("kling_official_image") + isolated_tool_registry.discover("tools") + tool = isolated_tool_registry.get("kling_official_image") assert tool is not None assert tool.capability == "image_generation" assert tool.provider == "kling_official" @@ -255,12 +252,9 @@ def test_execute_image_omni_series_records_references_callback_and_artifacts(mon assert Path(result.artifacts[1]).name == "series_2.png" -def test_image_selector_prefers_official_provider(monkeypatch): +def test_image_selector_prefers_official_provider(monkeypatch, isolated_tool_registry): monkeypatch.setenv("KLING_API_KEY", "test-key") - registry.clear() - registry.register(KlingOfficialImage()) - registry.register(ImageSelector()) - registry._discovered_packages.add("tools") + isolated_tool_registry.discover("tools") def fake_execute(self, inputs): from tools.base_tool import ToolResult @@ -268,10 +262,11 @@ def test_image_selector_prefers_official_provider(monkeypatch): return ToolResult(success=True, data={"output_path": "out.png"}, artifacts=["out.png"]) monkeypatch.setattr(KlingOfficialImage, "execute", fake_execute) - result = registry.get("image_selector").execute( + result = isolated_tool_registry.get("image_selector").execute( { "prompt": "official image", "preferred_provider": "kling_official", + "allowed_providers": ["kling_official"], "api_family": "omni", "image_reference": "subject", } diff --git a/tests/contracts/test_kling_official_video.py b/tests/contracts/test_kling_official_video.py index 45cc7b5d..3660ca1b 100644 --- a/tests/contracts/test_kling_official_video.py +++ b/tests/contracts/test_kling_official_video.py @@ -10,17 +10,14 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) from tools._kling.account import reset_account_usage_cache -from tools.tool_registry import registry from tools._kling.errors import KlingAPIError from tools.video.kling_official_video import KlingOfficialVideo -from tools.video.video_selector import VideoSelector -def test_registry_discovers_kling_official_video(monkeypatch): +def test_registry_discovers_kling_official_video(monkeypatch, isolated_tool_registry): monkeypatch.delenv("KLING_API_KEY", raising=False) - registry.clear() - registry.discover("tools") - tool = registry.get("kling_official_video") + isolated_tool_registry.discover("tools") + tool = isolated_tool_registry.get("kling_official_video") assert tool is not None assert tool.capability == "video_generation" assert tool.provider == "kling_official" @@ -355,14 +352,13 @@ def test_execute_downloads_all_omni_video_outputs_and_records_metadata(monkeypat assert Path(result.artifacts[1]).name == "out_2.mp4" -def test_video_selector_prefers_official_provider_without_fal_upload(monkeypatch, tmp_path): +def test_video_selector_prefers_official_provider_without_fal_upload( + monkeypatch, tmp_path, isolated_tool_registry +): monkeypatch.setenv("KLING_API_KEY", "test-key") image_path = tmp_path / "ref.png" image_path.write_bytes(b"fake") - registry.clear() - registry.register(KlingOfficialVideo()) - registry.register(VideoSelector()) - registry._discovered_packages.add("tools") + isolated_tool_registry.discover("tools") seen = {} @@ -377,11 +373,12 @@ def test_video_selector_prefers_official_provider_without_fal_upload(monkeypatch monkeypatch.setattr(KlingOfficialVideo, "execute", fake_execute) monkeypatch.setattr("tools.video._shared.upload_image_fal", fail_upload) - result = registry.get("video_selector").execute( + result = isolated_tool_registry.get("video_selector").execute( { "prompt": "animate", "operation": "image_to_video", "preferred_provider": "kling_official", + "allowed_providers": ["kling_official"], "reference_image_path": str(image_path), } ) diff --git a/tests/contracts/test_kling_tts.py b/tests/contracts/test_kling_tts.py index 89dc54b6..da1d4aec 100644 --- a/tests/contracts/test_kling_tts.py +++ b/tests/contracts/test_kling_tts.py @@ -9,15 +9,12 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) from tools.audio.kling_tts import KlingTTS -from tools.audio.tts_selector import TTSSelector -from tools.tool_registry import registry -def test_registry_discovers_kling_tts(monkeypatch): +def test_registry_discovers_kling_tts(monkeypatch, isolated_tool_registry): monkeypatch.delenv("KLING_API_KEY", raising=False) - registry.clear() - registry.discover("tools") - tool = registry.get("kling_tts") + isolated_tool_registry.discover("tools") + tool = isolated_tool_registry.get("kling_tts") assert tool is not None assert tool.capability == "tts" assert tool.provider == "kling_official" @@ -153,12 +150,9 @@ def test_execute_accepts_synchronous_create_response(monkeypatch, tmp_path): assert result.data["audio_duration_seconds"] == 2.5 -def test_tts_selector_prefers_kling_official(monkeypatch): +def test_tts_selector_prefers_kling_official(monkeypatch, isolated_tool_registry): monkeypatch.setenv("KLING_API_KEY", "test-key") - registry.clear() - registry.register(KlingTTS()) - registry.register(TTSSelector()) - registry._discovered_packages.add("tools") + isolated_tool_registry.discover("tools") def fake_execute(self, inputs): from tools.base_tool import ToolResult @@ -166,11 +160,12 @@ def test_tts_selector_prefers_kling_official(monkeypatch): return ToolResult(success=True, data={"output_path": "out.mp3"}, artifacts=["out.mp3"]) monkeypatch.setattr(KlingTTS, "execute", fake_execute) - result = registry.get("tts_selector").execute( + result = isolated_tool_registry.get("tts_selector").execute( { "text": "official speech", "voice_id": "voice-a", "preferred_provider": "kling_official", + "allowed_providers": ["kling_official"], } ) assert result.success diff --git a/tests/fixtures/kling_official/schema_snapshot.json b/tests/fixtures/kling_official/schema_snapshot.json index ad136c3d..425a9a3d 100644 --- a/tests/fixtures/kling_official/schema_snapshot.json +++ b/tests/fixtures/kling_official/schema_snapshot.json @@ -1,5 +1,5 @@ { - "build_id": "97344324", + "build_id": "97939672", "source_urls": [ "https://kling.ai/document-api/api/get-started/authentication", "https://kling.ai/document-api/api/get-started/error-codes", @@ -18,10 +18,10 @@ "https://kling.ai/document-api/api/video/effects" ], "chunk_names": [ - "index-B9E4in0e.js", - "document-navigation-nxVgwiS5.js" + "index-0m3slU3p.js", + "document-navigation-Dk7H_V3n.js" ], - "extracted_at": "2026-07-03T08:12:55Z", + "extracted_at": "2026-07-10T08:34:20Z", "api_base": { "default": "https://api-singapore.klingai.com", "env_override": "KLING_API_BASE_URL", @@ -145,6 +145,7 @@ "classic_image_results": "data.task_result.images[]", "classic_audio_results": "data.task_result.audios[]", "identify_face_session": "data.session_id", + "identify_face_results": "data.face_data[]", "turbo_results": "data[0].outputs[]" }, "core_field_enums": { @@ -220,7 +221,8 @@ ] }, "notes": [ - "Official docs were fetched as a SPA on 2026-07-03. Current HTML exposes buildId 97344324.", + "Official docs were fetched as a SPA on 2026-07-10. Current HTML exposes buildId 97939672.", + "Advanced lip sync currently accepts one face_choose item. The item contains face_id, audio_id or sound_file, and sound_start_time, sound_end_time, and sound_insert_time.", "The current entry/navigation bundles no longer expose the older api-*.js chunk names as literal references, so this fixture records the current entry assets and the Phase 1 core schema facts used by implementation and tests." ] } diff --git a/tools/avatar/kling_lip_sync.py b/tools/avatar/kling_lip_sync.py index 272386f4..b27bab55 100644 --- a/tools/avatar/kling_lip_sync.py +++ b/tools/avatar/kling_lip_sync.py @@ -99,6 +99,23 @@ class KlingLipSync(BaseTool): "type": "string", "description": "Alias for sound_file_path for compatibility with local lip_sync.", }, + "sound_start_time": { + "type": "integer", + "minimum": 0, + "description": "Audio crop start in milliseconds.", + }, + "sound_end_time": { + "type": "integer", + "minimum": 0, + "description": "Audio crop end in milliseconds; inferred for full_lip_sync when possible.", + }, + "sound_insert_time": { + "type": "integer", + "minimum": 0, + "description": "Video timeline insertion point in milliseconds.", + }, + "sound_volume": {"type": "number", "minimum": 0, "maximum": 2}, + "original_audio_volume": {"type": "number", "minimum": 0, "maximum": 2}, "faces_artifact_path": {"type": "string"}, "callback_url": {"type": "string"}, "external_task_id": {"type": "string"}, @@ -121,7 +138,17 @@ class KlingLipSync(BaseTool): backoff_seconds=2.0, retryable_errors=["1302", "1303", "5000", "5001", "5002"], ) - idempotency_key_fields = ["video_id", "video_url", "session_id", "face_id", "audio_id", "sound_file_path"] + idempotency_key_fields = [ + "video_id", + "video_url", + "session_id", + "face_id", + "audio_id", + "sound_file_path", + "sound_start_time", + "sound_end_time", + "sound_insert_time", + ] side_effects = [ "paid remote generation via official Kling API", "writes face selection artifact", @@ -192,6 +219,8 @@ class KlingLipSync(BaseTool): model="kling-official-lip-sync", ) merged = {**inputs, "session_id": identify["session_id"], "face_choose": face_choose} + selected_face = self._selected_face_record(identify["faces"], face_choose) + self._apply_face_timing_defaults(merged, selected_face) request = self._build_advanced_request(merged) result = self._run_advanced_lip_sync(client, merged, request, start) result.data["faces_artifact_path"] = str(artifact_path) @@ -225,7 +254,8 @@ class KlingLipSync(BaseTool): if not session_id: raise ValueError(f"Kling identify-face response missing data.session_id: {data}") faces = ( - payload.get("faces") + payload.get("face_data") + or payload.get("faces") or payload.get("face_list") or payload.get("face_infos") or payload.get("faces_info") @@ -286,7 +316,9 @@ class KlingLipSync(BaseTool): "task_id": task_id, "operation": request["operation"], "session_id": request["payload"]["session_id"], - "face_choose": request["payload"]["face_choose"], + "face_choose": self._face_choose_result_metadata( + request["payload"]["face_choose"] + ), "audio_source": request["audio_source"], "remote_outputs": outputs, "output": str(paths[0]), @@ -328,11 +360,15 @@ class KlingLipSync(BaseTool): face_choose = self._normalize_face_choose(inputs) if not face_choose: raise ValueError("advanced_lip_sync requires face_choose or face_id") + if len(face_choose) != 1: + raise ValueError("advanced_lip_sync currently supports exactly one face_choose item") + face_item = face_choose[0] + audio_source = self._copy_audio_input(inputs, face_item) + self._copy_timing_fields(inputs, face_item) payload: dict[str, Any] = { "session_id": session_id, "face_choose": face_choose, } - audio_source = self._copy_audio_input(inputs, payload) self._copy_common_task_fields(inputs, payload) return { "protocol": "classic", @@ -438,12 +474,156 @@ class KlingLipSync(BaseTool): return max(float(width), 0.0) * max(float(height), 0.0) return 0.0 + @staticmethod + def _selected_face_record( + faces: list[dict[str, Any]], face_choose: list[dict[str, Any]] + ) -> dict[str, Any]: + selected_id = str(face_choose[0].get("face_id") or "") + for face in faces: + if str(face.get("face_id") or face.get("id") or "") == selected_id: + return face + raise ValueError(f"Selected face_id {selected_id!r} was not returned by identify_face") + + def _apply_face_timing_defaults( + self, inputs: dict[str, Any], face: dict[str, Any] + ) -> None: + face_start = int(face.get("start_time") or 0) + face_end = int(face.get("end_time") or 0) + face_choose = self._normalize_face_choose(inputs) + face_item = face_choose[0] if face_choose else {} + if inputs.get("sound_start_time") is None and face_item.get("sound_start_time") is None: + inputs["sound_start_time"] = 0 + if inputs.get("sound_insert_time") is None and face_item.get("sound_insert_time") is None: + inputs["sound_insert_time"] = face_start + if inputs.get("sound_end_time") is not None or face_item.get("sound_end_time") is not None: + return + + candidates: list[int] = [] + audio_duration = self._local_audio_duration_ms(inputs) + if audio_duration: + candidates.append(audio_duration) + if face_end > face_start: + candidates.append(face_end - face_start) + if not candidates: + raise ValueError( + "full_lip_sync could not infer sound_end_time; provide it explicitly" + ) + inputs["sound_end_time"] = min(candidates) + + @staticmethod + def _local_audio_duration_ms(inputs: dict[str, Any]) -> int | None: + sound_path = inputs.get("sound_file_path") or inputs.get("audio_path") + if not sound_path: + return None + path = Path(sound_path) + if not path.is_file(): + return None + seconds = probe_output(path).get("duration_seconds") + if not seconds: + return None + return int(round(float(seconds) * 1000)) + + @staticmethod + def _copy_timing_fields(inputs: dict[str, Any], face_item: dict[str, Any]) -> None: + for key, default in (("sound_start_time", 0), ("sound_insert_time", 0)): + nested = face_item.get(key) + top_level = inputs.get(key) + if nested is not None and top_level is not None and int(nested) != int(top_level): + raise ValueError( + f"Conflicting {key} values between top-level input and face_choose[0]" + ) + value = nested if nested is not None else top_level + if value is None: + value = default + face_item[key] = int(value) + + nested_end = face_item.get("sound_end_time") + top_level_end = inputs.get("sound_end_time") + if ( + nested_end is not None + and top_level_end is not None + and int(nested_end) != int(top_level_end) + ): + raise ValueError( + "Conflicting sound_end_time values between top-level input and face_choose[0]" + ) + sound_end = nested_end if nested_end is not None else top_level_end + if sound_end is None: + raise ValueError("advanced_lip_sync requires sound_end_time") + face_item["sound_end_time"] = int(sound_end) + if face_item["sound_end_time"] - face_item["sound_start_time"] < 2000: + raise ValueError("advanced_lip_sync requires at least 2000ms of cropped audio") + + for key in ("sound_volume", "original_audio_volume"): + nested = face_item.get(key) + top_level = inputs.get(key) + if nested is not None and top_level is not None and float(nested) != float(top_level): + raise ValueError( + f"Conflicting {key} values between top-level input and face_choose[0]" + ) + value = nested if nested is not None else top_level + if value is not None: + face_item[key] = float(value) + + @staticmethod + def _face_choose_result_metadata( + face_choose: list[dict[str, Any]], + ) -> list[dict[str, Any]]: + metadata: list[dict[str, Any]] = [] + for item in face_choose: + record = {key: value for key, value in item.items() if key != "sound_file"} + if item.get("sound_file"): + record["sound_file_provided"] = True + metadata.append(record) + return metadata + @staticmethod def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: - audio_id = str(inputs.get("audio_id") or "").strip() - if audio_id: - payload["audio_id"] = audio_id - return {"type": "audio_id", "value": audio_id} + nested_audio_id = str(payload.get("audio_id") or "").strip() + nested_sound_file = payload.get("sound_file") + if nested_audio_id and nested_sound_file: + raise ValueError( + "Conflicting audio input in face_choose[0]; provide audio_id or sound_file, not both" + ) + + top_level_audio_id = str(inputs.get("audio_id") or "").strip() + top_level_sound_requested = any( + inputs.get(key) + for key in ("sound_file", "sound_file_url", "sound_file_path", "audio_path") + ) + + if nested_audio_id: + if ( + (top_level_audio_id and top_level_audio_id != nested_audio_id) + or top_level_sound_requested + ): + raise ValueError( + "Conflicting audio input between top-level fields and face_choose[0]" + ) + payload["audio_id"] = nested_audio_id + return {"type": "audio_id", "value": nested_audio_id} + + if nested_sound_file: + if top_level_audio_id: + raise ValueError( + "Conflicting audio input between top-level fields and face_choose[0]" + ) + if top_level_sound_requested: + top_level_sound_file = normalize_media_input( + url=inputs.get("sound_file_url"), + path=inputs.get("sound_file_path") or inputs.get("audio_path"), + value=inputs.get("sound_file"), + label="Lip-sync audio file", + ) + if top_level_sound_file != nested_sound_file: + raise ValueError( + "Conflicting audio input between top-level fields and face_choose[0]" + ) + return {"type": "sound_file", "source": "face_choose[0]"} + + if top_level_audio_id: + payload["audio_id"] = top_level_audio_id + return {"type": "audio_id", "value": top_level_audio_id} sound_path = inputs.get("sound_file_path") or inputs.get("audio_path") sound_file = normalize_media_input(