diff --git a/.agents/skills/fish-audio-tts/SKILL.md b/.agents/skills/fish-audio-tts/SKILL.md new file mode 100644 index 00000000..a7c1dac3 --- /dev/null +++ b/.agents/skills/fish-audio-tts/SKILL.md @@ -0,0 +1,107 @@ +--- +name: fish-audio-tts +description: Generate expressive, multilingual narration with fish.audio (S1 / S2-generation models) and reuse cloned voices via reference_id. Use when the user prefers fish.audio/Fish Audio TTS, wants a specific playground voice model, or needs high-emotion voice-clone narration. +--- + +# fish.audio TTS + +Requires `FISH_AUDIO_API_KEY` in `.env` (create one at https://fish.audio/go-api/api-keys/). +Create voice models in the fish.audio playground and pass their id as `reference_id` to reuse a cloned voice. + +## Current API + +Single synchronous call returning raw audio bytes: + +```text +POST https://api.fish.audio/v1/tts +Authorization: Bearer ${FISH_AUDIO_API_KEY} +Content-Type: application/json +model: # HTTP header selects the backend, e.g. s1 +``` + +The backend model is chosen with the `model` **HTTP header**, not a body field. In OpenMontage this maps to the tool's `model` input. + +## Backend models + +`model` is **required — there is no default**. Pass one of: + +- `s2.1-pro` — latest generation. Best quality: inline emotion tags, 80+ languages, multi-speaker. Hero narration. +- `s2.1-pro-free` — free tier of s2.1-pro. Drafts, samples, and validation runs at $0. +- `s2-pro` — first S2 generation. Stable high quality with emotion-tag support. +- `s1` — previous flagship. Kept for compatibility with existing integrations. + +Billing is **per UTF-8 byte of input text** (not per character). CJK text and emoji cost 3-4x an ASCII character of the same visible length. Approximate: `s1` / `s2-pro` / `s2.1-pro` ≈ $15 per 1M bytes, `s2.1-pro-free` = $0. Verify current pricing at https://fish.audio before large batches. + +## Inline emotion tags (S2 models only) + +`s2-pro` / `s2.1-pro` / `s2.1-pro-free` interpret inline emotion tags embedded in the text: + +- Tags like `[laugh]`, `[whispers]` change the delivery mid-sentence. +- Example: `"That's hilarious [laugh] but let me explain seriously."` +- `s1` does not interpret emotion tags — they may be read out as plain text, so strip them when targeting s1. + +## Voice selection (reference_id) + +- Build or pick a voice in the fish.audio playground, then copy its model id. +- Pass it as `reference_id`. The selector's generic `voice_id` is accepted as an alias when `reference_id` is absent. +- Without a `reference_id`, fish.audio uses its default voice for the chosen model. + +Inline on-the-fly cloning (uploading reference audio + text per request) is **not** supported by this tool — create a voice model in the playground first. + +## OpenMontage Usage + +Generate with the TTS selector: + +```python +from tools.audio.tts_selector import TTSSelector + +result = TTSSelector().execute({ + "preferred_provider": "fish_audio", + "text": "Here's why compound interest quietly beats every get-rich-quick scheme.", + "model": "s1", + "reference_id": "", + "output_path": "projects/my-video/assets/audio/narration.mp3", +}) +``` + +Or call the provider directly: + +```python +from tools.audio.fish_audio_tts import FishAudioTTS + +result = FishAudioTTS().execute({ + "text": "Short sample line for approval.", + "model": "s1", + "reference_id": "", + "output_path": "projects/my-video/assets/audio/fish_sample.mp3", +}) +``` + +The provider writes the audio to `output_path` and returns `data.output` plus the resolved `model` and `reference_id`. + +## Quality & latency tuning + +- `latency`: `normal` (default, best quality), `balanced` (a little faster), or `low` (fastest, slight quality cost). +- `normalize`: default `true`; keep it on so numbers, dates, and currency read naturally. +- `prosody`: optional `{ "speed": 1.0, "volume": 0 }` to nudge pace/loudness. +- `mp3_bitrate`: `128` is a good default; raise to `192` for music-bed-heavy mixes. +- `temperature`: default `0.7`. Raise toward `0.9` for more expressive reads (recommended when leaning on emotion tags); lower for a steadier, more predictable delivery. +- `top_p` / `repetition_penalty`: usually leave at the defaults (`0.7` / `1.2`). + +## Recommended Workflow + +1. Generate a 10-15 second sample with the chosen `model` + `reference_id` before a full paid narration. +2. Ask the user to approve voice naturalness, emotion, and pace. +3. Generate the full narration only after approval. +4. For batch/localization variants where cost matters, prototype on `s2.1-pro-free` ($0) and upgrade the final to `s2.1-pro`. + +## Troubleshooting + +- `401 Unauthorized`: wrong or missing `FISH_AUDIO_API_KEY`. +- `402` / payment errors: account credit exhausted. +- `404` / bad voice: the `reference_id` is wrong or not owned by this account. +- Empty/short audio: check that `text` is non-empty and `normalize` is not stripping the whole input. + +## Safety + +Never print or write the API key to logs, metadata, patches, or project artifacts. `.env.example` should contain only empty variable names. diff --git a/.env.example b/.env.example index 348d4c5e..2aa08dd9 100644 --- a/.env.example +++ b/.env.example @@ -38,6 +38,7 @@ OPENAI_API_KEY= # OpenAI TTS fallback and GPT Image 2 image generat XAI_API_KEY= # Grok image generation/editing and Grok video generation DOUBAO_SPEECH_API_KEY= # Volcengine Doubao Speech TTS (new console API Key) DOUBAO_SPEECH_VOICE_TYPE= # Default Doubao speaker/voice type, e.g. zh_female_vv_uranus_bigtts +FISH_AUDIO_API_KEY= # fish.audio TTS (S1 / speech-1.x, reference_id voice cloning) # Piper local voices do not require env vars; install `piper-tts` via pip # --- DashScope (Alibaba Cloud Bailian) --- diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 2a0b6bd5..af23adb5 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -501,7 +501,7 @@ Three selector tools abstract multi-provider capabilities. **Selectors auto-disc | Selector | Routes to | How it discovers | |----------|-----------|-----------------| -| `tts_selector` | All tools with `capability="tts"` (ElevenLabs, Google TTS, OpenAI, Piper) | `registry.get_by_capability("tts")` | +| `tts_selector` | All tools with `capability="tts"` (ElevenLabs, Google TTS, OpenAI, Piper, fish.audio) | `registry.get_by_capability("tts")` | | `image_selector` | All tools with `capability="image_generation"` (FLUX, Google Imagen, GPT Image, Recraft, etc.) | `registry.get_by_capability("image_generation")` | | `video_selector` | All tools with `capability="video_generation"` | `registry.get_by_capability("video_generation")` | diff --git a/tests/tools/test_fish_audio_tts.py b/tests/tools/test_fish_audio_tts.py new file mode 100644 index 00000000..0f45b263 --- /dev/null +++ b/tests/tools/test_fish_audio_tts.py @@ -0,0 +1,204 @@ +"""Unit tests for the fish.audio TTS provider (tools/audio/fish_audio_tts.py). + +The real API is never called — requests.post is patched to return synthetic +audio bytes. Covers status gating, the required-model contract, the +voice_id -> reference_id alias, request shape, output writing, cost, and +API-key redaction on error. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from tools.audio.fish_audio_tts import FishAudioTTS +from tools.base_tool import ToolStatus + + +class _FakeResponse: + def __init__(self, content: bytes = b"ID3fake-audio", status_code: int = 200): + self.content = content + self.status_code = status_code + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + +@pytest.fixture +def api_key(monkeypatch): + monkeypatch.setenv("FISH_AUDIO_API_KEY", "sk-fish-secret") + return "sk-fish-secret" + + +@pytest.fixture +def no_api_key(monkeypatch): + monkeypatch.delenv("FISH_AUDIO_API_KEY", raising=False) + + +# ---------------------------------------------------------------------- +# Status gating +# ---------------------------------------------------------------------- + + +class TestStatus: + def test_unavailable_without_key(self, no_api_key): + assert FishAudioTTS().get_status() == ToolStatus.UNAVAILABLE + + def test_available_with_key(self, api_key): + assert FishAudioTTS().get_status() == ToolStatus.AVAILABLE + + def test_execute_fails_without_key(self, no_api_key): + result = FishAudioTTS().execute({"text": "hello", "model": "s1"}) + assert result.success is False + assert "fish.audio API key" in result.error + + +# ---------------------------------------------------------------------- +# Required-model contract +# ---------------------------------------------------------------------- + + +class TestModelContract: + def test_missing_model_errors_with_valid_values(self, api_key): + result = FishAudioTTS().execute({"text": "hello"}) + assert result.success is False + assert "requires an explicit 'model'" in result.error + assert "s1" in result.error + + def test_unknown_model_errors(self, api_key): + result = FishAudioTTS().execute({"text": "hello", "model": "gpt-voice"}) + assert result.success is False + assert "Unknown fish.audio model" in result.error + + def test_retired_legacy_model_rejected(self, api_key): + # speech-1.x / s1-mini are gone from the current fish.audio lineup. + result = FishAudioTTS().execute({"text": "hello", "model": "speech-1.5"}) + assert result.success is False + assert "Unknown fish.audio model" in result.error + + +# ---------------------------------------------------------------------- +# Happy path +# ---------------------------------------------------------------------- + + +class TestGenerate: + def test_writes_audio_and_reports_metadata(self, api_key, tmp_path): + out = tmp_path / "narration.mp3" + with patch("requests.post", return_value=_FakeResponse(b"AUDIOBYTES")) as mock_post: + result = FishAudioTTS().execute( + {"text": "hello world", "model": "s1", "output_path": str(out)} + ) + + assert result.success is True + assert out.read_bytes() == b"AUDIOBYTES" + assert result.data["provider"] == "fish_audio" + assert result.data["model"] == "s1" + assert result.data["format"] == "mp3" + assert str(out) in result.artifacts + assert result.cost_usd > 0 + assert result.model == "fish-audio/s1" + + # model goes in the HTTP header, not the body + _, kwargs = mock_post.call_args + assert kwargs["headers"]["model"] == "s1" + assert kwargs["headers"]["Authorization"] == "Bearer sk-fish-secret" + assert kwargs["json"]["text"] == "hello world" + + def test_s2_pro_model_sent_in_header(self, api_key, tmp_path): + out = tmp_path / "s2.mp3" + with patch("requests.post", return_value=_FakeResponse()) as mock_post: + result = FishAudioTTS().execute( + {"text": "hello", "model": "s2-pro", "output_path": str(out)} + ) + + assert result.success is True + assert result.model == "fish-audio/s2-pro" + _, kwargs = mock_post.call_args + assert kwargs["headers"]["model"] == "s2-pro" + + def test_temperature_and_top_p_sent_in_body(self, api_key, tmp_path): + out = tmp_path / "expressive.mp3" + with patch("requests.post", return_value=_FakeResponse()) as mock_post: + FishAudioTTS().execute( + { + "text": "hello", + "model": "s2.1-pro", + "temperature": 0.9, + "top_p": 0.5, + "output_path": str(out), + } + ) + _, kwargs = mock_post.call_args + assert kwargs["json"]["temperature"] == 0.9 + assert kwargs["json"]["top_p"] == 0.5 + + def test_voice_id_maps_to_reference_id(self, api_key, tmp_path): + out = tmp_path / "clone.mp3" + with patch("requests.post", return_value=_FakeResponse()) as mock_post: + result = FishAudioTTS().execute( + { + "text": "cloned voice", + "model": "s1", + "voice_id": "voice-abc123", + "output_path": str(out), + } + ) + + assert result.success is True + assert result.data["reference_id"] == "voice-abc123" + _, kwargs = mock_post.call_args + assert kwargs["json"]["reference_id"] == "voice-abc123" + + def test_reference_id_takes_precedence_over_voice_id(self, api_key, tmp_path): + out = tmp_path / "clone.mp3" + with patch("requests.post", return_value=_FakeResponse()) as mock_post: + FishAudioTTS().execute( + { + "text": "x", + "model": "s1", + "reference_id": "primary", + "voice_id": "fallback", + "output_path": str(out), + } + ) + _, kwargs = mock_post.call_args + assert kwargs["json"]["reference_id"] == "primary" + + +# ---------------------------------------------------------------------- +# Cost — byte-based, not char-based +# ---------------------------------------------------------------------- + + +class TestCost: + def test_cost_uses_utf8_bytes(self): + tool = FishAudioTTS() + # 3 CJK chars = 9 UTF-8 bytes, larger than a 3-char ASCII cost. + cjk = tool.estimate_cost({"text": "你好吗", "model": "s1"}) + ascii_cost = tool.estimate_cost({"text": "abc", "model": "s1"}) + assert cjk > ascii_cost + + def test_s2_1_pro_free_costs_zero(self): + tool = FishAudioTTS() + assert tool.estimate_cost({"text": "same text here", "model": "s2.1-pro-free"}) == 0.0 + + +# ---------------------------------------------------------------------- +# Safety — never leak the API key +# ---------------------------------------------------------------------- + + +class TestKeyRedaction: + def test_error_does_not_leak_key(self, api_key): + def _boom(*args, **kwargs): + raise RuntimeError("upstream error for key sk-fish-secret at host") + + with patch("requests.post", side_effect=_boom): + result = FishAudioTTS().execute({"text": "hello", "model": "s1"}) + + assert result.success is False + assert "sk-fish-secret" not in result.error + assert "***" in result.error diff --git a/tools/audio/fish_audio_tts.py b/tools/audio/fish_audio_tts.py new file mode 100644 index 00000000..e5c802fc --- /dev/null +++ b/tools/audio/fish_audio_tts.py @@ -0,0 +1,326 @@ +"""fish.audio text-to-speech provider tool. + +fish.audio offers high-quality S1/S2-generation models and reference_id voice +cloning (reusing voice models created in the fish.audio playground). Strong +for expressive, multilingual narration with inline emotion tags on S2 models. +Billed per UTF-8 byte of input text. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class FishAudioTTS(BaseTool): + name = "fish_audio_tts" + version = "0.1.0" + tier = ToolTier.VOICE + capability = "tts" + provider = "fish_audio" + stability = ToolStability.EXPERIMENTAL + execution_mode = ExecutionMode.SYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = [] + install_instructions = ( + "Set FISH_AUDIO_API_KEY to an API key from https://fish.audio/go-api/api-keys/\n" + "Create voice models in the fish.audio playground and pass their id as\n" + "reference_id to reuse a cloned voice." + ) + fallback = "elevenlabs_tts" + fallback_tools = ["elevenlabs_tts", "google_tts", "openai_tts", "piper_tts"] + agent_skills = ["fish-audio-tts", "text-to-speech"] + + capabilities = [ + "text_to_speech", + "voice_selection", + "voice_cloning", + "multilingual", + ] + supports = { + "voice_cloning": True, + "multilingual": True, + "offline": False, + "native_audio": True, + "ssml": False, + } + best_for = [ + "high-quality voice-clone narration via reference_id", + "expressive S2-model read-throughs with inline emotion tags", + "multilingual narration", + ] + not_good_for = [ + "fully offline production", + "SSML markup control", + "deterministic reproducible output", + ] + + _VALID_MODELS = ("s1", "s2-pro", "s2.1-pro", "s2.1-pro-free") + + input_schema = { + "type": "object", + "required": ["text"], + "properties": { + "text": {"type": "string", "description": "Text to convert to speech"}, + "model": { + "type": "string", + "enum": list(_VALID_MODELS), + "description": ( + "Backend TTS model (sent as the 'model' HTTP header). Required — no " + "default. s2.1-pro = latest flagship (emotion tags, 80+ languages), " + "s2.1-pro-free = free tier for drafts, s2-pro = first S2 generation, " + "s1 = previous flagship kept for compatibility." + ), + }, + "reference_id": { + "type": "string", + "description": "fish.audio voice model id (from the playground) to clone/reuse.", + }, + "voice_id": { + "type": "string", + "description": "Alias for reference_id (selector compatibility). Used only when reference_id is absent.", + }, + "format": { + "type": "string", + "default": "mp3", + "enum": ["mp3", "wav", "pcm", "opus"], + "description": "Audio output format.", + }, + "mp3_bitrate": { + "type": "integer", + "default": 128, + "enum": [64, 128, 192], + "description": "MP3 bitrate (kbps). Only applies when format=mp3.", + }, + "chunk_length": { + "type": "integer", + "default": 300, + "minimum": 100, + "maximum": 300, + "description": "Text chunk length for streaming synthesis.", + }, + "normalize": { + "type": "boolean", + "default": True, + "description": "Normalize numbers/dates for stable pronunciation.", + }, + "latency": { + "type": "string", + "default": "normal", + "enum": ["low", "balanced", "normal"], + "description": ( + "Latency mode. 'normal' = best quality, 'balanced' trades a little " + "quality for speed, 'low' = fastest." + ), + }, + "temperature": { + "type": "number", + "default": 0.7, + "description": "Sampling temperature. Higher = more expressive, less stable.", + }, + "top_p": { + "type": "number", + "default": 0.7, + "description": "Nucleus sampling threshold.", + }, + "repetition_penalty": { + "type": "number", + "default": 1.2, + "description": "Penalty against repeated phrasing.", + }, + "sample_rate": { + "type": "integer", + "description": "Output sample rate in Hz (optional; API default when omitted).", + }, + "prosody": { + "type": "object", + "description": "Optional prosody controls, e.g. {\"speed\": 1.0, \"volume\": 0}.", + }, + "output_path": {"type": "string"}, + }, + } + + output_schema = { + "type": "object", + "properties": { + "output": {"type": "string"}, + "provider": {"type": "string"}, + "model": {"type": "string"}, + "reference_id": {"type": ["string", "null"]}, + "format": {"type": "string"}, + "text_length": {"type": "integer"}, + }, + } + artifact_schema = { + "type": "array", + "items": {"type": "string"}, + } + + resource_profile = ResourceProfile( + cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True + ) + retry_policy = RetryPolicy( + max_retries=2, backoff_seconds=2.0, retryable_errors=["rate_limit", "timeout"] + ) + idempotency_key_fields = ["text", "model", "reference_id", "format"] + side_effects = [ + "writes audio file to output_path", + "calls the fish.audio TTS API", + ] + user_visible_verification = [ + "Listen to generated audio for natural speech quality and voice-clone fidelity", + ] + quality_score = 0.9 + latency_p50_seconds = 6.0 + + API_URL = "https://api.fish.audio/v1/tts" + + # Approximate fish.audio pricing per UTF-8 byte of input text. fish bills by + # bytes, not characters — matters for CJK/emoji where one char is 3-4 bytes. + # Kept here (not pricing.yaml, which is fal-only) to mirror google_tts / + # doubao_tts. Verify against https://fish.audio pricing when refreshing. + _FALLBACK_RATES = { + "s1": 0.000015, # ~$15 / 1M bytes + "s2-pro": 0.000015, # ~$15 / 1M bytes + "s2.1-pro": 0.000015, # ~$15 / 1M bytes + "s2.1-pro-free": 0.0, # free tier + } + _DEFAULT_RATE = 0.000015 + + _EXT_MAP = {"mp3": "mp3", "wav": "wav", "pcm": "pcm", "opus": "opus"} + + def _get_api_key(self) -> str | None: + return os.environ.get("FISH_AUDIO_API_KEY") + + def get_status(self) -> ToolStatus: + if self._get_api_key(): + return ToolStatus.AVAILABLE + return ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + byte_count = len(str(inputs.get("text", "")).encode("utf-8")) + model = inputs.get("model", "") + rate = self._FALLBACK_RATES.get(model, self._DEFAULT_RATE) + return round(byte_count * rate, 4) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + api_key = self._get_api_key() + if not api_key: + return ToolResult( + success=False, + error="No fish.audio API key. " + self.install_instructions, + ) + + model = inputs.get("model") + if not model: + return ToolResult( + success=False, + error=( + "fish_audio_tts requires an explicit 'model'. " + f"Valid values: {', '.join(self._VALID_MODELS)}." + ), + ) + if model not in self._VALID_MODELS: + return ToolResult( + success=False, + error=( + f"Unknown fish.audio model '{model}'. " + f"Valid values: {', '.join(self._VALID_MODELS)}." + ), + ) + + start = time.time() + try: + result = self._generate(inputs, api_key=api_key, model=model) + except Exception as exc: + return ToolResult( + success=False, error=f"fish.audio TTS failed: {self._safe_error(exc, api_key)}" + ) + + result.duration_seconds = round(time.time() - start, 2) + if not result.cost_usd: + result.cost_usd = self.estimate_cost(inputs) + return result + + def _generate(self, inputs: dict[str, Any], *, api_key: str, model: str) -> ToolResult: + import requests + + text = inputs["text"] + fmt = inputs.get("format", "mp3") + reference_id = inputs.get("reference_id") or inputs.get("voice_id") + + body: dict[str, Any] = { + "text": text, + "format": fmt, + "normalize": bool(inputs.get("normalize", True)), + "latency": inputs.get("latency", "normal"), + "chunk_length": int(inputs.get("chunk_length", 300)), + } + if fmt == "mp3": + body["mp3_bitrate"] = int(inputs.get("mp3_bitrate", 128)) + if reference_id: + body["reference_id"] = reference_id + if inputs.get("temperature") is not None: + body["temperature"] = float(inputs["temperature"]) + if inputs.get("top_p") is not None: + body["top_p"] = float(inputs["top_p"]) + if inputs.get("repetition_penalty") is not None: + body["repetition_penalty"] = float(inputs["repetition_penalty"]) + if inputs.get("sample_rate") is not None: + body["sample_rate"] = int(inputs["sample_rate"]) + if isinstance(inputs.get("prosody"), dict): + body["prosody"] = inputs["prosody"] + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "model": model, + } + + response = requests.post(self.API_URL, headers=headers, json=body, timeout=120) + response.raise_for_status() + audio_content = response.content + + ext = self._EXT_MAP.get(fmt, "mp3") + output_path = Path(inputs.get("output_path", f"fish_audio_tts.{ext}")) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(audio_content) + + return ToolResult( + success=True, + data={ + "provider": self.provider, + "model": model, + "reference_id": reference_id, + "format": fmt, + "text_length": len(text), + "output": str(output_path), + }, + artifacts=[str(output_path)], + cost_usd=self.estimate_cost(inputs), + model=f"fish-audio/{model}", + ) + + @staticmethod + def _safe_error(exc: Exception, api_key: str | None) -> str: + message = str(exc) + if api_key: + message = message.replace(api_key, "***") + return message