diff --git a/.agents/skills/hyperframes-media/references/tts.md b/.agents/skills/hyperframes-media/references/tts.md index 22ec77f0..f2355293 100644 --- a/.agents/skills/hyperframes-media/references/tts.md +++ b/.agents/skills/hyperframes-media/references/tts.md @@ -65,6 +65,27 @@ node skills/hyperframes-media/scripts/heygen-tts.mjs --list # public starfish | Offline, no API key, fast iteration | **Kokoro** | | Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) | +## Expressive narration contract + +Before generating narration, write a compact voice-performance plan: + +- `performance_intent` - who the narrator is and how they should feel +- `pacing_profile` - contemplative, conversational, energetic, technical, or custom +- `energy_curve` - how the read changes across the piece +- `pause_policy` - where silence should happen and why +- section-level cues - `pace`, `energy`, `emphasis_words`, `pause_before_seconds`, + `pause_after_seconds`, and optional provider-ready text + +Do not rely on a vague instruction like "make it natural." Put the direction in +the text or provider settings: + +- Use short sentences and purposeful punctuation. +- Use `` to `` for important pauses when + the chosen provider supports SSML-style break tags. +- Generate a sample from the most performance-sensitive section before batching. +- If the sample sounds monotone, rushed, or ignores pauses, revise the plan or + provider settings before generating the rest. + ## ffmpeg requirement HeyGen + ElevenLabs return mp3. The CLI transcodes to wav when `--output` ends in `.wav` (the default and what downstream `ffprobe` + Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`. Without `ffmpeg` on PATH, `.wav` output from the cloud providers fails — install ffmpeg or use `.mp3`. diff --git a/.agents/skills/text-to-speech/SKILL.md b/.agents/skills/text-to-speech/SKILL.md index 6699a626..63a0e986 100644 --- a/.agents/skills/text-to-speech/SKILL.md +++ b/.agents/skills/text-to-speech/SKILL.md @@ -356,6 +356,25 @@ Rules: - Must have spaces before and after the tag - Self-closing tag format +## Expressive Voice Direction + +For narration, create a short voice-performance plan before generating audio: + +- narrator persona and emotional intent +- pacing profile +- energy curve across the script +- where pauses should land +- words or phrases that need emphasis + +Use concrete cues, not generic instructions. "Warm but decisive; pause before +the contrast; slow down on the final sentence" is useful. "Sound natural" is +not. + +When the selected voice supports pauses, put the most important pauses directly +in the text with break tags. Generate a sample from the most performance-heavy +section first, and do not batch-generate the rest if the sample sounds flat, +rushed, or ignores the intended breaks. + ## Best Practices 1. **Use `GET /v1/audio/voices`** to find compatible voices — not all voices from `GET /v2/voices` support Starfish TTS diff --git a/.claude/skills/text-to-speech/SKILL.md b/.claude/skills/text-to-speech/SKILL.md index 6699a626..63a0e986 100644 --- a/.claude/skills/text-to-speech/SKILL.md +++ b/.claude/skills/text-to-speech/SKILL.md @@ -356,6 +356,25 @@ Rules: - Must have spaces before and after the tag - Self-closing tag format +## Expressive Voice Direction + +For narration, create a short voice-performance plan before generating audio: + +- narrator persona and emotional intent +- pacing profile +- energy curve across the script +- where pauses should land +- words or phrases that need emphasis + +Use concrete cues, not generic instructions. "Warm but decisive; pause before +the contrast; slow down on the final sentence" is useful. "Sound natural" is +not. + +When the selected voice supports pauses, put the most important pauses directly +in the text with break tags. Generate a sample from the most performance-heavy +section first, and do not batch-generate the rest if the sample sounds flat, +rushed, or ignores the intended breaks. + ## Best Practices 1. **Use `GET /v1/audio/voices`** to find compatible voices — not all voices from `GET /v2/voices` support Starfish TTS diff --git a/pipeline_defs/animated-explainer.yaml b/pipeline_defs/animated-explainer.yaml index bb92a465..fb6c9f84 100644 --- a/pipeline_defs/animated-explainer.yaml +++ b/pipeline_defs/animated-explainer.yaml @@ -40,6 +40,7 @@ required_skills: - meta/checkpoint-protocol - meta/skill-creator - meta/animation-runtime-selector + - meta/voice-performance-director orchestration: mode: executive-producer @@ -130,6 +131,7 @@ stages: - "Enhancement cue density: 1 per 8-10 seconds" - "Narrative arc: hook -> setup -> build -> climax -> landing" - Speaker directions present for TTS + - Voice performance plan includes concrete pacing, pause, emphasis, and sample cues - Script incorporates research data points and cited facts success_criteria: - Schema-valid script with enhancement_cues per section @@ -186,6 +188,7 @@ stages: review_focus: - All asset files exist on disk - Narration covers all script sections + - Narration assets apply the approved voice performance plan and sample settings - Total cost within approved budget from proposal_packet - Image style consistency across all generated visuals success_criteria: diff --git a/pipeline_defs/animation.yaml b/pipeline_defs/animation.yaml index 953d27f7..e2361cc3 100644 --- a/pipeline_defs/animation.yaml +++ b/pipeline_defs/animation.yaml @@ -40,6 +40,7 @@ required_skills: - meta/reviewer - meta/checkpoint-protocol - meta/animation-runtime-selector + - meta/voice-performance-director orchestration: mode: executive-producer @@ -137,6 +138,7 @@ stages: - Script beats are concise and animation-friendly - On-screen text remains readable and restrained - Timing structure leaves room for visual holds + - Narration-led scripts include concrete voice performance cues - Word count within +/-10% of duration target from selected concept - Script incorporates research data points where relevant success_criteria: @@ -196,6 +198,7 @@ stages: - Asset production path is explicit per scene - Reusable motifs and templates are prepared and referenced - Missing tool paths are surfaced honestly + - Narration assets apply the approved voice performance plan and sample settings - Total cost within approved budget from proposal_packet - "Layer 3 skills read for EVERY generation tool before writing prompts (check agent_skills field)" - "Clip duration maximized (prefer 10s over 5s) to reduce API calls and cost" diff --git a/schemas/artifacts/asset_manifest.schema.json b/schemas/artifacts/asset_manifest.schema.json index 596b85d5..291b0b05 100644 --- a/schemas/artifacts/asset_manifest.schema.json +++ b/schemas/artifacts/asset_manifest.schema.json @@ -33,7 +33,24 @@ "generation_summary": { "type": "string", "description": "Brief summary of how the asset was generated or sourced" }, "provider": { "type": "string", "description": "Provider name (e.g. pixabay, google_imagen)" }, "license": { "type": "string", "description": "License type (e.g. Pixabay License, CC0)" }, - "original_url": { "type": "string", "description": "Source URL if downloaded from a stock service" } + "original_url": { "type": "string", "description": "Source URL if downloaded from a stock service" }, + "voice_performance": { + "type": "object", + "description": "Applied voice-performance contract for narration assets.", + "properties": { + "source_section_id": { "type": "string" }, + "delivery_cues_applied": { "type": "boolean" }, + "provider_text_used": { "type": "boolean" }, + "provider_settings": { + "type": "object", + "additionalProperties": true + }, + "sample_approved": { "type": "boolean" }, + "sample_path": { "type": "string" }, + "review_notes": { "type": "string" } + }, + "additionalProperties": false + } }, "additionalProperties": false } diff --git a/schemas/artifacts/proposal_packet.schema.json b/schemas/artifacts/proposal_packet.schema.json index 55a11699..c6af6308 100644 --- a/schemas/artifacts/proposal_packet.schema.json +++ b/schemas/artifacts/proposal_packet.schema.json @@ -183,7 +183,19 @@ "provider": { "type": "string" }, "voice_id": { "type": "string" }, "rationale": { "type": "string" }, - "estimated_cost_usd": { "type": "number", "minimum": 0 } + "estimated_cost_usd": { "type": "number", "minimum": 0 }, + "delivery_style": { + "type": "string", + "description": "Plain-language voice direction, e.g. warm expert, expressive narrator, crisp tutorial." + }, + "pacing_policy": { + "type": "string", + "description": "How the voice should use pauses, speed, and emphasis across the piece." + }, + "sample_approval_required": { + "type": "boolean", + "description": "True when a TTS sample must be approved before batch narration generation." + } }, "additionalProperties": false }, diff --git a/schemas/artifacts/script.schema.json b/schemas/artifacts/script.schema.json index 2a0ba40d..dfa66cce 100644 --- a/schemas/artifacts/script.schema.json +++ b/schemas/artifacts/script.schema.json @@ -9,6 +9,28 @@ "version": { "type": "string", "const": "1.0" }, "title": { "type": "string" }, "total_duration_seconds": { "type": "number", "minimum": 1 }, + "voice_performance": { + "type": "object", + "description": "Narration delivery contract used by TTS asset generation.", + "properties": { + "performance_intent": { "type": "string" }, + "pacing_profile": { + "type": "string", + "enum": ["contemplative", "conversational", "energetic", "technical", "cinematic", "custom"] + }, + "energy_curve": { "type": "string" }, + "pause_policy": { "type": "string" }, + "sample_section_id": { + "type": "string", + "description": "Most performance-sensitive section to use for TTS sample approval." + }, + "provider_notes": { + "type": "object", + "additionalProperties": { "type": "string" } + } + }, + "additionalProperties": false + }, "sections": { "type": "array", "items": { @@ -21,6 +43,29 @@ "start_seconds": { "type": "number", "minimum": 0 }, "end_seconds": { "type": "number", "minimum": 0 }, "speaker_directions": { "type": "string" }, + "delivery_cues": { + "type": "object", + "description": "Structured voice-performance cues that TTS generation must apply.", + "properties": { + "pace": { + "type": "string", + "enum": ["slow", "measured", "conversational", "brisk", "fast", "custom"] + }, + "energy": { "type": "string" }, + "emphasis_words": { + "type": "array", + "items": { "type": "string" } + }, + "pause_before_seconds": { "type": "number", "minimum": 0 }, + "pause_after_seconds": { "type": "number", "minimum": 0 }, + "delivery_note": { "type": "string" }, + "provider_text": { + "type": "string", + "description": "Provider-ready narration text, including purposeful punctuation or SSML break tags when supported." + } + }, + "additionalProperties": false + }, "enhancement_cues": { "type": "array", "items": { diff --git a/skills/meta/voice-performance-director.md b/skills/meta/voice-performance-director.md new file mode 100644 index 00000000..72739664 --- /dev/null +++ b/skills/meta/voice-performance-director.md @@ -0,0 +1,93 @@ +# Voice Performance Director + +Use this meta skill whenever a pipeline will generate narration with TTS. + +The goal is to make generated narration sound directed, not merely read. Do not +leave expressiveness as "read naturally" in a prompt. Carry a concrete voice +performance plan from script to asset generation, then verify it with a sample. + +## Required Contract + +Every narration-led script should include a top-level `voice_performance` +object and section-level `delivery_cues` where the schema allows it. + +Top-level voice performance: + +```json +{ + "performance_intent": "Warm, decisive product narrator with human pauses.", + "pacing_profile": "conversational", + "energy_curve": "measured hook, warmer middle, more deliberate close", + "pause_policy": "Use short pauses after setup lines and longer pauses before reversals or important claims.", + "provider_notes": { + "openai": "Use instructions for emotional arc and emphasis.", + "google_tts": "Use SSML input with break tags when the selected voice supports it.", + "elevenlabs": "Use lower stability and moderate style for expressive narration." + } +} +``` + +Section-level delivery cues: + +```json +{ + "pace": "measured", + "energy": "curious", + "emphasis_words": ["not", "process"], + "pause_before_seconds": 0.2, + "pause_after_seconds": 0.7, + "delivery_note": "Set up the contrast, then slow down on the final phrase.", + "provider_text": "This is not just another tool. It is a process." +} +``` + +## Writing Rules + +- Write spoken language, not essay language. Prefer short sentences, light + contractions, and clear punctuation. +- Use silence as structure. Add a pause before reversals, after surprising + claims, and before the final takeaway. +- Keep pause tags purposeful. Too many breaks sound theatrical and slow. +- Avoid generic directions such as "natural", "engaging", or "expressive" + unless they are paired with exact pace, emphasis, pause, or energy cues. +- Prefer one delivery idea per section. If a section needs three emotional + turns, split it. + +## Provider Mapping + +- OpenAI TTS: use `model: "gpt-4o-mini-tts"` when sending `instructions`. + Put the emotional arc, pacing, emphasis, and role in `instructions`; keep the + input text clean but punctuated. Do not send `instructions` to `tts-1` or + `tts-1-hd`. +- Google TTS: use `input_type: "ssml"` only when adding break tags or other + SSML. The tool maps this to Google `input.ssml` and wraps the utterance in + `...` when needed. Keep `speaking_rate` in Google's supported + `0.25..2.0` range and pitch in `-20..20`. +- ElevenLabs: use lower `stability` for more variation, moderate `style` for + expressiveness, `speed` in the provider's `0.7..1.2` range, and keep + `similarity_boost` high enough to preserve the voice. +- Offline/basic voices: rely on punctuation, shorter sentences, and explicit + segment splitting because provider-level emotion controls may be unavailable. + +## Sample Gate + +Before batch narration generation: + +1. Generate a sample from the most performance-sensitive section, not + automatically the first section. +2. Verify voice, pace, pauses, emphasis, and emotional arc. +3. If the sample is flat, adjust the `voice_performance` plan or provider + settings before generating the rest. +4. Record the approved sample path and provider settings in the asset manifest. + +## Failure Conditions + +Treat these as quality failures: + +- A narration-led script has no `voice_performance` plan. +- Section directions only say "read naturally" or "expressive" with no concrete + pause, emphasis, pace, or energy cue. +- TTS provider, voice, speed, or model changes after sample approval without a + new sample. +- Final narration is generated from raw script text while structured + `provider_text` or `delivery_cues` were present. diff --git a/skills/pipelines/animation/asset-director.md b/skills/pipelines/animation/asset-director.md index 180a0f5b..26830c00 100644 --- a/skills/pipelines/animation/asset-director.md +++ b/skills/pipelines/animation/asset-director.md @@ -49,7 +49,7 @@ Prefer the lowest-variance useful path: Before batch-generating assets, produce one sample of each expensive type and show the user: -1. **TTS sample** (if narration-led): Generate one section. Confirm voice and tone before batching. +1. **TTS sample** (if narration-led): Generate `script.voice_performance.sample_section_id` when present; otherwise choose the section with the strongest emotional or pacing change. Confirm voice, pace, pauses, emphasis, and tone before batching. 2. **Visual sample**: Generate one representative scene visual (diagram, illustration, or motion background). Confirm style and quality before batching the rest. If rejected, adjust parameters and retry (max 3 iterations). Do not batch until approved. @@ -100,7 +100,12 @@ Create once: ### 3. Narration Is Optional, But The Plan Must Be Explicit -If the project is narration-led, produce or source narration. If it is text-led or music-led, say so clearly in metadata. +If the project is narration-led, produce or source narration. Read +`skills/meta/voice-performance-director.md`, then apply `script.voice_performance` +and each section's `delivery_cues` when building TTS requests. Use +`provider_text` when present, map cues to provider controls, and record the +applied settings on each narration asset. If it is text-led or music-led, say so +clearly in metadata. ### 4. Use Metadata For Feasibility Truth @@ -109,6 +114,7 @@ Recommended metadata keys: - `tool_path_map` - `reusable_assets` - `narration_assets` +- `voice_performance`: sample approval path, provider settings, and whether delivery cues were applied - `scene_asset_index` - `blocked_assets` @@ -117,7 +123,8 @@ Recommended metadata keys: - the asset path is explicit per scene, - reusable assets are actually reused, - missing capabilities are surfaced honestly, -- every referenced file exists. +- every referenced file exists, +- narration-led assets apply the approved voice-performance settings. ### Mid-Production Fact Verification @@ -135,6 +142,8 @@ the AI model's training data — it may be wrong or outdated. - Using high-variance generation when a deterministic asset would work better. - Rebuilding the same title or label system repeatedly. - Hiding failed asset paths instead of reporting them. +- Treating TTS as raw text-to-audio. Narration-led animation needs pauses, + emphasis, and pace cues carried from the script into the generated audio. - Treating "consistency" as "same prompt every time." Good animation keeps a recognizable world while still letting each beat feel fresh. diff --git a/skills/pipelines/animation/script-director.md b/skills/pipelines/animation/script-director.md index 0fa96710..11f56ef1 100644 --- a/skills/pipelines/animation/script-director.md +++ b/skills/pipelines/animation/script-director.md @@ -11,6 +11,7 @@ This stage turns the approved proposal into animation-ready beats. The script mu | Schema | `schemas/artifacts/script.schema.json` | Artifact validation | | Prior artifact | `proposal_packet` from Proposal Director | Selected concept, animation mode, target duration, reuse strategy | | Optional artifact | `research_brief` from Research Director | Data points, audience insights, accuracy constraints | +| Meta skill | `skills/meta/voice-performance-director.md` | Structured TTS delivery cues for natural, expressive narration | | Tools | `transcriber` | Optional source transcript support | ## Process @@ -41,6 +42,11 @@ Each section should express ONE clear visual idea: - **Comparison** — show two things side by side (split screen or sequential) - **Conclusion** — land the insight (hold + emphasis) +If the piece is narration-led, also create `voice_performance` and section +`delivery_cues` using `skills/meta/voice-performance-director.md`. Motion-heavy +videos need vocal breathing room: mark pauses where the viewer must absorb an +animation, and mark emphasis where a reveal, transform, or comparison lands. + **Animation mode affects writing style:** | Mode | Writing Style | @@ -82,6 +88,7 @@ Recommended metadata keys per section: - `narration_plan`: how narration relates to visual (describes / complements / silent) - `visual_priority`: what the viewer should focus on (the animation, the text, the data) - `hold_time_seconds`: minimum visual hold time after this section's content +- `delivery_cues`: pace, energy, emphasis, pause timing, and provider-ready narration text for TTS - `data_source`: if this section uses a research data point, reference it ### 6. Research Integration @@ -101,6 +108,7 @@ Before submitting the script, verify: - [ ] Every section supports ONE strong visual idea - [ ] On-screen text is concise (phrases, not paragraphs) - [ ] Timing is animation-friendly (holds budgeted) +- [ ] Narration-led sections include concrete delivery cues and a voice-performance sample section - [ ] Word count is within ±10% of target duration - [ ] Animation mode is respected in writing style - [ ] Research data points are integrated (if research_brief available) diff --git a/skills/pipelines/explainer/asset-director.md b/skills/pipelines/explainer/asset-director.md index ac9f091a..b98c358d 100644 --- a/skills/pipelines/explainer/asset-director.md +++ b/skills/pipelines/explainer/asset-director.md @@ -70,7 +70,7 @@ Before generating anything: Before batch-generating assets, produce one sample of each expensive asset type and present them to the user for approval: -1. **TTS sample**: Generate narration for the first script section only. Play it for the user. Confirm voice, pace, and tone are acceptable before generating the rest. +1. **TTS sample**: Generate narration for `script.voice_performance.sample_section_id` when present; otherwise pick the section with the most demanding delivery. Play it for the user. Confirm voice, pace, pauses, emphasis, and tone are acceptable before generating the rest. 2. **Image sample**: Generate one image for the most representative scene. Show it to the user. Confirm the style, quality, and prompt approach before batch-generating all images. 3. **Music sample** (if using `music_gen`): Generate one short clip. Confirm mood and energy before committing. @@ -85,13 +85,24 @@ This step typically costs $0.03–0.08 total and prevents $1–3 of wasted gener For each script section: 1. Extract the narration text -2. Apply speaker directions from the script (pace, emphasis, emotion) -3. Apply the playbook's `audio.voice_style` -4. Generate using `tts_selector` — it auto-routes to the best available TTS provider based on user preference and availability. Check the registry's `best_for` fields to understand each provider's strengths. -5. Verify the audio file exists and duration matches expected timing (±15%) +2. Read `script.voice_performance` and section `delivery_cues` +3. Use `delivery_cues.provider_text` when present; otherwise transform the section text with purposeful punctuation and break tags only when the selected provider supports them +4. Apply speaker directions from the script (pace, emphasis, emotion) +5. Apply the playbook's `audio.voice_style` +6. Map cues to provider parameters: + - OpenAI: `instructions` only with `model: "gpt-4o-mini-tts"`; use `response_format` for output format + - Google TTS: `input_type: "ssml"` when using `` tags, plus `speaking_rate` in `0.25..2.0` and `pitch` in `-20..20` + - ElevenLabs: `stability`, `similarity_boost`, `style`, `speed`, and `use_speaker_boost` +7. Generate using `tts_selector` — it auto-routes to the best available TTS provider based on user preference and availability. Check the registry's `best_for` fields to understand each provider's strengths. +8. Record the applied `voice_performance` metadata on each narration asset +9. Verify the audio file exists and duration matches expected timing (±15%) **Pronunciation guide**: If the script contains technical terms, jargon, or names with non-obvious pronunciation, include a pronunciation map in the TTS request. +**Flat voice failure:** If the approved voice sounds monotone, robotic, rushed, +or ignores intended pauses, do not batch the remaining sections. Revise the +`voice_performance` plan or provider parameters and regenerate the sample. + ### Step 4: Generate Visual Assets Process asset tasks grouped by tool for efficiency: @@ -204,6 +215,8 @@ Assemble all generated assets into the manifest: **Quality check:** - [ ] Narration durations within ±15% of expected timing +- [ ] Narration assets record `voice_performance.delivery_cues_applied` +- [ ] Approved TTS sample uses the same provider, voice, and expressive settings as the batch - [ ] Images match the playbook's style (review consistency anchors) - [ ] Diagrams are legible and complete - [ ] Total cost within budget @@ -242,6 +255,7 @@ the AI model's training data — it may be wrong or outdated. - **Generating before checking budget**: Always estimate total cost first. A 60-second video with 15 images can burn $3+ quickly. - **Inconsistent image style**: Each image_selector call is independent. Use consistent anchors, but adapt them per scene. If you paste the same style prefix into every prompt, the video will feel machine-made and repetitive. - **Ignoring narration timing**: If TTS produces 12s of audio for a 10s section, the edit phase will struggle. Check durations. +- **Ignoring delivery cues**: Generating raw script text when `provider_text` or `delivery_cues` exist will flatten the read. Apply the voice-performance contract first. - **Missing pronunciation guide**: "PostgreSQL" or "Kubernetes" will be mispronounced without explicit guidance. - **One retry then give up**: If an image doesn't match, refine the prompt specifically — don't just retry the same prompt. - **AI-generating images with exact text (CTA, business names, contact info)**: AI image models frequently hallucinate wrong text — wrong business name, wrong phone number, misspelled words. **Never use AI image generation for scenes where text must be verbatim.** Use Remotion `text_card` type instead. This applies to: CTA screens, title cards with business names, contact info overlays, legal disclaimers. If a scene's `type` is `text_card` in the scene plan, do NOT generate an image for it — skip it and let the compose stage render it natively in Remotion. diff --git a/skills/pipelines/explainer/script-director.md b/skills/pipelines/explainer/script-director.md index 80c61606..42005743 100644 --- a/skills/pipelines/explainer/script-director.md +++ b/skills/pipelines/explainer/script-director.md @@ -14,6 +14,7 @@ The script is the backbone of the video. Every visual, every scene, every audio | Prior artifact | `proposal_packet` | Selected concept with title, hook, key_points, core_message, tone, narrative_structure, duration | | Prior artifact | `research_brief` (optional but high-value) | Data points, audience insights, expert quotes — ground the script in real facts | | Playbook | Active style playbook from `proposal_packet.selected_concept.suggested_playbook` | Voice style, pacing rules | +| Meta skill | `skills/meta/voice-performance-director.md` | Structured TTS delivery cues for natural, expressive narration | | Layer 3 | TTS provider skills (check `agent_skills` on the selected TTS tool) | TTS capabilities for speaker directions | ## Process @@ -77,6 +78,11 @@ Map each of the brief's `key_points` to a specific section in the BUILD phase. ### Step 4: Write the Script +Before writing sections, create a top-level `voice_performance` plan using +`skills/meta/voice-performance-director.md`. The plan must describe the vocal +intent, pacing profile, energy curve, pause policy, and which section should be +used for TTS sample approval. Do not leave this as a vague "natural voice" note. + Write each section with these fields: ```json @@ -87,6 +93,14 @@ Write each section with these fields: "start_seconds": 0, "end_seconds": 5, "speaker_directions": "Emphasize 'every single row' with measured pacing. Brief pause before the question.", + "delivery_cues": { + "pace": "measured", + "energy": "curious", + "emphasis_words": ["every", "single"], + "pause_after_seconds": 0.6, + "delivery_note": "Let the repetition feel intentional, then soften into the question.", + "provider_text": "Your database searches every single row. Every. Single. One. What if it didn't have to?" + }, "enhancement_cues": [ { "type": "animation", @@ -117,7 +131,8 @@ Count your words. If you're 20%+ over budget, the TTS will either rush or exceed #### Speaker Directions -Write directions that TTS can actually implement. Reference ElevenLabs capabilities: +Write directions that TTS can actually implement. Prefer structured +`delivery_cues` over prose-only `speaker_directions`: | Direction | TTS Implementation | |-----------|-------------------| @@ -129,6 +144,12 @@ Write directions that TTS can actually implement. Reference ElevenLabs capabilit Avoid directions TTS can't do: "smile while speaking", "gesture toward screen", "look at camera." +**Expressive narration rule:** every narration-led section must include at +least two concrete cues among `pace`, `energy`, `emphasis_words`, +`pause_before_seconds`, `pause_after_seconds`, `delivery_note`, or +`provider_text`. Use `provider_text` when punctuation or SSML break tags are +needed to make the read sound human. + #### Enhancement Cues Every section should have at least one enhancement cue. These tell the Scene Planner and Asset Generator what visuals to create. @@ -162,6 +183,7 @@ Read the active style playbook and verify: |----------------|---------------| | `identity.pace` | Match word density. `contemplative` = fewer words, longer pauses | | `audio.voice_style` | Shape tone of speaker directions | +| `voice_performance` | Confirm pacing, pauses, and energy curve are explicit enough for TTS | | `motion.pacing_rules` | E.g., "hold establishing shots for 2s minimum" affects section timing | | `identity.mood` | Word choice: `warm` uses casual language; `professional` uses precise language | @@ -175,6 +197,7 @@ Score your script (1-5): | **Word count accuracy** | Within ±10% of target for the duration? | | **Narrative flow** | Does each section build on the last? "Therefore/but" not "and then"? | | **Enhancement density** | At least one cue every 8-10 seconds? | +| **Voice performance** | Are pauses, emphasis, pace, and sample section explicit? | | **Jargon management** | Technical terms explained or have pronunciation guides? | | **Climax payoff** | Does the aha moment deliver on the hook's promise? | | **CTA relevance** | Is the call to action specific and actionable? | diff --git a/tests/tools/test_hyperframes_compose.py b/tests/tools/test_hyperframes_compose.py index 35655880..dc465c3c 100644 --- a/tests/tools/test_hyperframes_compose.py +++ b/tests/tools/test_hyperframes_compose.py @@ -1062,6 +1062,69 @@ def test_proposal_packet_schema_accepts_render_runtime(): assert props["render_runtime"]["enum"] == ["remotion", "hyperframes", "ffmpeg"] +def test_schemas_accept_voice_performance_contract(): + root = Path(__file__).resolve().parent.parent.parent + + script_schema = json.loads( + (root / "schemas" / "artifacts" / "script.schema.json").read_text( + encoding="utf-8" + ) + ) + assert "voice_performance" in script_schema["properties"] + section_props = script_schema["properties"]["sections"]["items"]["properties"] + assert "delivery_cues" in section_props + assert "provider_text" in section_props["delivery_cues"]["properties"] + + proposal_schema = json.loads( + (root / "schemas" / "artifacts" / "proposal_packet.schema.json").read_text( + encoding="utf-8" + ) + ) + voice_selection = proposal_schema["properties"]["production_plan"]["properties"][ + "voice_selection" + ]["properties"] + assert "delivery_style" in voice_selection + assert "pacing_policy" in voice_selection + assert "sample_approval_required" in voice_selection + + asset_schema = json.loads( + (root / "schemas" / "artifacts" / "asset_manifest.schema.json").read_text( + encoding="utf-8" + ) + ) + asset_props = asset_schema["properties"]["assets"]["items"]["properties"] + assert "voice_performance" in asset_props + assert "provider_settings" in asset_props["voice_performance"]["properties"] + + +def test_tts_provider_contracts_match_supported_fields(): + from tools.audio.elevenlabs_tts import ElevenLabsTTS + from tools.audio.google_tts import GoogleTTS + from tools.audio.openai_tts import OpenAITTS + + google_props = GoogleTTS.input_schema["properties"] + assert google_props["input_type"]["enum"] == ["text", "ssml"] + assert google_props["speaking_rate"]["maximum"] == 2.0 + assert google_props["pitch"]["minimum"] == -20.0 + assert google_props["pitch"]["maximum"] == 20.0 + + openai_props = OpenAITTS.input_schema["properties"] + assert "response_format" in openai_props + assert {"mp3", "opus", "aac", "flac", "wav", "pcm"}.issubset( + set(openai_props["response_format"]["enum"]) + ) + assert OpenAITTS._supports_instructions("gpt-4o-mini-tts") + assert not OpenAITTS._supports_instructions("tts-1") + assert not OpenAITTS._supports_instructions("tts-1-hd") + + eleven_props = ElevenLabsTTS.input_schema["properties"] + assert {"stability", "similarity_boost", "style", "speed", "use_speaker_boost"}.issubset( + set(eleven_props) + ) + assert eleven_props["speed"]["minimum"] == 0.7 + assert eleven_props["speed"]["maximum"] == 1.2 + + def test_edit_decisions_schema_accepts_render_runtime(): schema_path = ( Path(__file__).resolve().parent.parent.parent diff --git a/tools/audio/elevenlabs_tts.py b/tools/audio/elevenlabs_tts.py index d0207f34..85c34e63 100644 --- a/tools/audio/elevenlabs_tts.py +++ b/tools/audio/elevenlabs_tts.py @@ -96,6 +96,16 @@ class ElevenLabsTTS(BaseTool): "minimum": 0, "maximum": 1, }, + "speed": { + "type": "number", + "default": 1.0, + "minimum": 0.7, + "maximum": 1.2, + }, + "use_speaker_boost": { + "type": "boolean", + "default": True, + }, "output_path": {"type": "string"}, "output_format": { "type": "string", @@ -109,7 +119,16 @@ class ElevenLabsTTS(BaseTool): cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True ) retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) - idempotency_key_fields = ["text", "voice_id", "model_id"] + idempotency_key_fields = [ + "text", + "voice_id", + "model_id", + "stability", + "similarity_boost", + "style", + "speed", + "use_speaker_boost", + ] side_effects = ["writes audio file to output_path", "calls ElevenLabs API"] user_visible_verification = ["Listen to generated audio for natural speech quality"] @@ -145,6 +164,13 @@ class ElevenLabsTTS(BaseTool): voice_id = inputs.get("voice_id", self.DEFAULT_VOICE_ID) model_id = inputs.get("model_id", "eleven_multilingual_v2") output_format = inputs.get("output_format", "mp3_44100_128") + voice_settings = { + "stability": inputs.get("stability", 0.5), + "similarity_boost": inputs.get("similarity_boost", 0.75), + "style": inputs.get("style", 0.0), + "speed": inputs.get("speed", 1.0), + "use_speaker_boost": inputs.get("use_speaker_boost", True), + } response = requests.post( f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}", @@ -156,11 +182,7 @@ class ElevenLabsTTS(BaseTool): json={ "text": text, "model_id": model_id, - "voice_settings": { - "stability": inputs.get("stability", 0.5), - "similarity_boost": inputs.get("similarity_boost", 0.75), - "style": inputs.get("style", 0.0), - }, + "voice_settings": voice_settings, }, params={"output_format": output_format}, timeout=120, @@ -178,6 +200,7 @@ class ElevenLabsTTS(BaseTool): "provider": self.provider, "model": model_id, "voice_id": voice_id, + "voice_settings": voice_settings, "text_length": len(text), "output": str(output_path), "format": output_format, diff --git a/tools/audio/google_tts.py b/tools/audio/google_tts.py index 91e48485..b6218a1c 100644 --- a/tools/audio/google_tts.py +++ b/tools/audio/google_tts.py @@ -78,6 +78,12 @@ class GoogleTTS(BaseTool): "required": ["text"], "properties": { "text": {"type": "string", "description": "Text to convert to speech"}, + "input_type": { + "type": "string", + "default": "text", + "enum": ["text", "ssml"], + "description": "Set to 'ssml' when text contains SSML tags such as or .", + }, "voice": { "type": "string", "default": "en-US-Chirp3-HD-Orus", @@ -92,7 +98,7 @@ class GoogleTTS(BaseTool): "type": "number", "default": 1.0, "minimum": 0.25, - "maximum": 4.0, + "maximum": 2.0, "description": "Speaking speed. 1.0 = normal, 0.5 = half speed, 2.0 = double speed", }, "pitch": { @@ -116,7 +122,7 @@ class GoogleTTS(BaseTool): cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True ) retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) - idempotency_key_fields = ["text", "voice", "language_code", "speaking_rate", "pitch"] + idempotency_key_fields = ["text", "input_type", "voice", "language_code", "speaking_rate", "pitch"] side_effects = ["writes audio file to output_path", "calls Google Cloud TTS API"] user_visible_verification = ["Listen to generated audio for natural speech quality"] @@ -200,14 +206,33 @@ class GoogleTTS(BaseTool): import requests text = inputs["text"] + input_type = inputs.get("input_type", "text") voice_name = inputs.get("voice", "en-US-Chirp3-HD-Orus") language_code = inputs.get("language_code", "en-US") speaking_rate = inputs.get("speaking_rate", 1.0) pitch = inputs.get("pitch", 0.0) audio_encoding = inputs.get("audio_encoding", "MP3") + if not 0.25 <= speaking_rate <= 2.0: + return ToolResult( + success=False, + error="Google TTS speaking_rate must be between 0.25 and 2.0.", + ) + if not -20.0 <= pitch <= 20.0: + return ToolResult( + success=False, + error="Google TTS pitch must be between -20.0 and 20.0 semitones.", + ) + + if input_type == "ssml": + stripped = text.strip() + ssml = stripped if stripped.startswith("{stripped}" + synthesis_input = {"ssml": ssml} + else: + synthesis_input = {"text": text} + payload = { - "input": {"text": text}, + "input": synthesis_input, "voice": { "languageCode": language_code, "name": voice_name, @@ -253,6 +278,7 @@ class GoogleTTS(BaseTool): "voice": voice_name, "language_code": language_code, "text_length": len(text), + "input_type": input_type, "output": str(output_path), "format": audio_encoding, "speaking_rate": speaking_rate, diff --git a/tools/audio/openai_tts.py b/tools/audio/openai_tts.py index 29ebd7d4..52d4259c 100644 --- a/tools/audio/openai_tts.py +++ b/tools/audio/openai_tts.py @@ -80,10 +80,24 @@ class OpenAITTS(BaseTool): "type": "string", "default": "mp3", "enum": ["mp3", "wav", "pcm"], + "description": "Backward-compatible alias for response_format.", + }, + "response_format": { + "type": "string", + "default": "mp3", + "enum": ["mp3", "opus", "aac", "flac", "wav", "pcm"], + "description": "OpenAI speech response_format.", }, "instructions": { "type": "string", - "description": "Optional delivery instructions for the voice", + "description": "Optional delivery instructions. Supported by gpt-4o-mini-tts.", + }, + "speed": { + "type": "number", + "default": 1.0, + "minimum": 0.25, + "maximum": 4.0, + "description": "OpenAI speech speed multiplier.", }, "output_path": {"type": "string"}, }, @@ -93,7 +107,7 @@ class OpenAITTS(BaseTool): cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=50, network_required=True ) retry_policy = RetryPolicy(max_retries=2, retryable_errors=["rate_limit", "timeout"]) - idempotency_key_fields = ["text", "voice", "model", "format"] + idempotency_key_fields = ["text", "voice", "model", "format", "response_format", "instructions", "speed"] side_effects = ["writes audio file to output_path", "calls OpenAI API"] user_visible_verification = ["Listen to generated audio for intelligibility and tone"] @@ -105,6 +119,10 @@ class OpenAITTS(BaseTool): def estimate_cost(self, inputs: dict[str, Any]) -> float: return round(len(inputs.get("text", "")) * 0.000015, 4) + @staticmethod + def _supports_instructions(model: str) -> bool: + return model.startswith("gpt-4o-mini-tts") + def execute(self, inputs: dict[str, Any]) -> ToolResult: if not os.environ.get("OPENAI_API_KEY"): return ToolResult(success=False, error="No OpenAI API key. " + self.install_instructions) @@ -124,11 +142,20 @@ class OpenAITTS(BaseTool): from tools.analysis.audio_probe import probe_duration - client = OpenAI() text = inputs["text"] model = inputs.get("model", "gpt-4o-mini-tts") voice = inputs.get("voice", "alloy") - fmt = inputs.get("format", "mp3") + fmt = inputs.get("response_format") or inputs.get("format", "mp3") + if inputs.get("instructions") and not self._supports_instructions(model): + return ToolResult( + success=False, + error=( + "OpenAI TTS instructions are only supported by " + "gpt-4o-mini-tts. Use that model or omit instructions." + ), + ) + + client = OpenAI() output_path = Path(inputs.get("output_path", f"openai_tts.{fmt}")) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -155,6 +182,9 @@ class OpenAITTS(BaseTool): "model": model, "voice": voice, "format": fmt, + "response_format": fmt, + "instructions": inputs.get("instructions"), + "speed": inputs.get("speed", 1.0), "text_length": len(text), "audio_duration_seconds": round(audio_duration, 2) if audio_duration else None, "output": str(output_path), diff --git a/tools/audio/tts_selector.py b/tools/audio/tts_selector.py index 2d69af20..f8010827 100644 --- a/tools/audio/tts_selector.py +++ b/tools/audio/tts_selector.py @@ -61,6 +61,43 @@ class TTSSelector(BaseTool): "type": "number", "minimum": 0, "maximum": 1, "description": "Style exaggeration (ElevenLabs). Higher = more expressive.", }, + "instructions": { + "type": "string", + "description": "Provider-level delivery instructions for expressive narration when supported.", + }, + "speaking_rate": { + "type": "number", + "minimum": 0.25, + "maximum": 2.0, + "description": "Google-style speakingRate control. Use speed for OpenAI/ElevenLabs-style controls.", + }, + "speed": { + "type": "number", + "minimum": 0.25, + "maximum": 4.0, + "description": "Alias for speaking speed used by some providers.", + }, + "pitch": { + "type": "number", + "minimum": -50, + "maximum": 50, + "description": "Provider-specific pitch control. Google TTS accepts -20..20; HeyGen-style providers may accept wider ranges.", + }, + "input_type": { + "type": "string", + "enum": ["text", "ssml"], + "default": "text", + "description": "Use 'ssml' only when the selected provider supports tags such as .", + }, + "voice_performance": { + "type": "object", + "description": "Structured voice-performance plan or section delivery cues from the script artifact.", + }, + "sample_mode": { + "type": "boolean", + "default": False, + "description": "True when generating an approval sample before batch narration.", + }, "output_format": { "type": "string", "description": "Audio output format (e.g. mp3_44100_128). Passed through to provider.",