From e2d939ed0befad1de0320121617e7c24549addb8 Mon Sep 17 00:00:00 2001 From: pika-ai-code Date: Fri, 1 May 2026 16:39:48 +0000 Subject: [PATCH] Initial public release of the Pika Claude Code plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Pika Claude Code plugin ships with three curated /pika:* skills and 42 atomic MCP tools at https://mcp.pika.me/api/mcp: - /pika:podcast — 1-minute two-host podcast video for any URL or topic - /pika:explainer — ~60–80s explainer video for any URL - /pika:ugc-ads — 15s creator-style multi-cut UGC ad in 9:16 vertical Installs on every Claude Code surface — CLI and Desktop (macOS / Windows app). The 42 atomic MCP tools also work with any other MCP client (Claude.ai chat connectors, Cursor, Codex, etc.) at the raw endpoint https://mcp.pika.me/api/mcp. License: Apache 2.0. --- .claude-plugin/marketplace.json | 25 + .claude-plugin/plugin.json | 9 + .mcp.json | 8 + CHANGELOG.md | 23 + LICENSE | 202 ++++ README.md | 358 ++++++ SECURITY.md | 37 + skills/baseball-trend/SKILL.md | 183 +++ skills/explainer/SKILL.md | 439 +++++++ skills/podcast/SKILL.md | 173 +++ skills/ugc-ads/SKILL.md | 209 ++++ tools-manifest.json | 1969 +++++++++++++++++++++++++++++++ 12 files changed, 3635 insertions(+) create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json create mode 100644 .mcp.json create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 skills/baseball-trend/SKILL.md create mode 100644 skills/explainer/SKILL.md create mode 100644 skills/podcast/SKILL.md create mode 100644 skills/ugc-ads/SKILL.md create mode 100644 tools-manifest.json diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..3209791 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,25 @@ +{ + "name": "pika-plugins", + "metadata": { + "description": "Pika Claude Code plugin marketplace — distributes /pika:podcast (any URL or topic → 1-minute two-host video), /pika:explainer (any URL → 60–80s walkthrough video), and /pika:ugc-ads (product URL → 15s creator-style multi-cut UGC ad)." + }, + "owner": { + "name": "Pika Labs", + "email": "support@pika.art" + }, + "plugins": [ + { + "name": "pika", + "description": "Pika-fy your Claude — give Claude a face, name, voice, and personality, plus a full creative stack of image, video, audio, and editing tools. 3 curated /pika:* slash commands backed by 42 atomic media tools at mcp.pika.me.", + "source": "./", + "category": "creative", + "version": "1.0.0", + "author": { + "name": "Pika Labs", + "email": "support@pika.art" + }, + "homepage": "https://pika.me", + "tags": ["pika", "video", "podcast", "explainer", "ads", "ugc", "lipsync", "creative", "mcp", "claude-code", "agent"] + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..8717e99 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "pika", + "description": "Pika-fy your Claude — give Claude a face, name, voice, and personality, plus a full creative stack of image, video, audio, and editing tools. 3 curated /pika:* slash commands backed by 42 atomic media tools at mcp.pika.me.", + "version": "1.0.0", + "author": { "name": "Pika Labs", "email": "support@pika.art" }, + "homepage": "https://pika.me", + "repository": "https://github.com/Pika-Labs/Pika-Plugins", + "license": "Apache-2.0" +} diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..a2c5704 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "pika": { + "type": "http", + "url": "https://mcp.pika.me/api/mcp" + } + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0c8cd01 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +All notable changes to the Pika Claude Code plugin are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [1.0.0] — 2026-05-01 + +Initial public release of the Pika Claude Code plugin at [`Pika-Labs/Pika-Plugins`](https://github.com/Pika-Labs/Pika-Plugins). + +### What ships + +- **`/pika:podcast`** — Two-host podcast video for any URL or free-form topic. 1-minute, 4 acts × ~15s, native multi-shot Kling Omni dialogue with optional voice cloning for Host A. +- **`/pika:explainer`** — ~60–80s explainer video for any URL (GitHub repo, product page, docs site, blog post). Drives a real browser through the URL with element-targeted zoom, generates an avatar lipsync of the narration, and composites into a 1280×800 macOS Sonoma frame. +- **`/pika:ugc-ads`** — 15s creator-style multi-cut UGC product ad in 9:16 vertical (3:4 optional, seedance only). HOOK + 3 JUMP CUTs + OUTRO with spoken dialogue + native lip-sync on every beat, driven by a 5-act narrative arc (set → name → reveal → twist → punchline). Six category essences (HAUL / APP / FOOD / BEAUTY / FITNESS / TECH) auto-picked from the input URL. +- **42 atomic MCP tools** at [`mcp.pika.me`](https://mcp.pika.me/api/mcp) — generation (image / video / lipsync / music / speech / slide animation), editing (concat / mix / trim / captions / PiP / animate-zoom / browser-frame / beat-sync), capture (website / frame extraction), analysis (media / brief / transcribe), search (music / skill), identity (avatar / voice / persona / memory), persistent assets (Kling element / voice clone), and async (task status / cancel). Full schema at [`tools-manifest.json`](./tools-manifest.json). + +### Install paths + +Works on every Claude Code surface — CLI and Desktop (macOS / Windows app). The 42 atomic tools also work with any other MCP client (Claude.ai chat connectors, Claude Desktop chat, Cursor, Codex, etc.) at the raw MCP endpoint `https://mcp.pika.me/api/mcp`. See [README — Quickstart](./README.md#quickstart) for the install command per surface. + +### License + +Apache 2.0 — see [LICENSE](./LICENSE). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..57bc88a --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d05892 --- /dev/null +++ b/README.md @@ -0,0 +1,358 @@ +

Pika-fy your Claude

+ +

+ Give your Claude a face, name, voice, and personality — plus a full creative stack of image, video, audio, and editing tools. All in your terminal. +

+ +

+ Plugin version + MCP server + Tools + License +

+ +``` +> /pika:podcast https://pika.art + scraping page (capture_website)... + writing 4-act script (Host A · Host B)... + rendering 4 video acts × ~15s (native multi-shot)... + concatenating final clip... + done. +View video: https://cdn.pika.art/agent//podcast-final.mp4 +``` + +Until now, every Claude was just *Claude*. With the **Pika MCP + Plugin** it can be a person you design — a 3D avatar, a cloned voice, a long-term memory of who you are — driving Pika's complete creative stack: video, image, voice cloning, music generation, deterministic HTML→video rendering, automatic captions, and ffmpeg-based finishing. **AI-native by design — not a GUI wrapped in an API.** + +## How Pika fits together + +Pika has three layers — your **Agent** (persona), the **MCP server** (protocol), and the **Claude Code plugin** (curated skills). The plugin auto-registers the MCP via its bundled `.mcp.json` — you don't have to wire anything by hand. + +| Layer | What it is | Where it lives | +|---|---|---| +| **Pika Agent** | Your persona — name, face, voice, and persistent memory — applied to every tool call | [pika.me](https://pika.me/) · [iOS app](https://apps.apple.com/us/app/pika-ai-agent/id6758411447) | +| **Pika MCP** | Open-protocol server exposing 42 atomic creative tools (image, video, voice, music, edit) | `https://mcp.pika.me/api/mcp` | +| **Pika Plugin** | 3 curated `/pika:*` slash commands that orchestrate multi-step pipelines on top of the MCP | This repo | + +Same backend, same auth, same output. [MCP (Model Context Protocol)](https://modelcontextprotocol.io/) is Anthropic's open standard for connecting agents to external tools — Pika MCP works with any MCP-compatible client; see [Other Claude surfaces & MCP clients](#other-claude-surfaces--mcp-clients) for non-Claude-Code setups. + +> Tool names are shown unprefixed throughout this README (e.g. `generate_video`, `clone_voice`). Inside Claude Code the actual MCP tool prefix is `mcp__plugin_pika_pika__*` — both the curated skills and the tool catalog work without you ever needing to type the prefix. + +## Quickstart + +Requires Claude Code ≥ v2.0.12 (when the `/plugin` marketplace commands first shipped). + +### 1. Create your Pika Agent + +If you don't already have one, create your Pika Agent at **[pika.me](https://pika.me/)** or via the **[iOS app](https://apps.apple.com/us/app/pika-ai-agent/id6758411447)**. Your Pika Agent carries your persona, voice, avatar, and persistent memory across every Pika tool call — without it, the plugin has no identity to drive. + +### 2. Install the plugin + +Pick the surface you use: + +#### Claude Code CLI (terminal) + +```bash +claude plugin marketplace add Pika-Labs/Pika-Plugins +claude plugin install pika@pika-plugins +``` + +#### Claude Code Desktop (macOS / Windows app) + +UI-driven, with one slash-command line for the one-time marketplace registration. In the **Code** tab: + +1. In the prompt box, type `/plugin marketplace add Pika-Labs/Pika-Plugins` and hit Enter. +2. Click the **+** button next to the prompt box → **Plugins** → **Add plugin** → find **pika** → **Install**. + +Step 1 goes away once Pika lands in Anthropic's [official marketplace](https://claude.ai/settings/plugins/submit) — until then, the Desktop UI's plugin browser only surfaces plugins from already-configured marketplaces ([tracked in claude-code#52147](https://github.com/anthropics/claude-code/issues/52147)). + +### 3. Restart Claude Code + +Full quit and reopen — `.mcp.json` only loads at startup, and `/reload-plugins` alone isn't enough. + +### 4. Authenticate + +Inside Claude Code: + +``` +> /mcp +``` + +Find `pika`, hit **Authenticate** — your browser opens to the Pika sign-in page, sign in with the same account as your Pika Agent, the token is cached locally, and Claude Code reconnects automatically. Done. (See [Authentication](#authentication) for headless / CI options.) + +### 5. Use it + +You don't need to type slash commands — Pika skills auto-fire on natural-language intent. Both forms below do the same thing: + +``` +> /pika:podcast https://pika.art +> make me a podcast about https://pika.art + +> /pika:explainer https://github.com/anthropics/claude-code +> walk me through this repo: https://github.com/anthropics/claude-code + +> generate a 5-second video of a red panda dancing in the rain +``` + +The agent reads your prompt and runs the matching skill (or falls back to atomic MCP tools). Spending is bounded by your Pika account credit balance — no separate per-call confirmation step. + +### Verify + +```bash +claude plugin list +# pika@pika-plugins Version: 1.0.0 Status: enabled +claude mcp list +# pika: https://mcp.pika.me/api/mcp (HTTP) - Connected +``` + +## Other Claude surfaces & MCP clients + +The Pika MCP server is a standard MCP endpoint — it works with any MCP-compatible client. The Claude Code plugin (covered above) is the curated experience; below is everything else. + +| Client | How to add Pika | +|---|---| +| **Claude Code** (CLI / Desktop) | See [Quickstart Step 2](#2-install-the-plugin) above — full plugin with curated `/pika:*` skills | +| **Claude.ai chat** (regular web app) | Open [claude.ai/settings/connectors](https://claude.ai/settings/connectors) → **Add custom connector** → enter `https://mcp.pika.me/api/mcp` as the MCP server URL → complete the OAuth sign-in. Then enable the connector via the **+** button in any chat. | +| **Claude Desktop chat** (the Chat tab, not the Code tab) | Edit `claude_desktop_config.json` and add Pika as a remote MCP server pointing at `https://mcp.pika.me/api/mcp`. Restart Claude Desktop. | +| **Cursor / Codex / any MCP client** | Add `https://mcp.pika.me/api/mcp` as an HTTP MCP server in your client's MCP config; sign in with the same account as your Pika Agent. | + +On non-Claude-Code surfaces you get the **42 atomic tools** (image, video, voice, music, edit, identity) but **not** the curated `/pika:*` slash commands — those are Claude Code-specific. Same backend, same Pika Agent, same auth. + +## Launch-spotlight skills + +Curated skills designed to take you from a single prompt to a finished, shareable video. Three ship today (Podcast/Interview, Explainer, UGC Ads). All generation skills consume Pika credits (paid via your Pika account). + +> [!TIP] +> **Skills activate from natural language — the slash command is optional.** Saying _"make me a podcast about https://pika.art"_ or _"walk me through this repo: github.com/foo/bar"_ triggers the matching skill automatically. The `/pika:*` form is just an explicit shortcut. + +### Podcast / Interview Video — `/pika:podcast` + +Hand it a URL **or a free-form topic** — get back a finished **1-minute two-host conversational video**. 4 acts × ~15s each, native multi-shot dialogue, with the Matan-authenticity rules baked in: specific jokes tied to concrete details, "wait, actually..." pivots, mid-sentence interruptions, real reactions over generic praise. Optional voice cloning for Host A via `use_avatar`. ~25–30 min wall-clock. **Costs Pika credits.** + +**URL mode** — scrape and review a product page, GitHub repo, or blog post: + +``` +/pika:podcast https://pika.art +/pika:podcast https://github.com/anthropics/claude-code use_avatar +/pika:podcast # ← no args = print input menu +``` + +**Topic mode** — free-form brief; the skill writes the script from your prose: + +``` +/pika:podcast Two AI researchers debate whether AGI arrives before 2030 +/pika:podcast I and a Mars-obsessed tech CEO talk about colonization timelines +/pika:podcast interview with a seed-stage VC about what kills most startups +``` + +Triggers from natural language — _"make a podcast about [url-or-topic]"_, _"interview-style clip about X"_, _"two-host take on Y"_, _"I and [persona] talk about Z"_ — or call the slash command directly. Named real people get archetype portraits by default (no auto-deepfake); pass `host_b_img=` to override with a likeness you have rights to. + +### Explainer Video — `/pika:explainer` + +Hand it **any URL** — a GitHub repo, product page, docs site, blog post, launch announcement — get back a ~60–80s explainer at 1280×800. Drives a real browser through the URL along an element-targeted timeline, generates an avatar lipsync of the narration, and composites it all in a macOS Sonoma frame with a 246-pixel bottom-left circle avatar. **GitHub URLs** activate a repo-aware mode (README scan + live-demo detection); other URLs use a generic page-walkthrough flow. ~5–7 min wall-clock with default `pika` lipsync; ~10–25 min if you opt into `--lipsync-provider kling` for polished-presenter mode. **Costs Pika credits.** + +Triggers from natural language too — _"explain this URL"_, _"walk me through [url]"_, _"make a demo video of [product page]"_, _"explainer for [github/product/docs link]"_, _"Loom-style walkthrough of [url]"_ — or use the slash command: + +``` +/pika:explainer https://github.com/anthropics/claude-code +/pika:explainer https://github.com// --focus "architecture, demo" +/pika:explainer https://github.com// --avatar https://cdn.pika.art/<...>.png +/pika:explainer # ← no args = print URL menu +``` + +### UGC Ads — `/pika:ugc-ads` + +Hand it a product URL — get back a **15s creator-style multi-cut UGC ad** in 9:16 vertical: HOOK + 3 JUMP CUTs + OUTRO, POV first-person talking-head selfie, native lip-synced dialogue on every beat with a 5-act narrative arc (set → name → reveal → twist → punchline). The dialogue is the through-line; the screen close-up + finger-point lands on whichever JUMP CUT the *reveal* line falls on. Six category essences (HAUL / APP / FOOD / BEAUTY / FITNESS / TECH) auto-picked from the URL guide dialogue character per category. Built-in fallback Pixar-style avatar when no `avatar_url` supplied; auto-cartoonize-on-rejection via seedream when fal-queue moderation flags a photorealistic portrait; uses your Pika avatar + cloned voice silently when available. ~6–12 min wall-clock. **Costs Pika credits.** + +Triggers from natural language — _"make a UGC ad for [URL]"_, _"jump-cut product ad about [URL]"_, _"creator-style ad for X"_, _"talking-head TikTok ad about Y"_, _"haul-style ad"_, _"unboxing video about [URL]"_ — or use the slash command: + +``` +/pika:ugc-ads https://pika.art +/pika:ugc-ads https://maisonbrune.com avatar_url=https://cdn/face.png aspect_ratio=3:4 +/pika:ugc-ads https://oatly.com category=FOOD +/pika:ugc-ads https://glossier.com avatar_url=https://cdn/face.png provider=kling +/pika:ugc-ads # ← no args = print URL menu +``` + +## Authentication + +You need an authenticated MCP session before any Pika tool call works. The recommended path is one-time `/mcp` connect: + +``` +> /mcp +``` + +This opens the MCP manager UI. Find `pika`, hit **Authenticate** — your browser opens to the Pika sign-in page, sign in with the same account as your Pika Agent, the token is cached locally, and Claude Code reconnects automatically. After this, every Pika tool call works without re-prompting. + +
+Other auth flows: auto-OAuth on first call · static token (headless / CI) · troubleshooting + +### Auto-OAuth on first call + +Skip `/mcp` and just call any Pika tool. The first call returns `401`, Claude Code auto-discovers the OAuth metadata, opens your browser, you sign in, and the original call retries automatically. + +``` +> /pika:podcast https://pika.art +[browser opens for OAuth] +[returns to terminal, call retries, video URL appears] +``` + +### Static token (headless / CI / no browser) + +If you can't do an interactive browser flow, set a Pika token in your shell **before launching `claude`**: + +```bash +export MCP_AUTH_TOKEN="" +claude +``` + +Tokens come in two flavors — a **developer key** (`dk_*` prefix, long-lived; get one at [pika.me/dev](https://www.pika.me/dev/)) or an **agent key** (service-to-service for trusted automation). Don't hard-code `${MCP_AUTH_TOKEN}` into `.mcp.json` or any file you commit — keep it in your shell env. + +### Troubleshooting + +| Symptom | Fix | +|---|---| +| `401 Unauthorized` on every call | Token expired. `claude mcp remove pika` then re-run `/mcp` to re-auth. | +| Want to switch accounts | `claude mcp remove pika` → re-run `/mcp` and sign in with the other account. | +| Browser doesn't open during OAuth | Allow Claude Code to open default browser, or fall back to a static token. | +| `claude mcp list` shows `Disconnected` | Restart Claude Code (full quit). `.mcp.json` is only loaded at startup. | + +
+ +## What you can do with Pika + +The plugin ships 3 curated slash commands; underneath, **42 atomic MCP tools** are at your agent's disposal. You describe the outcome in plain English — Pika picks the right tool. + +### Generate video + +Text-to-video, image-to-video, multi-reference video, keyframe transitions, lipsync, and motion transfer — all under one schema. Pika auto-routes to the best model for each request; override via the `provider` field if you have a preference. + +``` +> make a 5-second video of a red panda dancing in the rain +> animate this image +> take these 3 reference images and this audio, build me a 10-second clip +> sync this audio to this face +> apply the motion from this reference video onto this character image +``` + +`generate_video` · `generate_reference_video` · `generate_keyframes_video` · `generate_lipsync` · `generate_motion_control_video` + +### Generate images + +Text-to-image, image editing, 4K resolution, ultra-fast iteration. Same auto-routing logic as video. + +``` +> generate a product photo of a coffee cup on white +> make a 4K vertical wallpaper based on this brief +> edit this image — add a sunset behind it +``` + +`generate_image` + +### Generate + clone voices + +Text-to-speech in 100+ languages, voice cloning from a 30-second sample. Cloned `voice_id`s are reusable across `generate_speech` and any video tool that takes voice IDs. + +``` +> read this script in a calm female voice +> clone my voice from this 30-second sample +> make a podcast where two AI hosts debate Bitcoin +``` + +`generate_speech` · `clone_voice` + +### Generate music + +Original music generation, licensed-catalog search, and beat-synced video cuts. + +``` +> compose a 60-second uplifting electronic track +> find royalty-free music about [topic] +> cut this video on the beat at 128 bpm +``` + +`generate_music` · `search_music` · `edit_beat_sync` + +### Edit + finish + +Pure-ffmpeg deterministic ops, ~30s each, stitched together by the agent. Concat, audio mix/trim, captions in 4 styles (`tiktok` · `hormozi` · `classic` · `karaoke`, 100+ languages), text overlays, picture-in-picture (rect or circular), animated zoom, macOS-frame wrap for screen recordings, frame extraction. + +`edit_concat` · `edit_audio_mix` · `edit_audio_trim` · `edit_text_overlay` · `edit_pip` · `edit_trim` · `edit_animate_zoom` · `edit_browser_frame` · `add_captions` · `extract_frame` + +### Render HTML → video (HyperFrames) + +Claude writes HTML, HyperFrames renders deterministically — same input produces byte-identical output. Outputs MP4, WebM, or MOV with transparency. + +``` +> make me a 30-second animated slide deck about my product launch +> render this HTML composition as a video +``` + +`generate_slide_animation` · `render_html_animation` + +### Analyze + transcribe + +Describe media, extract structured briefs from mixed sources, transcribe audio or video, and capture live websites with timed scroll/click actions. + +``` +> describe this video +> extract a structured product brief from these 3 sources +> transcribe this audio +> screencap this URL with timed scrolling +``` + +`analyze_media` · `analyze_brief` · `transcribe_audio` · `capture_website` + +### Identity + memory + +Your Pika Agent's persona, voice, avatar, and a persistent memory store — auto-injected as defaults on every Pika tool call so you never have to repeat yourself. + +`identity_whoami` · `identity_persona_read` · `identity_avatar_url` · `identity_voice_id` · `identity_voice_info` · `identity_set_avatar` · `identity_set_voice` · `identity_memory_search` · `identity_memory_append` + +### Full schema + +→ [`tools-manifest.json`](./tools-manifest.json) — all 42 tools with complete input/output schemas, ready for raw-MCP / OpenAPI consumers. + +## FAQ + +**How does pricing work?** +All generation skills consume **Pika credits** from your Pika account. Free credits ship with every account; top up at [pika.me](https://pika.me/). Atomic-tool calls (e.g. `generate_video`) and curated skills (`/pika:podcast`, `/pika:explainer`, `/pika:ugc-ads`) both draw from the same balance. + +**Can I use Pika MCP without Claude Code?** +Yes — the MCP server at `https://mcp.pika.me/api/mcp` is a standard HTTP MCP endpoint and works with Claude Desktop, Cursor, Codex, or any MCP-compatible client. The curated `/pika:*` slash commands are Claude Code-specific, but the 42 atomic tools are universal. See [Other Claude surfaces & MCP clients](#other-claude-surfaces--mcp-clients). + +**Is my voice clone, avatar, and memory private?** +Yes. Your Pika Agent's persona, cloned voice, avatar, and memory are scoped to your Pika account and only accessible to your authenticated MCP sessions. They are not used to train shared models. Manage or delete them at [pika.me](https://pika.me/). + +**Can I bring my own model API keys?** +No — Pika manages provider routing internally so you don't have to juggle API keys, rate limits, or billing across multiple model vendors. One Pika account, one billing surface, every model. + +**What's the difference between this and the Pika web app?** +The web app is a hosted creative environment with a UI. The MCP + Plugin is **AI-native** — designed to be driven by an agent in plain English, with no UI. Same backend, same Pika Agent, same output. Use the web app for hands-on direction; use the MCP + Plugin to let an agent compose multi-tool pipelines for you. + +## Related projects + +| Repo | What it is | Use when | +|---|---|---| +| **[Pika-Labs/Pika-Skills](https://github.com/Pika-Labs/Pika-Skills)** | Open-source `SKILL.md` modules powered by the Pika Developer API. No MCP, no plugin — drop a folder into your agent workspace, set `PIKA_DEV_KEY`, and go. | You want a single-purpose skill (e.g. video meeting agent) without installing the full plugin, or you're on an agent harness without MCP support. | +| **This repo** ([`Pika-Labs/Pika-Plugins`](https://github.com/Pika-Labs/Pika-Plugins)) | Full Claude Code plugin + remote MCP server with 42 atomic tools and curated `/pika:*` slash commands. | You want the complete creative stack inside Claude Code with one install. | + +## Manage your install + +```bash +claude plugin update pika +claude plugin disable pika +claude plugin uninstall pika +claude plugin marketplace remove pika-plugins +``` + +## Acknowledgments + +Built on [Anthropic Claude Code](https://www.anthropic.com/) and the open [Model Context Protocol](https://modelcontextprotocol.io/), with [ffmpeg](https://ffmpeg.org/) and [Playwright](https://playwright.dev/) for finishing and capture. + +## Contributing + +Issues + PRs welcome at [`Pika-Labs/Pika-Plugins`](https://github.com/Pika-Labs/Pika-Plugins/issues). + +## License + +See [LICENSE](./LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..de181ad --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ +# Security policy + +## Reporting a vulnerability + +If you discover a security issue in the Pika Claude Code plugin (this repo), please email **support@pika.art** rather than opening a public issue or PR. + +We aim to: +- Acknowledge within 2 business days +- Provide a fix or mitigation timeline within 7 days for high-severity issues +- Coordinate public disclosure once a fix is shipped + +## What this repo's security boundary is + +This repo ships **prose** (`skills/*/SKILL.md` files) and the **plugin manifest** (`.claude-plugin/`, `.mcp.json`). There is **no application code** here — no Python, TypeScript, or runtime hooks. All generation happens in: + +- **Claude Code itself** (when a user invokes a `/pika:*` slash command) +- **`https://mcp.pika.me/api/mcp`** (the upstream MCP server — runtime issues there should be reported to **support@pika.art** with subject "MCP server: ...") +- **Per-user credentials** (OAuth tokens stored locally by Claude Code; or `MCP_AUTH_TOKEN` / `dk_*` developer keys set in the user's shell environment) + +## What's in scope + +Issues we want to know about: +- Plugin manifest issues (`.claude-plugin/plugin.json`, `.claude-plugin/marketplace.json`, `.mcp.json`) that would cause unintended MCP server registration +- A SKILL.md prose payload that could be coerced into instructing the agent to call tools or exfiltrate data via adversarial user input + +## What's out of scope (forwarded upstream) + +- OAuth / token / identity issues → `pika-mcp-server` +- Provider-side issues → upstream provider +- Cost / quota issues → `pika-mcp-server` rate limiting and quotas +- Bugs in Claude Code itself → [`anthropics/claude-code` issues](https://github.com/anthropics/claude-code/issues) + +## Cost confirmation + +This plugin does **not** ship a client-side cost-confirmation gate. Spending is bounded by your Pika account credit balance and the upstream `pika-mcp-server`'s quota / rate limiting. There is no inline `--yes` / approval prompt to bypass. + +Last updated: 2026-05-01. diff --git a/skills/baseball-trend/SKILL.md b/skills/baseball-trend/SKILL.md new file mode 100644 index 0000000..9b09c4a --- /dev/null +++ b/skills/baseball-trend/SKILL.md @@ -0,0 +1,183 @@ +--- +name: baseball-trend +description: > + Viral fake "ESPN behind-home-plate broadcast cutaway" of a user — broadcast-style still + + 15s Kling-omni clip with native two-announcer commentary that names the user. Fixed + trend: Yankees vs Red Sox ALCS Game 3 at Fenway Park, premium seats, scorebug + chyron + with the user's name. Triggers: "make me a behind-home-plate cutaway", "fake MLB + broadcast of me", "AI ESPN baseball crowd shot", "viral MLB broadcast trend", + "Yankees Red Sox cutaway with me". Needs the user's name + one reference photo. +argument-hint: +--- + +# baseball-trend + +15-second ESPN-style broadcast cutaway of a user, sitting behind home plate at a fake Yankees vs Red Sox ALCS Game 3 game at Fenway Park, with two announcers naming them on air. + +Fixed-recipe skill — the prompts below are calibrated. Substitute the username; do not rephrase. + +## Stage 0 — Intake + +Two questions, one at a time. Don't bundle. + +**1. Username** *(required)* — used both in the broadcast chyron and in the announcers' commentary, e.g. `"Jane Doe"`. Save as `state.username`. This replaces every literal `${username}` in the prompts below. + +**2. Reference image** *(required)* — one front-facing 3/4 portrait, good lighting, one face. Resolve to a CDN URL and save as `state.reference_image_url`: + +- **`https://…` URL** → use as-is. +- **Local path** → call `mcp__pika__upload_asset({ filename, mime_type, size_bytes })`, then PUT the bytes to the returned `presigned_url`, then use the returned `public_url`. (Required args without it: schema rejects with 400.) +- **Claude Desktop, photo pasted inline** → inline pastes don't reach MCP tools yet (Anthropic limitation). Reply with: + + > Heads up — pasted images don't reach MCP tools on Claude Desktop yet. Two options: + > - **Paste a URL** if it's already hosted somewhere — fastest. + > - **Zip it and attach the `.zip`** — right-click → Compress (macOS) / Send to → Compressed folder (Windows) / `zip pic.zip pic.png` (Linux). I'll unzip + upload. + + When a `.zip` arrives: unzip via Bash, call `upload_asset` as above, use `public_url`. + +After both answers are in, echo one short confirmation ("Generating behind-home-plate cutaway for **{username}**…") and start the pipeline. **No further yes/no gates after this point** — the pipeline runs end-to-end. + +## Pipeline + +Two Pika MCP calls, sequential. Engines are locked: `gpt-image-2` for the still, `kling-v3-omni` for the video. + +### Step 1 — Broadcast still (`generate_image`) + +The chyron + scorebug get baked into the still at frame 0 (load-bearing — when Kling is asked to "pop in" the chyron mid-clip it appears at second 4–5 with a visible flash and breaks the trend; baking it into the first frame makes Kling treat it as pixel-locked burned-in UI). + +Call `mcp__pika__generate_image` with: + +- `provider`: `gpt-image-2` +- `reference_images`: `[state.reference_image_url]` +- `aspect_ratio`: `16:9` +- `quality`: `medium` (don't pass `high` — exceeds the proxy 180s timeout) +- `output_format`: `png` +- `prompt` (verbatim, `${username}` substituted): + +``` +A screenshot from a live MLB game TV broadcast on ESPN. The camera cuts to the audience — our reference image person, sitting smiling in premium field-level seats behind home plate at Fenway Park, smiling naturally and unaware they're on camera. Hardlock: Do not alter their facial structure and maintain their likeness. The subject must match the reference person. + +The image looks exactly like a real TV screenshot — broadcast color grading, slight compression artifacts, interlacing grain, telephoto broadcast camera feel. It's the New York Yankees vs Boston Red Sox, MLB American League Championship Series (ALCS), Game 3, Boston home stadium (Fenway Park). Yankees lead 2-0 in the ALCS so far. + +CRITICAL — broadcast graphics that MUST be visible in this image: +1. A real ESPN-style bottom scorebug for MLB, showing Yankees vs Red Sox with team logos, inning, outs, balls/strikes count, and score (with a small runners-on-base diamond), looking like a real live broadcast scorebug. +2. Directly above the scorebug, a clean broadcast-style lower-third name graphic / chyron that reads exactly: "${username}" — set in a classic ESPN sans-serif, in the network's color treatment. The chyron sits in the lower-left area, above the scorebug, like a real broadcast identifier for the on-camera guest. +3. The ESPN network logo watermark in a corner. + +All three graphics must look like real burned-in broadcast UI — not Photoshop overlays. 16:9 aspect ratio. +``` + +Save the returned URL as `state.broadcast_still_url`. + +**Agent-side self-check before Step 2**: the chyron must spell the username correctly and the scorebug must look like real broadcast UI. If either looks wrong, re-roll Step 1 (everything downstream pixel-locks to this frame). This is the agent's own check — do not ask the user. + +### Step 2 — 15s broadcast video (`generate_reference_video`) + +`image_types: ["first_frame"]` is a parallel-array param that locks `state.broadcast_still_url` as Kling's literal frame 0, keeping the chyron + scorebug pixel-static for the full 15s. + +Call `mcp__pika__generate_reference_video` with: + +- `provider`: `kling` +- `kling_model`: `kling-v3-omni` +- `duration`: `15` +- `aspect_ratio`: `16:9` +- `quality_mode`: `pro` +- `reference_images`: `[state.broadcast_still_url]` +- `image_types`: `["first_frame"]` +- `sound`: `true` +- `prompt_adherence`: `strict` *(load-bearing — without it the scorebug animates and identity drifts late in the clip)* +- `negative_prompt` *(verbatim, load-bearing — without these entries Kling occasionally morphs the scorebug or fades the chyron)*: + +``` +scene cuts, camera angle changes, scorebug animation, chyron pop-in, chyron fade-in, chyron text changes, graphics animating, exaggerated acting, direct address to camera, blurry face, identity drift, distorted anatomy +``` + +- `prompt` (verbatim, `${username}` substituted everywhere; pre-trimmed to fit Kling's 2500-char cap; chyron-on-frame-0 lock at top): + +``` +First frame is the provided reference image. The ESPN scorebug AND the "${username}" lower-third chyron are ALREADY on screen at frame 0 — keep them visible, unchanged, pixel-locked across all 15 seconds. Do NOT animate them, do NOT change their text. + +Realistic live MLB broadcast shot of the subject sitting in premium field-level seats behind home plate at Yankees vs Red Sox ALCS Game 3 in Boston, Fenway Park. The shot feels like a real TV cutaway when the broadcast camera finds a notable guest in the crowd between innings. + +The subject is seated in his field-level seat, smiling naturally and not over-performing. Not locked into eye contact with the lens. Occasionally glances toward the field, then toward camera, then back to the field — like a real in-game crowd reaction. One continuous take. No cuts. No angle changes. + +Action timeline: +0-4s: smiling casually in his seat as the camera lands on him; looks around naturally, not paying attention to camera. +4-7s: relaxed natural wave toward the camera (crowd cheers when he waves the first time); glances up at the Jumbotron above him then back to camera. +7-11s: cheers briefly with visible excitement, reacting to the playoff atmosphere; turns to his friend on the left, exchanges words, laughs (we don't hear him speak). +11-15s: claps naturally while smiling. + +Keep all movement subtle, believable, human. No exaggerated acting. No direct talking to camera. + +Broadcast styling: real live sports broadcast look, telephoto broadcast camera feel, natural ballpark lighting, slight broadcast compression, slight interlacing / TV grain, authentic crowd movement in the background, realistic field-level framing. Subject remains seated behind home plate the full shot. + +Audio: Natural live sports-broadcast commentary from two male announcers talking about him being at the game tonight. Casual, warm, authentic — like real MLB commentators noticing a known guest. Sample lines: +"${username} is here tonight at Fenway, taking in this massive playoff matchup." +"You can see he's enjoying himself here behind home plate for Game 3." +"Great atmosphere in the building, and ${username} getting a lot of love from the crowd." + +Constraints: Preserve identity strongly. Keep him seated behind home plate throughout. No constant eye contact with camera. No talking to camera. No exaggerated gestures. No scene cuts. Scorebug + chyron do not change at any point. Genuine MLB TV broadcast crowd cutaway feel. +``` + +Save the returned video URL as `state.broadcast_video_url`. If the call returns a `task_id`, poll `mcp__pika__task_status(task_id)` per MCP server instructions. + +### Step 3 — Deliver + +Per project convention (`feedback_media_display_workflow`): download both URLs locally, then emit `[[image:]]` / `[[video:]]` markers with **absolute paths** (relative `./tmp/` is ambiguous depending on cwd): + +```bash +DEST="$(pwd)/tmp" +mkdir -p "$DEST" +curl -fsSL "$STILL_URL" -o "$DEST/behind-home-plate-still.png" +curl -fsSL "$VIDEO_URL" -o "$DEST/behind-home-plate-cutaway.mp4" +test -s "$DEST/behind-home-plate-still.png" && test -s "$DEST/behind-home-plate-cutaway.mp4" +``` + +Then emit (with absolute paths) AND echo the CDN URLs as plaintext fallback: + +``` +[[image:/absolute/path/to/tmp/behind-home-plate-still.png]] +[[video:/absolute/path/to/tmp/behind-home-plate-cutaway.mp4]] +``` + +One-line summary: *"Behind-home-plate cutaway for {username} — 15s, 16:9, 1080p, Kling v3-omni, native two-announcer commentary."* + +## Load-bearing phrases (don't strip these) + +These are empirical behavior dependencies, not writing style — removing them breaks the recipe: + +- In the **still prompt**: `Hardlock: Do not alter their facial structure and maintain their likeness` + `The subject must match the reference person` (without these, identity drifts on the first frame, and everything downstream inherits the drift). +- In the **video prompt**: `Preserve identity strongly` + `The ESPN scorebug AND the "${username}" lower-third chyron are ALREADY on screen at frame 0 — keep them visible, unchanged, pixel-locked` (without these, Kling re-animates the chyron mid-clip). +- The full **negative_prompt** list — every entry there came from a specific failure mode in prior runs. +- `prompt_adherence: "strict"` and `image_types: ["first_frame"]` — see inline notes above. + +## Engine choice: Kling-only (with one caveat) + +Seedance has a two-stage `partner_validation_failed` 422 gate (validated 2026-05-12 across 4 runs on the NBA sibling skill): + +- **Input-side** (`body.image_urls`): rejects if the reference contains a recognizable real person. +- **Output-side** (`body.generated_video`): rejects AFTER generation if the produced clip contains recognizable-looking faces — and every broadcast cutaway has a crowd full of faces. + +The output-side gate is unavoidable for this trend regardless of subject, so Seedance is functionally unusable here. Kling is the engine that works **for ordinary user photos**. + +**Kling caveat — recognizable celebrities are blocked too.** Kling has its own content-moderation gate that fires on celebrity references (validated 2026-05-13: a Michael Jordan reference + "Ke Wang" chyron returned `task_status: failed, task_status_msg: "Failure to pass the risk control system"` at submit-time). This is correct behavior — the trend illusion only works with a non-public-figure reference where the chyron name + face are coherent. If a user supplies a celebrity photo, surface the gate to them and ask for a non-celebrity reference instead. + +**Kling trade-offs**: 2500-char `prompt` cap (recipe above is pre-trimmed), no `seed` param (re-rolls are non-reproducible — to re-roll just call again). + +## Failure modes + +| Symptom | Cause | Fix | +|---|---|---| +| Chyron pops in mid-clip (~4–5s flash) | Chyron not baked into the still | Re-run Step 1; verify chyron is visible in `state.broadcast_still_url` before Step 2 | +| Scorebug animates / morphs mid-clip | `prompt_adherence` not `strict`, or `negative_prompt` was trimmed | Restore strict adherence and the full negative_prompt | +| Identity drift late in the clip (face changes after ~10s) | Reference image too small / Kling losing the face | Re-run Step 2; if drift persists, re-run Step 1 with a tighter face crop on the still (more facial pixels = stronger lock) | +| Username mispronounced by announcers | Native audio is one take | Re-run Step 2 | +| Seedance `partner_validation_failed` 422 | Tried Seedance instead of Kling | Use Kling only — see engine-choice section above | +| Kling `task_status: failed` with `task_status_msg: "Failure to pass the risk control system"` | Reference photo is a recognizable celebrity / public figure | Ask the user for a non-celebrity reference. Kling correctly blocks impersonation patterns (celebrity face + fake-event chyron) | +| `generate_image` 400 `invalid_image_file` from `openai v1/images/edits` | Reference is an iPhone HEIC-derived JPEG with heavy EXIF and/or extreme aspect ratio (e.g. 2316×3088) | Re-encode the reference before upload: `convert in.jpg -strip -auto-orient -resize 1536x1536\> out.png`, then upload the cleaned PNG | +| `generate_image` 502 / proxy timeout | Quality was set to `high` (>180s) | Pass `quality: "medium"` | + +## What NOT to do + +- Don't sport-swap. NBA / NFL / soccer variants → fork this skill; don't parameterize this one. +- Don't add suffixes to the chyron (e.g. " - AI Creator"). Chyron is the username alone — the trend illusion depends on it reading like a real broadcast identifier. +- Don't add post-edits — no `add_captions`, `generate_music`, `edit_*`. Kling burns the scorebug + chyron + native commentary directly; anything added afterward breaks the broadcast illusion. diff --git a/skills/explainer/SKILL.md b/skills/explainer/SKILL.md new file mode 100644 index 0000000..c79230f --- /dev/null +++ b/skills/explainer/SKILL.md @@ -0,0 +1,439 @@ +--- +name: explainer +description: ~60-80s explainer video for any URL — GitHub repo, product page, docs site, blog post, or launch. CANONICAL workflow for URL walkthroughs. Use when the user asks to "explain this URL / repo / website / product", "make a walkthrough video for [url]", "demo this site", "Loom-style explainer of [url]", "explainer for github.com/...", or "explain this product link". Drives a real browser through the URL, generates an avatar lipsync, and composites in a 1280×800 macOS Sonoma frame with a 246-pixel bottom-left avatar circle. GitHub URLs activate a repo-aware mode (README scan + live-demo detection); other URLs use a generic page-walkthrough flow. +argument-hint: [--focus angles] [--avatar url] [--voice id] [--live-url url] [--lipsync-provider pika|kling] [--no-captions] [--preview] +--- + +# /pika:explainer + +Generate a ~60–80s URL explainer video: drive a real browser through the URL along a beat-sheet timeline, generate an avatar lipsync of the narration, and composite it all in a 1280×800 macOS Sonoma frame with a 240-pixel inner avatar (246-pixel outer including 3px white stroke ring) at canvas (20, 476) and element-targeted zoom on every mid-section beat. Works on any URL — product pages, docs sites, blog posts, launches. GitHub URLs activate a repo-aware mode (README scan + live-demo detection); all other URLs use a generic page-walkthrough flow. + +**Usage:** `/pika:explainer [--focus "angles"] [--avatar ] [--voice ] [--lipsync-provider pika|kling] [--preview] [--live-url ]` + +## Behavior + +### Defaults — fire fast, no mid-flow confirmation + +- **Use identity-store defaults silently for avatar / voice.** Never ask "should I use your avatar?" or "which voice?" before firing. Honor explicit overrides (`--avatar`, `--voice`) when supplied; otherwise resolve via `identity_avatar_url` / `identity_voice_id` and proceed. See Step 1 for the full resolution waterfall (including the silent fallback when identity returns null). +- **No mid-flow "type yes to proceed" gates by default.** Step 5 preview is **opt-in via `--preview`** (for power users testing new avatar/voice combos before the long-pole render); the default flow runs end-to-end without pausing. +- **Do not solicit `--focus` either.** Make a confident first attempt from page structure; users re-run with `--focus "X"` if the angle missed. + +These defaults match industry standard for media-gen tools (Midjourney / Sora / Runway / HeyGen / Pika.art): submit → render → return. Account credit balance + provider failover (Step 9) are the canonical guardrails. + +### Local avatar images on Claude Desktop + +Claude Desktop can't pass inline-pasted images to MCP tools yet (Anthropic-side limitation). If the user pastes a photo inline, or mentions a local file they want as `--avatar`, pause Step 1 and kindly send them this — something like: + +> Heads up — pasted images don't reach MCP tools on Claude Desktop yet (Anthropic limitation). Two easy options for your avatar: +> +> - **Paste a URL** if it's already hosted (Imgur, S3, your site) — fastest +> - **Zip it and attach the `.zip`** — right-click → Compress (macOS) / Send to → Compressed folder (Windows) / `zip pic.zip pic.png` (Linux). I'll take it from there. + +When a `.zip` arrives, unzip it via Bash, call `upload_asset` for a presigned PUT URL, push the bytes with `curl -X PUT`, then use the returned `public_url` as `--avatar ` — all before Step 1. Already-hosted `https://...` URLs work as-is and skip this entirely. If no avatar is supplied at all, the identity-store default fires. + +### Step 0 — Resolve URL (empty-args menu) + +Strip flags (`--focus`, `--avatar`, `--voice`, `--live-url`, `--lipsync-provider`, `--no-captions`, `--preview`, `--skip-preview`, `--yes`) and `key=value` parameters from `$ARGUMENTS`. **If what remains contains no `https://...` URL** (or is empty / whitespace-only), print this menu **verbatim** as your full response, then **stop and wait for the user's next message** — do NOT call any tool, do NOT proceed to Step 1, do NOT invent a URL. If `$ARGUMENTS` already carries a URL, skip this step silently and proceed to Step 1. + +> **Which URL would you like me to walk through?** Works on any of: +> +> - **A GitHub repo** — e.g. `https://github.com/anthropics/claude-code` (activates repo-aware mode: README scan + live-demo detection) +> - **A product page / launch page** — e.g. `https://pika.art` +> - **A docs site** — e.g. `https://docs.anthropic.com` +> - **A blog post / article URL** +> +> Output: 1280×800 macOS Sonoma frame with a bottom-left avatar lipsync and element-targeted zoom on every mid-section beat. Default flow runs end-to-end with no confirmation gates — pass `--preview` if you want a 3-second lipsync sanity check first. +> +> Reply with the URL and I'll start. +> +> *Tip: you don't need to type `/pika:explainer` — just say things like "walk me through ", "make a demo video of ", or "explain this repo: " and I'll fire this skill automatically.* + +When the user replies with a URL, treat it as the resolved input and proceed to Step 1. Do not re-prompt. + +### Step 1 — Parse input + detect mode + +Required: `url` (must be `https://...`). +Optional: `--avatar ` (overrides identity-store default), `--voice `, `--focus "..."` (editorial guidance woven into vo_text), `--live-url ` (force-supply live demo URL — GitHub mode only), `--lipsync-provider ` (defaults to **`pika`** — parrot a2v, ~2-5 min wall-clock, slightly more dramatic head motion. Pass `kling` for tighter face-centered output at ~5-30 min wall-clock — Kling produces minimal-head-motion presenter shots but is the long-pole stage; reserve for high-stakes renders), `--no-captions` (skip the Step 11 caption burn — default is captions on), `--preview` (opt-in to the Step 5 preview gate — ~3s lipsync of "Hi, I'm your presenter" for testing new avatar/voice combos before the long-pole render; default is no preview). `--skip-preview` and `--yes` are accepted as no-ops for backward compatibility. + +**Mode detection:** +- **GitHub mode** — URL host is `github.com` AND path matches `/{owner}/{repo}` (no further path segments past the repo root). Activates the repo-aware extras: README scan, live-demo detection, GitHub-specific selectors. +- **Generic-URL mode** — anything else (a product page, docs site, blog post, deeper GitHub path like `/blob/HEAD/path`). Skips the GitHub extras; uses generic CSS selectors and walks through the URL itself. + +**Avatar resolution (silent — never ask the user):** +1. If `--avatar ` was passed, use it. +2. Else call `mcp__pika__identity_avatar_url`. If non-null, use it. +3. Else (fresh user, no identity avatar set yet) call `mcp__pika__generate_image` once with prompt `"professional presenter, friendly tech narrator, studio portrait, 1:1, natural lighting"` and use the returned URL. Do **not** ask the user "should I generate one?" — just generate silently. + +**Voice resolution (silent — never ask the user):** +1. If `--voice ` was passed, use it. +2. Else call `mcp__pika__identity_voice_id`. If non-null, use it. +3. Else pick a casual MiniMax `speech-2.8-hd` preset matching the resolved avatar's apparent gender: + - **Female-coded avatar** → `English_PlayfulGirl` (warm, casual, clearly female-voiced — verified) + - **Male-coded avatar** → `English_Jovialman` (warm, casual male) + - **Unclear / gender-neutral** → `English_Jovialman` (default) + + Determine gender from `mcp__pika__identity_persona_read` (look for a gender / pronouns field) when identity exists; otherwise infer from the resolved avatar image. Do **not** call `analyze_media` for this — it's not worth the extra ~30s round-trip. Do **not** ask the user. + + **Do NOT use `English_FriendlyPerson`** — despite being categorized under "female" in MiniMax's catalog, its display name is "Friendly Guy" and it reads as male in playback. `English_PlayfulGirl` is the canonical casual-female pick. Other verified-female alternates: `English_Upbeat_Woman`, `English_LovelyGirl`, `English_radiant_girl`. + +The flow below is annotated per step: **GitHub-only**, **Generic-only**, or **Both** modes. + +### Step 2 — Read source (no MCP call) + +**Both modes:** use Claude's `WebFetch` on the input URL to pull the page's main content (h1, hero section, headings, primary copy). + +**GitHub mode additions:** also fetch top-level file tree, (best-effort) `package.json` / `pyproject.toml`, and GitHub API repo metadata via `gh api repos/{owner}/{repo}` for `homepage`, `description`, `language`, `topics`. Detect a candidate `live_url` in this priority: + +1. User-supplied `--live-url`. +2. **GitHub API `meta.homepage` field** — set when the maintainer configured the repo's homepage in GitHub settings (matches tarball `repo_analyzer.py:66-77`). +3. `package.json` `"homepage"` field. +4. First match in README of `https?://[^\s)\"'<>]+(?:vercel\.app|netlify\.app|github\.io|fly\.dev|railway\.app|render\.com|herokuapp\.com|surge\.sh)[^\s)\"'<>]*`. +5. **Any other URL in README that the badge area / "Live Demo" / "Project Page" / "Demo" text points at.** The allowlist regex above misses arbitrary custom domains (e.g. `-project-page.com`); when the README explicitly designates a project page, prefer that over the github.io fallback. +6. GitHub Pages convention `https://{owner}.github.io/{repo}` — but only if the deep tree contains a frontend signal (one of `index.html`, `App.tsx`, `App.jsx`, `App.vue`, `app.py`, `main.py`). + +If no candidate resolves, the beat sheet skips beats 6–7. + +**Generic-URL mode:** the input URL itself is the only URL the beats walk through — no `live_url` inference, no extra metadata fetches. Skip Step 2.5 and Step 3.0; jump straight to Step 3. + +### Step 2.5 — Verify `live_url` reachability (GitHub mode only, no MCP call) + +If a candidate `live_url` was selected, verify it serves real content **before** authoring beats 6–7. Use `WebFetch` on the candidate and check the response: + +- If the response status is 4xx / 5xx, **drop `live_url` to None** and skip beats 6–7. The github.io fallback in particular is reachable as a hostname but often returns 404 ("There isn't a GitHub Pages site here") for repos that haven't enabled Pages — recording that 404 page wastes ~12s of the explainer on wrong content. +- If the response renders the GitHub Pages "404 — There isn't a GitHub Pages site here." template (heuristic: response body contains `"There isn't a GitHub Pages site here"`), drop `live_url` and skip beats 6–7. +- Otherwise, keep `live_url` for beats 6–7. + +This mirrors the original tarball's `requests.head(live_url, timeout=6, allow_redirects=True)` reachability gate. + +### Step 2.6 — Generic-URL pre-flight (Generic-URL mode only, no MCP call) + +Before authoring beats for a non-GitHub URL, WebFetch the input URL and inspect the response. This step prevents three common Generic-URL failure modes: (a) recording a captcha / bot-block page instead of content, (b) the cookie/consent banner eating the first ~3 seconds of video, (c) generic CSS selectors missing the page's actual hero / sections. + +**A. Bot-block / captcha detection — abort if matched:** + +If the response body contains any of: + +- `"Verify you are human"` / `"verify you are not a robot"` +- `"captcha"` / `"CAPTCHA"` / `"reCAPTCHA"` +- `"403 Forbidden"` / `"Access Denied"` +- `"Just a moment"` + `cf-chl-bypass` (Cloudflare challenge) +- `"We're sorry, something went wrong"` (Amazon-style bot block) +- A `` or h1 of just "Robot Check" / "Are you a robot?" + +→ **ABORT** with a clear error to the user: "Generic-URL mode can't render this site — the page is showing a bot-detection / captcha challenge under headless Chrome. Try a different URL, or run a real-user version of the page first to verify it loads cleanly." + +**B. Cookie / consent-banner detection — defuse with `extra_css` + optional click:** + +Scan the response for these patterns (case-insensitive): + +- IDs / classes starting with `onetrust-`, `truste-`, `cookie-banner`, `cookie-consent`, `gdpr-`, `consent-`, `cmp-` +- Buttons matching `(?i)accept (all )?cookies` / `(?i)agree.{0,10}cookies` / `(?i)i (accept|agree)` +- Apple-specific banner: id `ac-gdpr-banner` or class `as-globalfooter-curtain` +- Google consent: `[role="dialog"]` with text "Before you continue" + +If detected, set `cookie_banner_present = true`. Defense in depth — the recording uses BOTH: + +1. **CSS injection (`extra_css`)** in the `capture_website` call to hide common banners universally — even if the click below misses, the banner is visually gone. +2. **A `click` `timed_action`** at `at_s: 0.0` against the most likely dismissal selector (extracted from the WebFetch DOM, e.g. `#onetrust-accept-btn-handler`, `[aria-label*="Accept all" i]`, `button[id*="accept"]`). + +The `extra_css` payload (use this verbatim — covers ~80% of consent platforms): + +``` +#onetrust-banner-sdk, #onetrust-pc-sdk, #onetrust-consent-sdk { display: none !important; } +#truste-consent-track, #truste-consent-content, .truste_box_overlay { display: none !important; } +[id*="gdpr-cookie"], [id*="cookie-consent"], [id*="cookie-banner"] { display: none !important; } +[class*="cookie-banner"], [class*="cookie-consent"], [class*="consent-banner"] { display: none !important; } +[class*="CookieBanner"], [class*="CookieConsent"], [class*="ConsentBanner"] { display: none !important; } +#ac-gdpr-banner, .as-globalfooter-curtain { display: none !important; } /* Apple */ +[role="dialog"][aria-label*="cookie" i], [role="dialog"][aria-label*="consent" i] { display: none !important; } +.cmp-container, .cmp-modal, .cmp-banner { display: none !important; } +``` + +**C. Real-DOM element identification — emit concrete selectors:** + +Generic CSS selectors (`h1`, `[class*="hero"]`, `section h2`) work on semantic / well-marked-up sites but miss obfuscated class names on big-name corporate sites (apple.com uses `tile-headline` / `as-headline-section-title`, not `hero-*`). For each beat, prefer the **actual DOM elements** observed in the WebFetch: + +- Read the rendered HTML/markdown WebFetch returned. Note the page's actual primary `<h1>` text and class. +- Note the page's section structure (h2 headings + their parent containers). +- Note any prominent CTA / signup / pricing element. +- Emit `zoom_target.selector` using **the actual class or id observed**, falling back to semantic structure (`main > section:nth-of-type(N) h2`) when class names look auto-generated (Tailwind `_1a2b3c`, CSS modules `module__hero___xYz`). + +**D. SPA / lazy-render detection — bump initial wait:** + +If the WebFetch response has fewer than 3 visible headings / minimal text content, the page may be SPA-rendered post-`domcontentloaded`. Emit a longer initial `wait` action (`{type: "wait", at_s: 0.0, ms: 2500}`) before any beat fires, instead of the default 600ms settle. + +**E. `--focus` is honored when supplied (do not solicit):** + +Without `--focus`, select beats from generic structure cues — proceed silently with a confident first attempt. Do **not** ask the user "what should I focus on?" before firing; users iterate by re-running with `--focus "the X feature"` if the first pass misses the angle they wanted. With `--focus` supplied, anchor beat selection on the phrase: uses concrete page sections that match it, ignores irrelevant marketing chrome. + +### Step 3.0 — Required README section scan (GitHub mode only, no MCP call) + +Before authoring the beat sheet, **scan the README** (case-insensitive, full-text) for any of these section names. If a match is found, you **must** add a dedicated beat for that section in Step 3, replacing one of the generic beats 4–5 if necessary: + +| README contains... | Required beat | +|---|---| +| `how it works` | scroll_to that heading; zoom `article h2:has(#user-content-how-it-works)` | +| `audio layer` / `audio timeline` | scroll_to the audio-layer diagram; zoom on the rendered figure or its surrounding heading | +| `claude code` / `mcp integration` | scroll_to that section; zoom `article pre` or `.highlight` (terminal screenshot / code block) | +| `architecture` / `system design` | scroll_to that section; zoom `article h2:has(#user-content-architecture)` | +| `features` (when prominent at top) | scroll_to that heading; zoom `article h2:has(#user-content-features)` | +| `getting started` / `quick start` / `installation` | scroll_to that heading; zoom `article h2:has(#user-content-installation)` (or the matching slug) — falls back to `article pre` if you want the install code block instead | +| `usage` / `examples` | scroll_to that heading; zoom `article h2:has(#user-content-usage)` (or the matching slug) — or the first code block under it | + +**GitHub heading slug rule:** lowercase, spaces → dashes, strip non-`[a-z0-9-]` characters. So "How it works" → `#user-content-how-it-works`, "Quick Start" → `#user-content-quick-start`. GitHub injects the `<a id="user-content-{slug}">` anchor inside each rendered `<hN>`, so `hN:has(#user-content-{slug})` reliably grabs the heading element across any GitHub README. + +**Selector contract:** `bbox_selector` MUST be vanilla CSS that resolves via `document.querySelector` (`capture_website` runs the post-action smooth-scroll JS via `page.evaluate`, which uses the browser's native selector engine). **Do NOT use Playwright extensions like `:has-text("...")`, `text=...`, or `:visible`** — those resolve in Playwright's `page.query_selector` (so the bbox capture finds the element) but silently fail in the smooth-scroll's `document.querySelector` (so the page never scrolls to the target, and `bbox.y` ends up at document-Y instead of `top - 60 px`, which trips Step 8b's `bbox.y > recording_viewport.h` degenerate filter and falls back to default-position zoom). CSS Level 4 `:has(...)` IS vanilla and supported in modern Chromium. + +These sections are the highest-information visuals in most explainer-worthy repos. Missing them produces a generic walkthrough; including them gives the explainer a concrete "show, don't tell" beat. The original tarball SKILL.md flagged the first four with `SPECIAL` rules in the Gemini prompt; this Step 3.0 promotes them from incidental guidance to a hard requirement and adds three more high-signal headings common in OSS READMEs. + +### Step 3 — Author beat sheet (main thread, no MCP call) + +Write a JSON array of 8–10 beats, **with a hard total duration of 65–80 seconds and a hard total word count of 165–200 words** (assuming a speaking rate of 2.5 words/sec). Each beat: + +```jsonc +{ + "t_start": 0.0, + "t_end": 7.5, + "action": { "type": "navigate" | "scroll_to" | "hover", "url": "...", "selector": "..." }, + "zoom_target": { "selector": "...", "description": "..." }, + "vo_text": "exact words to speak — 1 to 2 conversational sentences" +} +``` + +**Hard constraints (validate before emitting the beat sheet — reject the draft if any fails):** +1. Every beat MUST have all five fields: `t_start`, `t_end`, `action` (with `type` and `url`), `zoom_target` (with `selector`), `vo_text`. Missing fields ⇒ reject and re-author. (Mirrors tarball's `github_explainer.py:183-190` validation pass.) +2. `t_start` of beat 0 = 0.0; `t_end[i] == t_start[i+1]` (continuity). +3. `len(vo_text.split()) / 2.5` ≈ `t_end - t_start` per beat. Aim for ±10% of this estimate; if your draft is denser than 2.5 wps, tighten the `vo_text` until it fits. +4. **Total `t_end` of last beat ≤ 80 seconds.** (Reference output is 86.5s including intro; lipsync audio is ~83s. Kling avatar/image2video stalls reliably past ~90s of audio under current load — going over 80s risks a 20-min Kling timeout.) +5. **Total spoken word count between 165 and 200 words.** +6. Every beat's `zoom_target.selector` MUST be a valid CSS selector for the page that beat lands on. **GitHub mode prefers** GitHub-specific selectors: `h1.f1`, `#readme`, `article h2`, `.blob-code-inner`, `.highlight`, `.octicon-star`, `nav`. **Generic-URL mode prefers** robust generic selectors: `h1`, `[role="main"]`, `main`, `header`, `nav`, `.hero`, `.feature`, `section h2`, `[class*="cta"]`, `[class*="hero"]`, `button`, `a[href]`. **Selectors must resolve on the rendered page after the beat's action settles** — verify against the DOM you can see via WebFetch before emitting. +7. `vo_text` is 1-2 conversational sentences. Dev voice. No stage directions. No markdown. +8. `action.url` is a valid `https://...` URL when `action.type == "navigate"`; required. + +**Self-check before Step 4:** verify `total_words` is in `[165, 200]` AND `total_seconds` (= `beats[-1].t_end`) is in `[65, 80]`. If either misses bounds, re-author the beat sheet — do not proceed to TTS. (No need to "print" anywhere — this is an internal draft validation; just reject the draft and re-author until it passes.) + +**Structural skeleton — GitHub mode (load-bearing for the visual contract — match origin, but Step 3.0 overrides if applicable):** + +- **Beat 1:** `navigate` repo root, zoom `h1.f1` (repo title), hook sentence. +- **Beats 2–3:** `navigate` to specific source files (`https://github.com/{owner}/{repo}/blob/HEAD/<path>`), zoom `.blob-code-inner` or `.highlight`. Pick files that match the narration's claim — don't navigate to a file you won't talk about. +- **Beats 4–5:** `scroll_to` README sections, zoom `article h2` or `#readme`. **If Step 3.0 surfaced required sections, replace these slots with the required ones.** +- **Beats 6–7 (only if `live_url` survived Step 2.5):** `navigate` to `live_url`, zoom `nav` / `h1` / `.hero` / `main` / `button` / `.feature`. +- **Beat 8:** back to repo root, zoom `.octicon-star`, outro. + +**Structural skeleton — Generic-URL mode:** + +- **Beat 1:** `navigate` to the input URL, zoom `h1` or `[class*="hero"] h1` (the page's primary headline), hook sentence. +- **Beats 2–3:** `scroll_to` the page's hero / value-prop / first feature section. Zoom `.hero`, `[class*="hero"]`, `[class*="feature"]`, or `section:nth-of-type(1) h2`. Pick visible elements the narration references. +- **Beats 4–5:** `scroll_to` deeper sections — feature lists, screenshots, pricing, social proof. Zoom `section h2`, `[class*="feature"] img`, `[class*="testimonial"]`, `[class*="pricing"]`, or any prominent semantic element on the page. +- **Beats 6–7:** `scroll_to` CTA / signup / demo embed. Zoom `[class*="cta"]`, `button`, `a[class*="button"]`, or `[id*="signup"]`. (No live-demo navigation in generic mode — the input URL IS the demo.) +- **Beat 8:** `scroll_to` footer / closing element, zoom `footer h2`, `footer`, or back to top with `h1`. Outro sentence. + +If `--focus` is supplied, weave its angles into `vo_text` without mutating the structural skeleton. Prefer **CSS selectors over `text_content`** in `zoom_target.selector` — bbox capture is selector-only (see Known gaps). + +### Step 4 — TTS + +Call `mcp__pika__generate_speech` with `provider: "minimax-tts"`, `text: <full vo_text join>`, optional `voice_id`. Capture `result.audio_url` (the dispatcher returns audio under `audio_url`, not `url`) and `result.duration_seconds`. Voice defaults to identity-store injection in plugin mode. + +### Step 4.5 — Audio length verification + +The TTS engine's actual output rate often diverges from the 2.5 wps estimate in Step 3. Verify before committing to the long-pole lipsync step. + +- If `|audio_duration_seconds - beats[-1].t_end| > 10` seconds, **abort** and re-author the beat sheet with a tighter / looser word budget. +- If `audio_duration_seconds > 90`, **abort** regardless — Kling avatar/image2video stalls reliably past 90s of audio under current load. + +This gate exists to catch length drift before it costs 20 minutes of Kling timeout. Tarball had an equivalent verification table. + +### Step 5 — Preview gate (opt-in via `--preview`) + +**Skip Step 5 entirely by default.** Proceed directly to Step 6 unless the user explicitly passed `--preview` — do **not** generate a preview, do **not** ask for confirmation. This matches industry standard for media-gen tools (Midjourney / Sora / Runway / HeyGen / Pika.art): submit → render → return; account credit balance + provider failover are the canonical guardrails. + +`--skip-preview` and `--yes` are accepted as no-ops for backward compatibility — they were the old opt-out flags. + +If `--preview` was supplied: + +1. `mcp__pika__generate_speech` with `text: "Hi, I'm your presenter. Let's explore this repo together."` → `preview_audio_url`. +2. `mcp__pika__generate_lipsync` with `provider: <resolved_lipsync_provider>` (defaults to `pika`; honor `--lipsync-provider kling` if supplied), `image: <avatar>`, `audio: preview_audio_url` → `preview_lipsync_url` (bare lipsync, ~3s). Use the same provider here as Step 9 will use for the full audio — the preview's job is to confirm the avatar+voice+provider combo before the long-pole render. +3. Present to the user verbatim: + + > Preview ready: `<preview_lipsync_url>` + > This confirms the avatar + voice combo. The full render is a long pole (~5–30 min Kling lipsync on the full audio). + > Reply `yes` to proceed, or anything else to cancel. + +4. Match `^(yes|go|proceed|confirm|y)$` (case-insensitive). Anything else → STOP, no further MCP calls. + +### Step 6 — Build `timed_actions` and record + +Translate the beat sheet into `capture_website` `timed_actions`. **One `timed_action` per beat** — set `bbox_selector` to the beat's `zoom_target.selector` and `capture_website` captures the post-action bbox of that element internally (legacy 600 ms settle → smooth-scroll-to-`top - 60 px` → 1300 ms post-anim → measure, all server-side). + +For each beat in order, emit one entry: + +- **`navigate` beats**: `{type: "navigate", at_s: <t_start>, url: <action.url>, bbox_selector: <zoom_target.selector>}`. The worker navigates, waits to absolute `at_s + 0.6 s`, scrolls `bbox_selector` into view, and measures the bbox — all without the caller scheduling a follow-up step. +- **`scroll_to` / `hover` beats**: `{type: "scroll", at_s: <t_start>, selector: <action.selector or zoom_target.selector>, bbox_selector: <zoom_target.selector>}`. The action's own `selector` drives the page scroll; `bbox_selector` drives the bbox measurement (it can be the same selector or different — usually the same). (`capture_website` has no `hover`; scroll-into-view is the analog.) + +**Do NOT prepend the eight-step intro scroll-through** that the tarball ran. The lipsync audio is timed from `t=0` of the beat sheet; a prepended intro shifts the screen recording forward by ~3 s while leaving the audio un-shifted, causing audio/video desync. The capture_website recording begins at `t=0` with beat 0's URL already loaded — that's the orientation the tarball's intro scroll provided, minus the desync. + +Call `mcp__pika__capture_website`: + +- `url: <beat 0's action.url>` +- `timed_actions: <the N-element list built above>` (one entry per beat) +- `duration_s: max(ceil(beats[-1].t_end), ceil(audio_duration_seconds))` — covers both the beat budget AND any TTS overrun. MiniMax-TTS commonly produces audio ~5-10% longer than the 2.5 wps estimate (Step 4.5 already gates drift > 10s); using the max ensures the screen recording covers the full lipsync, otherwise `edit_pip`'s `shortest=1` would clip the recording's tail and you'd lose the last few seconds of audio with no screen behind it. + +**Generic-URL mode additions** (per Step 2.6 pre-flight): + +- `extra_css: <the cookie-banner-hiding CSS payload from Step 2.6 §B>` — defensive: hides common consent platforms via `display: none !important;` so even if the optional click misses, the banner is invisible in the recording. +- **Prepend a `wait` action** `{type: "wait", at_s: 0.0, ms: 2500}` for SPA / lazy-render pages (per Step 2.6 §D); use 1500ms for "normal" pages. This gives time for hero images to lazy-load, fonts to swap, and scroll-triggered animations to be ready before the first beat fires. +- **If `cookie_banner_present` from Step 2.6 §B**, also prepend a `click` action `{type: "click", at_s: 0.5, selector: <detected dismissal selector from WebFetch DOM>}` and shift all beat `t_start` / `t_end` values by `+1.5s` to compensate. Beat 1 navigates / scrolls at `t_start: 1.5` (or whatever offset accommodates the dismissal animation). The lipsync audio also needs to start with a 1.5s lead-in pause — easiest to just have beat 1's `vo_text` begin with a half-second pause-friendly opener like _"Alright,"_ or _"So,"_, or pad the audio externally before lipsync. +- **No cookie banner shifts needed** if `cookie_banner_present == false`; just the prepended wait action. + +Capture `video_url`, `recording_viewport`, `action_bboxes`. The result returns `recording_viewport: {w, h}` and `action_bboxes: [{idx, selector, found, bbox: {x,y,w,h}}]` alongside `video_url`. + +**`action_bboxes[].idx` semantics:** the `idx` field is the position in the **input** `timed_actions` array. + +- **GitHub mode**: with one timed_action per beat, `idx` maps 1:1 to beat index — Step 8 uses `entry.idx` directly as `beat_idx`. +- **Generic-URL mode**: the prepended `wait` (and optional cookie-dismissal `click`) shift the array by 1 or 2. Compute `beat_idx = entry.idx - prepend_count` where `prepend_count` is 1 (wait only) or 2 (wait + click). Skip entries where `beat_idx < 0` (those are the prepended setup actions, not beats). + +The `selector` field on each entry reports `bbox_selector` (i.e. `zoom_target.selector`), not the action's own `selector`. + +### Step 7 — Browser chrome + +`mcp__pika__edit_browser_frame`: + +- `video_url: <Step 6 video_url>` +- `url: (live_url if GitHub-mode and survived Step 2.5 else input_url, truncated to 65 chars)` +- `tab_title: <30-char title>` — GitHub mode: `(meta.description or repo_name or "")[:30]`. Generic-URL mode: the page's `<title>` (from WebFetch in Step 2) or the URL's hostname, truncated to 30 chars. Guard against `None`/empty. + +Returns `framed_url` (1280×800 Sonoma + chrome). + +### Step 8 — Build `zoom_keyframes` and apply + +Constants: + +- `INTRO_BEATS = 2` — gates by **beat-sheet index**. Skips zoom on beat indices 0 and 1 ("Beat 1" and "Beat 2" in the structural skeleton above). +- `HOLD_GAP = 0.6` — seconds of 1.0× before each zoom-in and after each zoom-out. +- `MIN_BEAT_DUR = 1.5` — beats shorter than this are skipped (no room for a meaningful zoom). +- `SCALE = 1.35` (precise element-targeted zoom). +- `FALLBACK_SCALE = 1.25` (default-position fallback when no usable bbox). +- `FALLBACK_RAMP = 0.4`. + +`edit_browser_frame`'s inner-content offsets: `CONTENT_X=56, CONTENT_Y=108, CONTENT_W=1168, CONTENT_H=637` (verified against the worker's `edit_browser_frame/main.py`). + +Coord transform (recording px → framed px): + +``` +cx_framed = 56 + (bbox.x + bbox.w/2) * (1168 / recording_viewport.w) +cy_framed = 108 + (bbox.y + bbox.h/2) * (637 / recording_viewport.h) +``` + +**Build the zoom list with a per-beat default + bbox override pattern.** The legacy rig followed an "every non-intro beat gets a zoom — bbox-derived if available, default-position otherwise" rule. Reproduce that here: + +**Step 8a — Pre-fill default-position keyframes for every non-intro, long-enough beat.** + +Constants for the default position: +- `DEFAULT_CX = 56 + 1168 // 2` (screen center of the framed canvas) +- `DEFAULT_CY = 108 + 637 // 3` (upper-third of the content area, where most GitHub UI prominence lives) + +Walk the beat sheet from index `INTRO_BEATS` (= 2) to the end. For each beat: + +- If `t_end - t_start < MIN_BEAT_DUR` (1.5s), skip — too short for a meaningful zoom. +- Compute the keyframe's interior interval as `[t_start + HOLD_GAP, t_end - HOLD_GAP]`. If that interval is shorter than 1.0s, skip. +- Otherwise pre-fill that beat's slot in a per-beat map (call it `zoom_keyframes_by_beat[beat_idx]`) with `{cx: DEFAULT_CX, cy: DEFAULT_CY, scale: FALLBACK_SCALE (1.25), ramp_s: FALLBACK_RAMP (0.4)}` plus the trimmed `t_start`/`t_end`. + +**Step 8b — Override with bbox-derived precise zoom where `action_bboxes` provided a usable measurement.** + +For each entry in `action_bboxes`: + +- `beat_idx = entry.idx` (since Step 6 emits one timed_action per beat). If `beat_idx < INTRO_BEATS`, skip. +- If `entry.found` is false, skip. +- If the beat isn't already in `zoom_keyframes_by_beat` (was filtered out in Step 8a by `MIN_BEAT_DUR`/`1.0s` rules), skip. +- **Filter degenerate bboxes:** skip if `bbox.y > recording_viewport.h` (offscreen capture — page didn't scroll the element into view in time) or `bbox.h > recording_viewport.h * 1.5` (full-page `<main>` element — yields a meaningless zoom center). +- Compute `cx_framed`/`cy_framed` from the bbox center using the recording-px → framed-px transform shown above. Override the beat's slot with `{cx: cx_framed, cy: cy_framed, scale: SCALE (1.35), ramp_s: min(0.5, (t_end - t_start) * 0.15)}`. + +**Final list:** sort the values of `zoom_keyframes_by_beat` by `t_start` to produce the `zoom_keyframes` array. + +This guarantees every non-intro, long-enough beat gets a zoom — precise when bbox capture worked, default-positioned otherwise. Avoids the "flat video for the whole runtime" failure mode. + +If `len(zoom_keyframes) > 0`, call `mcp__pika__edit_animate_zoom` with `video_url: framed_url, zoom_keyframes`. Returns `zoomed_url`. Otherwise (no qualifying beats — should be rare given Step 3's 65-80s constraint) skip and use `framed_url` as `zoomed_url`. + +### Step 9 — Lipsync the full audio + +`mcp__pika__generate_lipsync`: + +- `provider: <resolved_lipsync_provider>` — **default: `pika`** (parrot a2v). Honor `--lipsync-provider kling` if explicitly passed. +- `image: <avatar>` +- `audio: <Step 4 audio_url>` + +**Provider tradeoffs:** + +| Provider | Wall-clock | Head motion | When to use | +|---|---|---|---| +| **`pika`** (default) | ~2–5 min | Slightly more dramatic, naturalistic | Default for most runs — fast iteration, watchable output, ~10× faster than kling | +| `kling` (opt-in) | ~5–30 min | Minimal, face-centered, presenter-style | High-stakes renders where the avatar must read like a polished presenter; tolerate the long pole | + +Server-side-await covers the call inline; if the response shape is `{task_id, status: "queued"}`, poll `mcp__pika__task_status` in a tight loop (no sleep) until the status reaches a terminal state (`done`, `failed`, or `cancelled`). On `done`, capture `lipsync_url`. On `failed` / `cancelled`, fall back to the **other** provider (kling ↔ pika) per the failover note below. + +**Failover:** +- If `pika` fails (rare — parrot a2v is robust at typical explainer audio lengths) → retry once with `provider: "kling"`. +- If `kling` stalls past the worker's 1200s ceiling (visible as repeated `processing` status with no completion) → fall back to `provider: "pika"`. Step 4.5's audio-length gate should catch the long-audio case before it gets here, but the failover handles the residual risk. + +**Why pika is the default:** +- Speed — typical explainer wall-clock drops from ~10–15 min to ~5–7 min total because lipsync is the long pole. +- Quality is good enough — parrot a2v is naturalistic; the slight extra head motion reads as engaging rather than distracting in a 60-80s clip with avatar circle PiP. +- Kling-mode-pro polish is mostly invisible inside the 246-pixel circle anyway — face area is too small for the minimal-head-motion difference to register on most viewers. + +For the canonical "polished presenter" feel of the original tarball reference output, pass `--lipsync-provider kling` explicitly. + +### Step 10 — PiP composite + +`mcp__pika__edit_pip`: + +- `main_video_url: <zoomed_url>` +- `overlay_video_url: <lipsync_url>` +- `shape: "circle"` +- `size_px: 246` ← pixel-pinned 246px outer diameter (240 inner avatar + 3+3 stroke ring); matches tarball's `CIRCLE_OUT = CIRCLE_SIZE + STROKE * 2` +- `stroke_width_px: 3` +- `stroke_color: "white"` +- `position_px: {x: 20, y: 476}` ← `800 − 246 − 78` for dock clearance (matches tarball's `H − CIRCLE_OUT − 78`) + +Do NOT pass `size` — `size_px` and `size` are mutually exclusive. Returns `final_url`. + +**Master-duration / audio-source contract** (matching tarball `github_explainer.py:418-419, 531-533, 578-582`): `edit_pip` uses `shortest=1` semantics by default, which means the composite's duration is the shorter of (zoomed screen recording) and (lipsync video). Step 6's `duration_s = max(ceil(beats[-1].t_end), ceil(audio_duration_seconds))` ensures the screen recording is ≥ the lipsync, so the composite duration is set by the lipsync. Audio comes from the lipsync video's audio track (the lipsync embeds the original TTS audio); the standalone `audio_url` is not re-mixed. If the lipsync video is shorter than the screen recording (Kling sometimes trims trailing silence), the screen will get cut off at the lipsync end — accept this; the alternative (looping the screen) is worse for explainer content. + +### Step 11 — Burn captions + +Call `mcp__pika__add_captions(video_url=<final_url>, style="classic")`. `classic` renders a bottom subtitle bar — the right register for an explainer video (use `tiktok` / `hormozi` / `karaoke` only when the user explicitly asks for word-level highlight). The audio is extracted server-side from the PiP composite's lipsync track, so transcription matches the narration verbatim. Capture the result as `captioned_url`. + +Skip this step only if the user passed `--no-captions` (parsed in Step 1) — the default is captions on. (Note: `/pika:podcast` does **not** burn captions — narration in an explainer is more transcription-friendly than fast two-host dialogue.) + +### Step 12 — Return + +Emit `captioned_url` (or `final_url` if Step 11 was skipped) on one line: `Done: <url>`. + +## Known gaps (carried as follow-up server-side work) + +- **Kling avatar `mode:"pro"` and `prompt` not exposed.** Tarball calls Kling directly with `{mode:"pro", prompt:"talking head, face centered, mouth syncs to audio, minimal head movement, professional presenter"}`. The Pika MCP `generate_lipsync` wrapper drops both for the kling provider (schema says prompt is "parrot only"; mode is hardcoded). Real quality lever for reducing dramatic head motion in the lipsync. Server PR follow-up: surface `prompt` and `mode` on `generate_lipsync` for kling. +- **No white-frame trim on the screen recording.** Tarball's `recorder.py:165-185` detects mean-brightness > 245 in the first 4s and trims with ffmpeg. `capture_website` has internal trim heuristics but doesn't expose them to the caller. Visible as a brief white flash at the start of the explainer when the page is still loading. The 800ms `wait` action at `at_s: 0.0` mitigates this somewhat by giving the page time to paint, but doesn't trim already-recorded white frames. Worker enhancement. +- **No `networkidle` wait on per-beat navigation.** Tarball uses `page.goto(url, wait_until="networkidle", timeout=20000)` plus `wait_for_timeout(600)` after every navigate. `capture_website` settles to `domcontentloaded` plus the bbox-capture branch's 600 ms post-action settle (server-side, when `bbox_selector` is set), but SPA blob pages whose final render happens after `domcontentloaded` can still get bbox'd against unmounted code blocks. Worker enhancement: expose a `wait_until` knob on `timed_actions[].navigate`. +- **No per-step output-size verification gates.** Tarball's `verify()` helper at `github_explainer.py:35-39` checked TTS ≥ 50KB, preview ≥ 100KB, screen ≥ 200KB, lipsync ≥ 500KB, final ≥ 1MB after each step. The MCP path returns URLs only; verifying file size would require an extra `mcp__pika__analyze_media` call per step (~30s overhead each). Worth adding once user-side latency budget allows it. For now, a downstream-failure cascade (e.g. zero-byte TTS → silent lipsync → blank composite) only surfaces at Step 11. +- **`text_content` bbox capture not implemented.** `capture_website` v1 returns `action_bboxes` only for steps with a CSS `selector`. `text_content`-only steps produce no entry. Prefer CSS selectors in `zoom_target` for guaranteed zoom coverage. +- **Beat-sheet wording is non-deterministic.** Running the same input twice produces different vo_text and different zoom positions. Visual *kind* is the contract, not pixel-exact reproduction. +- **Generic-URL mode quality varies by site.** Modern indie / SaaS landing pages with semantic markup (`<h1>` + clear `<section>` + named class hooks) work well. Big-name corporate sites (apple.com, microsoft.com, amazon.com) hit several known limits: (a) **bot detection** — the page may serve a degraded version under headless Chrome, or a captcha; Step 2.6 §A aborts on these but the heuristics aren't exhaustive; (b) **obfuscated class names** — `tile-headline` instead of `hero-title` defeats generic selectors; Step 2.6 §C's WebFetch DOM scan helps but isn't perfect; (c) **scroll-triggered animations don't play** — IntersectionObserver-driven hero reveals fire on real user scrolls, not Playwright's `scrollIntoView`; the recorded frame may be a static placeholder; (d) **lazy-loaded images** — picture/source elements with `loading="lazy"` may not have resolved by the 600ms-or-2500ms settle window; the bbox lands on a transparent placeholder. Workarounds: prefer simpler / smaller marketing pages for launch demos, always pass `--focus "the X feature"` to anchor beat selection, accept that big-name sites need a follow-up server PR (cookie-banner click retry + `wait_until=networkidle` + animation-trigger via `IntersectionObserver` polyfill). +- **Cookie-banner click is single-attempt.** Step 2.6 §B emits one `click` against the dismissal selector extracted from the WebFetch DOM. If the WebFetch's HTML doesn't include the banner (rendered post-JS) or the selector is wrong, the click silently misses — the `extra_css` payload is the load-bearing defense. Worker enhancement: support a list of fallback selectors per `click` action so the worker tries each in order. + +## Auth + +If any call returns 401: the user's OAuth token has expired or hasn't been issued. The next authenticated MCP call triggers OAuth automatically (browser opens for `@pika.art` Google login). For non-interactive environments, set `MCP_AUTH_TOKEN`. + +## Examples + +GitHub-mode (repo-aware: README scan + live-demo detection): + +- `/pika:explainer https://github.com/leigest519/OpenGame` +- `/pika:explainer https://github.com/anthropics/claude-cookbooks --focus "Claude Code MCP integration"` +- `/pika:explainer https://github.com/openai/whisper --preview` (opt-in to the preview gate when testing a new avatar) + +Generic-URL mode (any non-GitHub URL — drives through the page directly): + +- `/pika:explainer https://pika.art` +- `/pika:explainer https://linear.app --focus "the cycle planning view"` +- `/pika:explainer https://docs.anthropic.com/en/docs/claude-code/plugins` +- `/pika:explainer https://your-product-page.com --avatar https://cdn.example.com/me.png --preview` diff --git a/skills/podcast/SKILL.md b/skills/podcast/SKILL.md new file mode 100644 index 0000000..4bfc027 --- /dev/null +++ b/skills/podcast/SKILL.md @@ -0,0 +1,173 @@ +--- +name: podcast +description: Two-host podcast video for any URL OR free-form topic — 1 minute, 4 acts × ~15s, native multi-shot dialogue, optional voice cloning for Host A. CANONICAL workflow for podcast-style spoken video. Accepts EITHER a URL (scraped and reviewed) OR a free-form brief like "I and Elon Musk talk about Mars", "two scientists debate AGI", "podcast about quantum computing", "interview with a VC about seed-stage fundraising". Use when the user asks to "make a podcast", "podcast about [thing]", "podcast review of [url]", "two-host explainer", "interview-style clip", "two-host take on [topic]", "two people talking on camera about [thing]", "podcast clip", "GRWM podcast", "I and X talk about Y", "interview with [persona] about [topic]" — or any variant of "two hosts discussing [anything]". No captions burned (native audio is the deliverable; auto-transcription mistranscribes domain-specific terms). +argument-hint: <url-or-topic> [bg_img=] [host_a_img=] [host_b_img=] [voice_a=] [voice_b=] [use_avatar] [aspect_ratio=16:9] +--- + +# /pika:podcast + +4 acts × 15s each = 60s. Host A always LEFT, Host B always RIGHT. Accepts a URL **or** a free-form topic / brief. + +## Parameters + +| Param | Default | Notes | +|---|---|---| +| `input` | required | URL to review **or** free-form topic / brief (e.g. "I and Elon Musk talk about Mars") | +| `bg_img` | auto-generated | Podcast studio background | +| `host_a_img` | auto-generated | Host A portrait — see Real-person handling below | +| `host_b_img` | auto-generated | Host B portrait — see Real-person handling below | +| `voice_a` | `876341503281471517` | Kling preset or cloned voice ID for Host A | +| `voice_b` | `829837252279803904` | Kling preset or cloned voice ID for Host B | +| `use_avatar` | off | Clone user's identity voice as Host A via `clone_voice` | +| `aspect_ratio` | `16:9` | Output aspect ratio | + +## Defaults — fire fast, no mid-flow confirmation + +- **Use the param-table defaults silently for voices.** `voice_a` defaults to the Kling preset `876341503281471517` and `voice_b` to `829837252279803904`. Do **not** ask "which voice?" or "should I clone yours?" before firing — only honor explicit overrides (`voice_a=`, `voice_b=`, `use_avatar`). +- **Auto-generate any missing host portraits silently** (Step 1's archetype prompts). Do **not** ask "should I generate a host image?" — just generate. +- **No "type yes to proceed" gates.** Submit → render the 4 acts → return URL. Account credit balance + provider failover are the canonical guardrails. The `--yes` flag is accepted as a no-op for backward compatibility. +- **Topic-mode personas (Step 3)** — when the user names a real public figure, follow Step 4 (Real-person handling) silently: archetype portrait by default, no auto-generated photographic likeness, no question to the user about likeness rights. + +## Local images on Claude Desktop + +Claude Desktop can't pass inline-pasted images to MCP tools yet (Anthropic-side limitation). If the user pastes a photo inline, or mentions a local file they want as `host_a_img` / `host_b_img`, pause Step 1 and kindly send them this — something like: + +> Heads up — pasted images don't reach MCP tools on Claude Desktop yet (Anthropic limitation). Two easy options for your photo: +> +> - **Paste a URL** if it's already hosted (Imgur, S3, your site) — fastest +> - **Zip it and attach the `.zip`** — right-click → Compress (macOS) / Send to → Compressed folder (Windows) / `zip pic.zip pic.png` (Linux). I'll take it from there. + +When a `.zip` arrives, unzip it via Bash, call `upload_asset` for a presigned PUT URL, push the bytes with `curl -X PUT`, then use the returned `public_url` as the parameter — all before Step 1. Already-hosted `https://...` URLs work as-is and skip this entirely. + +If the user names a real public figure without attaching anything, do NOT auto-generate their likeness — Step 4 (Real-person handling) uses an archetype portrait instead. + +## Steps + +### 0. Resolve input (empty-args menu) + +Strip flags (`--yes`, `--no-captions`, etc.) and `key=value` parameters from `$ARGUMENTS`. **If what remains is empty or whitespace-only**, print this menu **verbatim** as your full response, then **stop and wait for the user's next message** — do NOT call any tool, do NOT proceed to Step 1, do NOT invent a topic or URL. If the stripped input is non-empty (a URL or any prose), skip this step silently and proceed to Step 1. + +> **What would you like a podcast about?** I can take any of: +> +> - **A website URL** (product page, docs site, launch page) — e.g. `https://pika.art` +> - **A GitHub repo** — e.g. `https://github.com/anthropics/claude-code` +> - **A blog post / article URL** — e.g. a recent piece you'd like discussed +> - **A free-form topic or brief** — e.g. *"I and Elon Musk talk about Mars"* or *"two scientists debate AGI"* +> +> Reply with your choice and I'll generate a 1-minute two-host podcast video (4 acts × ~15s). +> +> *Tip: you don't need to type `/pika:podcast` — just say things like "make a podcast about <topic>", "podcast review of <url>", or "I and <persona> talk about <topic>" and I'll fire this skill automatically.* + +When the user replies, treat their reply as the resolved input (URL or topic) and proceed to Step 1. Do not re-prompt. + +### 1. Generate missing assets (parallel) + +Generate only what's not provided. Default archetype prompts: +- `bg_img` — modern podcast studio, two chairs, warm lighting, no people, 16:9 +- `host_a_img` — enthusiastic host, studio portrait, left-side framing, 1:1 +- `host_b_img` — pragmatic skeptic host, studio portrait, right-side framing, 1:1 + +If the input mentions specific personas (Step 3), tune the archetype to match the persona vibe — see Real-person handling below. + +### 2. Resolve voice IDs (only if `use_avatar` is set) + +1. Call `identity_voice_info` → `{ voice_id, platform, sample_url }` +2. If `sample_url` is present: call `clone_voice(voice_url=sample_url, voice_name="host_a_voice")` → set `voice_a` to the returned Kling voice ID + +### 3. Parse input mode — URL vs topic + +Strip flags (`--yes`, `--no-captions`, etc.) and key=value parameters from `$ARGUMENTS`. Inspect what remains. + +**URL mode** — input contains a `https?://` URL: +- Call `capture_website` on the URL. +- Extract: product name, value prop, 2–3 specific features or facts, pricing, one jokeable detail. +- Use these as the script's factual anchors. + +**Topic mode** — input is free-form prose (no URL): +- Treat the whole input as the brief. Parse for: + - **Subject** — what the conversation is about + - **Hosts** — explicit if mentioned ("I and Elon Musk", "two scientists", "Joe and Sarah"); otherwise use defaults (enthusiastic host + skeptic host) + - **Angle** — debate / interview / explainer / casual + - **Concrete facts** — any specific claims, numbers, dates, quotes the user gave +- If no concrete facts are given, invent **2–3 plausible specific claims** to anchor jokes and the "wait, actually..." pivot. Stay grounded — verifiable-sounding, not wild fabrications. The audience can tell. +- If the user says "I and X" or "me and X", Host A = the user (use `use_avatar` flow if not already, or default avatar) and Host B = X. + +### 4. Real-person handling (topic mode only) + +If the parsed input names a specific real public figure as a host (e.g. "Elon Musk", "Taylor Swift", "Joe Rogan"): + +- **Default behavior**: do NOT auto-generate that person's photographic likeness. Generate an **archetype portrait** matching the persona vibe — e.g. "tech-billionaire-energy CEO at a podcast desk" for an Elon-style host, "pop-star aesthetic" for a Taylor-style host. Clearly inspired-by, not impersonation. +- **Override**: if the user explicitly provides `host_a_img=<url>` or `host_b_img=<url>`, use the provided image as-is. The user takes responsibility for likeness rights. +- **Voices**: same logic — default to a generic Kling preset; only use a cloned voice when the user provides one (`voice_a=` / `voice_b=`) or invokes `use_avatar` (which clones the user's own voice for Host A). +- **Script tone**: the dialogue can riff on the named persona's known public positions or vibe (e.g. Mars enthusiasm for Elon-style) — public-record opinions are fair game. Do NOT put specific defamatory, off-character, or fabricated-private-life statements in their mouth. + +This guardrail keeps the skill creative ("I want a podcast where I argue with a tech CEO about Mars") without auto-generating deepfakes of named real people. + +### 5. Write script + +Write 4 acts × 2 lines (HOST_A / HOST_B). Each line ~10–12s of spoken dialogue. + +**Required (Matan rules — apply to both URL and topic modes):** +- One specific joke tied to a concrete detail (scraped fact in URL mode; topic-derived claim in topic mode) +- One "wait, actually..." skeptic-flip moment +- At least one mid-sentence interruption +- Natural filler: "okay so", "wait", "right?", "i mean", "honestly" +- Real reactions, not generic praise +- Reference at least one actual feature name, price, claim, or quote +- Natural ending — no forced "bye!" + +Acts: Hook → Feature deep-dive → The Turn → Verdict +(In topic mode the analogue: Hook → Substance → The Pivot → Verdict.) + +### 6. Generate video acts (subagent, sequential) + +Delegate to a subagent with all resolved assets and the script. The subagent runs acts 1→2→3→4 sequentially — do NOT parallelize. + +Each act: one `generate_reference_video` call (`kling-v3-omni`, `duration=15`, `sound=true`). Pass `reference_images=[bg_img, host_a_img, host_b_img]` and `voice_ids=[voice_a, voice_b]`. Three shots: + +- Wide 5s: both hosts, no voice token +- MCU-A 5s: `<<<voice_1>>> '<HOST_A line>'` +- MCU-B 5s: `<<<voice_2>>> '<HOST_B line>'` + +Emotional beats per act: +- Act 1: A excited, B skeptical +- Act 2: A gesturing/explaining, B questioning +- Act 3: A firm, B surprised and reconsidering +- Act 4: A satisfied, B conceding + +After act 4, subagent calls `edit_concat([act1, act2, act3, act4])` and returns the final video URL. + +### 7. Output + +Return the final video URL and a one-sentence verdict. **Do not call `add_captions`** — Whisper auto-transcription is unreliable on the domain-specific terms typical of podcast dialogue (product names, persona names, technical jargon). Native Kling Omni audio is the deliverable. + +--- + +**Rules:** +- `voice_ids` must be valid Kling voice IDs — never use name-style strings like `Calm_Man` +- Host A always LEFT (`<<<image_2>>>`), Host B always RIGHT (`<<<image_3>>>`) — never swapped + +## Examples + +URL mode (review a website / repo / blog): + +``` +/pika:podcast https://pika.art +/pika:podcast https://github.com/anthropics/claude-code +/pika:podcast https://cursor.com use_avatar +``` + +Topic mode (free-form brief): + +``` +/pika:podcast Two AI researchers debate whether AGI arrives before 2030 +/pika:podcast I and a Mars-obsessed tech CEO talk about colonization timelines +/pika:podcast interview with a seed-stage VC about what kills most startups +/pika:podcast podcast about quantum computing breakthroughs in 2026 +``` + +Mixed (URL inside a topic prompt — agent prefers URL mode if a valid URL is found): + +``` +/pika:podcast podcast about https://pika.art with skeptical investor energy +``` diff --git a/skills/ugc-ads/SKILL.md b/skills/ugc-ads/SKILL.md new file mode 100644 index 0000000..9a31e07 --- /dev/null +++ b/skills/ugc-ads/SKILL.md @@ -0,0 +1,209 @@ +--- +name: ugc-ads +description: >- + Multi-cut jump-cut UGC product ad — HOOK + 3 JUMP CUTs + OUTRO, 15s, 9:16 + vertical (3:4 optional, seedance only), POV first-person talking-head selfie, + every beat has spoken dialogue with native lip-sync, 5-act narrative arc + (set → name → reveal → twist → punchline). Six category essences + (HAUL / APP / FOOD / BEAUTY / FITNESS / TECH) auto-picked from the input URL. + Different from `/pika:short-ads`: that one is a polished single-clip brand + commercial with logo reveal + BGM; this one is creator-style raw UGC + talking-head with multi-beat conversational dialogue. Use when the user asks + to "make a UGC ad", "jump-cut product ad", "POV product reveal", + "creator-style ad", "haul-style ad", "unboxing ad", "TikTok-style product + video", "talking-head ad about [URL]". +argument-hint: <url> [avatar_url=<url>] [provider=seedance|kling] [aspect_ratio=9:16|3:4] [category=auto|HAUL|APP|FOOD|BEAUTY|FITNESS|TECH] [captions=true] +--- + +# /pika:ugc-ads + +## Parameters + +| Param | Default | Notes | +|---|---|---| +| `url` | required | product URL — drives category detection and beat substitution | +| `avatar_url` | built-in fallback | persona portrait URL; fed as `@Image1` reference. When omitted, the skill uses a pre-generated Pixar-style female creator portrait | +| `provider` | `seedance` | seedance: strong at UGC selfie / talking-head POV with native lip-sync, multi-segment in single prompt, supports 3:4. kling: explicit `shots[]`, 9:16/16:9 only | +| `aspect_ratio` | `9:16` | `3:4` is seedance-only (kling rejects 3:4) | +| `category` | auto | `HAUL` / `APP` / `FOOD` / `BEAUTY` / `FITNESS` / `TECH`; auto-picked from URL | +| `captions` | `true` | TikTok-style word-chunked captions burned on top of the final video | + +## Runtime expectations + +Typical end-to-end run: **6–12 minutes**. Breakdown: + +- Step 1 (WebFetch) + Step 3 (capture_website screenshot): ~10–30s +- Step 7 (`generate_reference_video`): ~3–5 min for seedance, ~5–7 min for kling +- Step 7b/c (cartoonize + retry): adds ~1–2 min if seedance moderation rejects the avatar +- Step 8 (captions): single `add_captions` call, ~30s–5 min (transcribe + burn in one shot) + +If the run exceeds 15 min without progress, something is wrong — check `task_status` returned errors. + +## Steps + +### 1. Fetch + categorize + +`WebFetch` the URL: pull `product_name`, value prop, brand color, product form, packaging, hero copy, target user, category, **and the primary language of the page**. Use `category=` if passed; else trust the WebFetch signal; fall back to HAUL for physical, APP for digital. + +### 2. Resolve the avatar (fallback to built-in if missing) + +- If `avatar_url` was passed → use it as-is. +- If NOT passed → use this built-in fallback: + ``` + https://cdn.pika.art/v2/files/agent/17d62bf9-0edb-49e4-9ba9-2c5419fa518f/seedream-1777624057811.jpeg + ``` + Pre-generated 3D animated Pixar-style portrait of a young female creator — pre-cartoonized so seedance moderation accepts it directly, neutral enough to fit any category. Note in the final summary that the fallback was used so the caller knows to supply their own portrait for persona consistency next time. + +### 3. Capture the product screenshot (best-effort) + +Call `capture_website` with `mode: "screenshot"`. Use `mobile=true` for handheld-product categories (APP / FITNESS / BEAUTY) so the captured page renders as a portrait phone screen; `mobile=false` for desktop-context categories (HAUL / TECH / FOOD). + +If the call fails (timeout, browser pool down), retry **once**. If still failing, proceed without the screenshot — the skill is degraded but functional. The close-up beat then describes the page from prose only and Beat 2's `reference_images` is just `[avatar_url]`. + +Capture URL → `screenshot_url` (or null). + +### 4. Compose the prompt + +The full prompt is a single multi-beat string passed to **one** `generate_reference_video` call. Structural prose (not markdown bullets). Every beat has a `Says: "..."` line for lip-sync. Pacing target ~5.5–6 words per second across the whole 15-second ad (≈85–90 words total). `@Image1` is the avatar, `@Image2` is the screenshot when available. + +**Write all `Says: "..."` lines in the language detected from step 1's WebFetch.** Both seedance and kling lip-sync handle multilingual; if the product page is Chinese / Japanese / Spanish / etc., the dialogue should be in that language. Hook archetypes from step 5 are language-agnostic — adapt the rhetorical move to the language's natural register. + +``` +HOOK (0–3 sec) <visual setting + creator framing + face/body cue>. Says to camera, fast and energetic: "<hook line>". <style anchor — POV handheld, authentic, raw>. + +JUMP CUT 1 (3–6 sec) <wide POV — creator's body language, product partially in frame edge>. <face cue>, says fast: "<setup line>". + +JUMP CUT 2 (6–9 sec) <next visual beat — could be the screen close-up showing @Image2 OR another reaction beat, depending on which beat the dialogue arc puts the reveal>. Says (or voice continues over the shot if it's a screen close-up), fast and confident: "<reveal line>". + +JUMP CUT 3 (9–12 sec) <next visual beat — same logic; one of the JUMP CUTs is the screen close-up, the others are wide-POV reaction shots>. Says, fast: "<insight twist line>". + +OUTRO (12–15 sec) <selfie POV, mid-chest framing, same setting>. Says to camera, fast: "<punchline line>". + +avatar is image 1, asset is image 2 +``` + +**Screen-close-up beat — exactly one across the ad, position is dialogue-driven:** +- Place the screen close-up on whichever JUMP CUT (1, 2, or 3) the *reveal* line lands on. Most ads put it on JUMP CUT 2; if the narrative needs it earlier or later, JUMP CUT 1 or JUMP CUT 3 is fine. Pick by content, not by slot number. +- The screen close-up beat shows `@Image2` exactly as-is and includes ONE finger-point gesture (a single finger entering from the frame edge, pointing at the hero text or product — no tap, no swipe, no scroll, no hover-on-CTA). The point gesture is the only screen interaction in the entire ad. +- The other JUMP CUTs are wide-POV reaction beats: hands stay on knees, on the bed, or at sides. + +**Trust `@Image2`** — when the product page is shown, reference the image; do NOT describe its UI in prose. Describing UI triggers the model to invent extra panels / dropdowns / sidebars / animations. Reference the image; trust it. + +### 5. Category essences + +Each essence is the brief you read before composing the 5 beats. Pick one from category in step 1 and write the actual `Says: "..."` lines tailored to the real product. + +#### HAUL_UNBOX +- **When to use & why**: fashion, handbags, jewelry, shoes, designer drops, streetwear, luxury cosmetics with packaging story, accessories — anything where brand packaging + texture/material is the value prop. Viewers convert on vicarious-unboxing dopamine + "I just got this" social proof; texture and hardware ARE what the customer pays for, so the close-up lands on materials, not function. Not TECH (→ TECH_UNBOX), not skincare/makeup application (→ BEAUTY_APPLY). +- **Sensory anchors**: tissue rustle, fabric slide, hardware clinks (chains / clasps / buckles), leather/fabric grain under fingertips, foil glint. +- **Setting**: white unmade bed in natural window light; bathroom mirror in background for the outro held-up reveal; streetwear drops may use desk/floor. +- **Close-up beat device**: NOT a screen — product close-up. `@Image2` is a product photo (or brand-site mobile view); the single finger-point lands on a hardware detail (chain, clasp, embossed logo). +- **Dialogue character**: hook is **mystery tease** — frame the unboxing as something the viewer doesn't yet know the contents of; do NOT name the product in the hook line. Arc: hook the unboxing mystery → brand name + drop context → reveal the material/silhouette while close-up holds on hardware → tactile/wearability insight (how it feels on the body) → punchline that invites the viewer to imagine themselves with the artifact. + +#### APP_REVEAL +- **When to use & why**: SaaS, AI tools, mobile/web apps, agent-style products, dev tools, productivity tools — anything where the screen IS the product. Viewers convert when they see live UI doing the thing in <5 seconds; the close-up beat is the demo, the bookends are the social proof. Not pure hardware (→ TECH_UNBOX). +- **Sensory anchors**: micro-thumb gesture, brand-color highlight, UI alive with small motion, ambient room tone. +- **Setting**: cozy bedroom or couch POV; jeans/joggers at frame edges; warm window light. +- **Close-up beat device**: laptop on bed (desktop screenshot) or phone in hand (mobile screenshot — set `mobile=true` in step 3). +- **Dialogue character**: hook is **bewildered curiosity** — the creator can't categorize the thing yet, that's the point. Do NOT use feature lists or marketing language in the hook; lean into "I don't know what to call this" / "this is wild" register that makes the viewer wait for the name. Arc: bewildered hook → name the product + interaction model in human terms ("you just talk to it", "it builds X from Y") → reveal what it produces (concrete comma-separated examples) while close-up shows the page → personal-insight twist (what it replaces / changes in the user's workflow) → punchline + implicit/explicit "go try it" CTA. + +#### FOOD_ASMR +- **When to use & why**: food brands, drinks, kitchen tools, snacks, restaurants with a takeout product — anything where the sensory peak (pour / sizzle / steam / first bite) carries the value prop. Viewers convert on hunger response — show the sensory peak, don't describe it. +- **Sensory anchors**: packaging rustle, knife-on-board, sizzle, pour stream, steam rising, satisfied exhale on the first bite. +- **Setting**: marble counter or warm wood kitchen, top-down framing. +- **Close-up beat device**: a product/dish close-up rather than a screen; phone in hand on the counter only if the brand has a delivery/recipe app. +- **Dialogue character**: hook is **show-don't-tell** — frame as a demonstration the viewer is watching unfold, not a description. The hook line lands while a hand or first ingredient is already in motion; the visual carries the curiosity. Arc: demonstration hook → name the product + first impression → narrate the sensory peak as it happens (pour / sizzle / steam) → satisfaction insight ("this is the new default") → punchline that hands off the recipe or shop link. + +#### BEAUTY_APPLY +- **When to use & why**: skincare, makeup, cosmetics, fragrance, hair products, body care — anything where before/after + application ritual is the value prop. Viewers convert on visual transformation under matched lighting; symmetry between hook and outro is what sells the result as real. Not packaging-heavy luxury (→ HAUL_UNBOX). +- **Sensory anchors**: pump press, squeeze, glide on skin, glow lift, droplet beading, brush sweep. +- **Setting**: bathroom mirror, natural daylight or vanity lighting; same angle for the hook and the outro. +- **Close-up beat device**: a product close-up (bottle / tube / compact held in hand), not a screen. +- **Dialogue character**: hook is **time-stamped social proof** — name the duration ("X days in", "morning of week 3", "after one tube") to signal real use rather than paid promo. The hook plants the symmetry payoff that arrives in the after-shot. Arc: timed-claim hook → name + key ingredient or claim → narrate the application as it happens (close-up of fingers/brush on skin) → after-shot reveal (same angle as hook) → punchline that signals exclusivity or repurchase intent. + +#### FITNESS_TRANSFORM +- **When to use & why**: workout equipment, supplements, recovery tools, activewear, fitness apps with tracking — anything where the work-to-result transformation is the value prop. Viewers convert on relatable struggle followed by earned payoff — showing the protein-shake bottle is not enough, you have to show the workout. +- **Sensory anchors**: heavy breathing, scoop hitting powder, equipment click, sweat catching light, post-workout exhale. +- **Setting**: gym or home-gym; workout gear at frame edges; floor or bench level. +- **Close-up beat device**: phone in hand showing app stats / heart rate / time elapsed, OR product packaging close-up (scoop in jar, bottle pour). +- **Dialogue character**: hook is **relatable resistance** — name the struggle / friction / not-wanting-to ("I did NOT want to do this", "almost skipped today", "this was supposed to be a rest day"); earns trust by sharing the tired feeling before showing the work. Arc: resistance hook → name the product + protocol ("I'm on day X of Y") → narrate mid-work moment while close-up shows the device or scoop → satisfaction insight that earns trust → punchline that frames continued use. + +#### TECH_UNBOX +- **When to use & why**: gadgets, hardware, electronics, smart-home devices, wearables, peripherals, AI hardware (Framework laptop, AirPods, Whoop, Rabbit r1, Friend pendant, mechanical keyboards, ergonomic gear) — anything where the device + first-use moment is the value prop. The box ceremony signals premium positioning; viewers convert on seeing "does it actually work / what does it do" — the first-use beat is the conversion moment. Not HAUL (→ HAUL_UNBOX), not pure software/SaaS (→ APP_REVEAL). +- **Sensory anchors**: utility-knife slice, plastic peel, foam slide-out, power-on chime, tactile button press, haptic click, fan spin-up. +- **Setting**: wood desk, top-down framing during unbox; handheld during first-use; desk/lap context for ongoing use. +- **Close-up beat device**: the device itself once unboxed and powered on. `@Image2` is typically a real photo of the device's screen at its key UI moment (first measurement, paired status, hero feature open); if the device has no screen, a clean hero photo of it mid-use. +- **Dialogue character**: hook is **arrival ceremony** — name that this is happening *now* ("just got this", "opening it"). Anticipation > description; the hook plants the question "what does it do?" that the first-use beat answers. Do NOT lead with specs. Arc: arrival hook → name + one-line spec headline → first-use reveal while close-up is on the device doing its thing → workflow-change insight ("this replaces / changes / fixes my X") → punchline that hands off urgency (price, where to find, time-limited). + +### 6. Voice input — fetch the user's voice sample (best-effort) + +Right before the generate call, fetch the user's voice sample URL: call `identity_voice_sample_url`. This returns a **short-lived** download URL (mp3/wav) backing the user's registered voice, OR `null` if no voice is on file. + +- If non-null → capture as `voice_sample_url` and pass it on the next call's `reference_audio` array. Both seedance and kling accept `reference_audio` (seedance up to 3, ≤15s combined; kling up to 8). The model uses the sample to clone the speaker's timbre for the lip-sync. +- If null → skip; the model uses its default voice. + +Always get this URL fresh right before step 7 — do NOT cache or reuse a stale URL across runs. + +### 7. Generate — first attempt with the avatar, cartoonize on rejection, retry + +Always attempt the call **first** with the avatar resolved in step 2 (caller-supplied or built-in fallback) exactly as-is. The skill does not pre-process or pre-judge it. Only when seedance rejects the call do we restyle. + +**7a. First attempt — avatar as-is** + +Call `generate_reference_video`: +- `provider`: `seedance` (default) or `kling` if user passed `provider=kling` +- `aspect_ratio`: `9:16` (default); `3:4` allowed only on seedance +- `resolution`: `720p` (seedance only) +- `duration`: 15 +- `reference_images`: `[avatar_url, screenshot_url]` (drop `screenshot_url` if step 3 failed) +- `reference_audio`: `[voice_sample_url]` (omit the param entirely if step 6 returned null) +- `prompt`: the multi-beat string from step 4 +- `sound`: true (default — ambient + lip-sync produced by the model) + +For `provider=kling`: convert the multi-beat prose into `shots: [{prompt, duration}, ...]` (5 shots × 3s = 15s sum), plus a top-level `prompt` summarizing the ad. References use `<<<image_1>>>` / `<<<image_2>>>` instead of `@Image1` / `@Image2`. + +If the call returns `{ task_id, status: "queued" }`, poll `task_status(task_id)` in a tight loop (no Bash, no sleep) until done. Capture `result.url` → `video_url` and proceed to step 8. + +**7b. On rejection — auto-cartoonize the avatar** + +If 7a returns `422 content_policy_violation` on `image_urls` / `reference_images` (seedance + fal-queue moderation flags portraits that read as too photorealistic — even some Pixar-style 3D avatars get flagged), restyle the avatar in-place: + +Call `generate_image`: +- `provider: "seedream"` (native Pixar/3D-animated look) +- `reference_image: <avatar_url>` +- `aspect_ratio`: same as the ad's aspect ratio +- `resolution: "1K"` +- `prompt: "Stylized 3D game character render — Unreal Engine 5 / Overwatch / Valorant / Apex Legends visual style. Anatomically grounded facial proportions with subtle stylization: slightly larger expressive eyes, defined sculpted cheekbone planes, smooth skin shader (smoother than photoreal, no micropore detail), idealized but believable features. PBR materials with subtle subsurface scattering, strand-based hair simulation, crisp cloth shader. Cinematic three-point studio lighting with strong rim light. Clearly a stylized AAA-game-character render — NOT photorealistic person, NOT Pixar plastic-toy cartoon, NOT exaggerated big-head proportions. Same person, same glasses, same outfit, same accessories. Centered medium portrait, neutral indoor background."` + +Capture returned URL → `avatar_url_cartoon`. + +**7c. Retry seedance with the cartoonized avatar** + +Re-run the exact same `generate_reference_video` call from 7a, swapping the avatar reference: `reference_images: [avatar_url_cartoon, screenshot_url]` (or `[avatar_url_cartoon]` if step 3 failed). All other params unchanged. Capture `result.url` → `video_url`. + +**7d. Final fallback — still rejected** + +If 7c also returns `content_policy_violation`, stop. Tell the user: the avatar reads as too realistic for seedance moderation even after auto-restyling; ask them to either supply a more stylized portrait themselves or rerun with `provider=kling` (kling has a separate moderation pipeline that accepts realistic avatars). + +### 8. Captions — single-shot styled burn (default on) + +Skip if `captions=false`. Use **one** `add_captions` call instead of chaining `edit_text_overlay` per chunk — much faster (≤5 min single call vs 5–8 min sequential), and the styles position captions correctly out of the box. + +Call `add_captions`: +- `video_url`: `video_url` from step 7 +- `style`: `"tiktok"` (default — word-by-word purple highlight, Bebas Neue, all caps, rendered at the **bottom** of the frame; classic TikTok-creator look that keeps the face and screen clear). Alternatives: `"hormozi"` (lower-middle yellow highlight, more aggressive — overlays part of the phone-in-hand close-up beat), `"classic"` (plain bottom subtitle bar, safest), `"karaoke"` (progressive color fill, also bottom). +- `font_size`: `60` — overrides the per-style default; tuned for 9:16 readability without dominating the frame. +- `language`: pass the BCP-47 code for the page language detected in step 1 (`"en"`, `"zh"`, `"ja"`, `"es"`, etc.) — skips auto-detect and avoids misrouting CJK to a Latin-only font path. + +Capture the returned URL → `final_url`. + +### 9. Return + +Return `final_url` on one line, plus a one-line summary: which category ran, whether the avatar was caller-supplied / built-in fallback / cartoonize-recovered, whether the screenshot was used or fell back to prose, whether the user's voice sample was used or default, the provider chosen, the language detected for dialogue, and whether captions were burned on. + +## Examples + +- `/pika:ugc-ads https://pika.me avatar_url=https://cdn/face.png` → APP_REVEAL, 9:16, seedance, real screenshot, captions on +- `/pika:ugc-ads https://maisonbrune.com avatar_url=https://cdn/face.png aspect_ratio=3:4` → HAUL_UNBOX, 3:4, seedance +- `/pika:ugc-ads https://pika.me avatar_url=https://cdn/face.png provider=kling captions=false` → APP_REVEAL, 9:16, kling shots[], no captions +- `/pika:ugc-ads https://pika.me` → no `avatar_url` → uses the built-in fallback Pixar-style female creator portrait, runs end-to-end diff --git a/tools-manifest.json b/tools-manifest.json new file mode 100644 index 0000000..ccdae16 --- /dev/null +++ b/tools-manifest.json @@ -0,0 +1,1969 @@ +[ + { + "name": "add_captions", + "description": "Auto-transcribe a video's audio (Whisper → Deepgram → Gemini waterfall, word-level timings) and burn styled captions onto the video. Four styles: `tiktok` (word-by-word purple highlight, Bebas Neue, all caps), `hormozi` (bold centered 1-3 words, yellow highlight), `classic` (bottom subtitle bar, semi-transparent), `karaoke` (word-by-word progressive color fill). CJK languages (zh/ja/ko) auto-detected and rendered with Noto CJK fonts bundled in the worker image.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "pattern": "^https:\\/\\/.*", + "description": "Source video URL (must be https://). Audio extracted server-side." + }, + "style": { + "default": "classic", + "description": "Caption visual style. tiktok / hormozi / karaoke = word-level highlight; classic = bottom subtitle bar.", + "type": "string", + "enum": [ + "tiktok", + "hormozi", + "classic", + "karaoke" + ] + }, + "language": { + "description": "Optional BCP-47 language hint (e.g. 'en', 'zh'). Auto-detect if omitted.", + "type": "string" + }, + "font_size": { + "description": "Optional font size override. Style-specific default applied if omitted.", + "type": "integer", + "minimum": 12, + "maximum": 200 + } + }, + "required": [ + "video_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "analyze_brief", + "description": "Extract a structured product brief from multiple sources: web pages, GitHub repos, PDFs, or raw text/chat history. Combines all sources and uses Gemini to output a normalised JSON brief (product name, tagline, features, pain points, tone, etc.) — ready for video script generation.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "sources": { + "minItems": 1, + "maxItems": 5, + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "url", + "github", + "text" + ], + "description": "\"url\" for any web page or PDF, \"github\" for a public repo (README extracted), \"text\" for raw text or chat history." + }, + "url": { + "description": "Required when type is url or github.", + "type": "string", + "format": "uri" + }, + "content": { + "description": "Required when type is text.", + "type": "string", + "maxLength": 50000 + } + }, + "required": [ + "type" + ] + }, + "description": "Input sources to extract product information from." + }, + "context": { + "description": "Optional extra context: target audience, video purpose, what to focus on.", + "type": "string", + "maxLength": 2000 + } + }, + "required": [ + "sources" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "analyze_media", + "description": "Analyze an image or short video with Gemini vision model.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "default": "gemini", + "description": "Vision provider. Default: `gemini` (gemini-3-pro). No alternative providers wired today.", + "type": "string", + "enum": [ + "gemini" + ] + }, + "media": { + "type": "string", + "format": "uri", + "description": "Image or video URL to analyze." + }, + "query": { + "type": "string", + "minLength": 1, + "description": "Natural-language question or directive about the media." + } + }, + "required": [ + "media", + "query" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "capture_website", + "description": "Capture a public website via server-side Playwright. mode=video (default): navigates to the URL and records for duration_s seconds; the recorder ping-pong-scrolls the page (top → bottom → top) so a 53s recording covers a typical README in motion. Returns video_url (MP4). mode=screenshot: executes actions then captures a single PNG; faster and more reliable for single-frame use cases. Returns image_url. Use timed_actions for frame-accurate scroll/click/zoom steps synchronized to a TTS audio track; use actions for natural-language steps translated by Claude. Set mobile=true for 390×844 iPhone viewport (portrait output). Only public https:// URLs are supported.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "format": "uri", + "pattern": "^https:\\/\\/.*", + "description": "The public URL to record or screenshot (must be https://)." + }, + "mode": { + "default": "video", + "description": "'video' (default) — record duration_s seconds, return video_url. 'screenshot' — execute actions then capture a PNG, return image_url.", + "type": "string", + "enum": [ + "video", + "screenshot" + ] + }, + "actions": { + "description": "Natural language description of actions to perform live during recording (e.g. 'scroll down slowly, click the sign-up button'). Translated to structured steps by Claude and executed by Playwright. Mutually exclusive with timed_actions.", + "type": "string" + }, + "timed_actions": { + "description": "Structured actions with explicit timestamps (seconds from page-ready). Use for frame-accurate sync with a TTS audio track. Mutually exclusive with actions.", + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "scroll", + "click", + "type", + "navigate", + "wait", + "zoom" + ] + }, + "at_s": { + "type": "number", + "minimum": 0, + "description": "Seconds from page-ready to fire this action." + }, + "selector": { + "description": "CSS selector. Scroll: element to scroll to. Click: element to click.", + "type": "string" + }, + "text_content": { + "description": "Visible heading text to scroll to — resolved at runtime via 7-pass DOM search.", + "type": "string" + }, + "bbox_selector": { + "description": "CSS selector to capture the post-action bbox of, separate from the action's own `selector`. Use when the action drives page state (e.g. `navigate`, px-based `scroll`) but the bbox you want measured is a different element on the resulting page — the worker scrolls `bbox_selector` into view (best-effort) before measuring. Timing replicates the legacy dual-emission follow-up: 600 ms post-action settle → scroll-into-view → 1300 ms post-scroll (1200 ms scroll-anim default + 100 ms) → measure. If both `selector` and `bbox_selector` are set, `bbox_selector` wins for the `action_bboxes` entry; `selector` still drives the action. Eliminates the dual-emission workaround where callers had to schedule a follow-up `scroll` step at `at_s + 0.6 s` purely to get a bbox.", + "type": "string" + }, + "px": { + "description": "Pixels to scroll (relative). For scroll actions without selector/text_content.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "direction": { + "description": "Scroll direction for px-based scroll.", + "type": "string", + "enum": [ + "up", + "down" + ] + }, + "duration_ms": { + "description": "Animation duration ms. Scroll default 1200, zoom default 800.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "text": { + "description": "Text to type. For type actions.", + "type": "string" + }, + "url": { + "description": "https:// URL to navigate to. For navigate actions.", + "type": "string" + }, + "ms": { + "description": "Milliseconds to wait. For wait actions.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "value": { + "description": "Zoom percent (e.g. 150). For zoom actions.", + "type": "integer", + "minimum": 25, + "maximum": 300 + } + }, + "required": [ + "type", + "at_s" + ] + } + }, + "mobile": { + "default": false, + "description": "When true, renders at 390×844 with iPhone UA — triggers mobile layouts and produces portrait output. Default false = 1280×720 desktop.", + "type": "boolean" + }, + "extra_css": { + "description": "CSS injected after page load. Use for loading web fonts (e.g. CJK fonts via @import). A 2-second settle is applied automatically.", + "type": "string" + }, + "duration_s": { + "default": 30, + "description": "Recording duration in seconds (default 30, max 300). Video mode only.", + "type": "integer", + "minimum": 10, + "maximum": 300 + } + }, + "required": [ + "url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "clone_voice", + "description": "Clone a custom voice from an audio/video file or an existing Kling video. Returns a voice_id you can bind to a Kling element via create_kling_element (element_voice_id). Audio must be 5-30s, single clean voice, no background noise.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "voice_name": { + "type": "string", + "maxLength": 20, + "description": "Voice name (max 20 chars)." + }, + "voice_url": { + "description": "Audio/video source URL (mp3/wav/mp4/mov). 5-30s, single clean voice. Mutually exclusive with video_id.", + "type": "string", + "format": "uri" + }, + "video_id": { + "description": "Existing Kling video ID to extract voice from. Must be a v2.6 sound=on video, avatar, or lipsync output. Mutually exclusive with voice_url.", + "type": "string" + } + }, + "required": [ + "voice_name" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "create_kling_element", + "description": "Create a Kling subject (element) from image or video references. Returns an element_id you can pass to generate_reference_video as element_ids[] for character/style consistency in kling omni r2v.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "element_name": { + "type": "string", + "maxLength": 20, + "description": "Subject name (max 20 chars)." + }, + "element_description": { + "type": "string", + "maxLength": 100, + "description": "Subject description (max 100 chars)." + }, + "reference_type": { + "type": "string", + "enum": [ + "image_refer", + "video_refer" + ], + "description": "\"image_refer\": define from multi-angle photos (all styles). \"video_refer\": define from a short video clip (realistic human only, kling-v3+ models)." + }, + "frontal_image": { + "description": "Front-facing reference image URL. Required when reference_type=image_refer.", + "type": "string", + "format": "uri" + }, + "refer_images": { + "description": "1-3 additional angle/detail reference images. Required when reference_type=image_refer.", + "minItems": 1, + "maxItems": 3, + "type": "array", + "items": { + "type": "string", + "format": "uri" + } + }, + "refer_video": { + "description": "Reference video URL (MP4/MOV, 3-8s, 1080p, 16:9 or 9:16). Required when reference_type=video_refer.", + "type": "string", + "format": "uri" + }, + "element_voice_id": { + "description": "Bind an existing voice ID to this element. Get one from clone_voice.", + "type": "string" + }, + "tag_ids": { + "description": "Category tags: o_101=趣梗 o_102=人物 o_103=动物 o_104=道具 o_105=服饰 o_106=场景 o_107=特效 o_108=其他", + "type": "array", + "items": { + "type": "string", + "enum": [ + "o_101", + "o_102", + "o_103", + "o_104", + "o_105", + "o_106", + "o_107", + "o_108" + ] + } + } + }, + "required": [ + "element_name", + "element_description", + "reference_type" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_animate_zoom", + "description": "Apply animated zoom-in/hold/zoom-out segments to a video, each centered on pixel coordinates with an ease-out ramp. Mirrors the original github_explainer zoom step packaged as an atomic primitive. Use to highlight specific UI elements during an explainer recording — feed (cx, cy) from `capture_website` element bounding boxes.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "pattern": "^https:\\/\\/.*", + "description": "Source video URL (must be https://). Audio preserved if present." + }, + "zoom_keyframes": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "t_start": { + "type": "number", + "minimum": 0, + "description": "Segment start time (seconds)." + }, + "t_end": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Segment end time (seconds). Must be > t_start." + }, + "cx": { + "type": "number", + "minimum": 0, + "description": "Zoom-target X in source frame pixels." + }, + "cy": { + "type": "number", + "minimum": 0, + "description": "Zoom-target Y in source frame pixels." + }, + "scale": { + "type": "number", + "exclusiveMinimum": 1, + "maximum": 4, + "description": "Target zoom factor. 1.35 matches the original recipe; max 4.0." + }, + "ramp_s": { + "default": 0.5, + "description": "Ease-in/out ramp duration. Held flat at target between.", + "type": "number", + "exclusiveMinimum": 0, + "maximum": 5 + } + }, + "required": [ + "t_start", + "t_end", + "cx", + "cy", + "scale" + ] + }, + "description": "Non-overlapping zoom segments, sorted by t_start. Up to 32 segments." + } + }, + "required": [ + "video_url", + "zoom_keyframes" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_audio_mix", + "description": "Overlay an audio track (music, voiceover) onto a video. Original video audio is preserved; the new track is mixed at the specified volume.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "description": "URL of the video" + }, + "audio_url": { + "type": "string", + "format": "uri", + "description": "URL of the audio file to overlay" + }, + "audio_volume": { + "default": 0.25, + "description": "Volume of overlaid audio (0–1). Default 0.25.", + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "video_url", + "audio_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_audio_trim", + "description": "Trim an audio file to a specified time range. Sample-accurate — re-encodes by default so output boundaries land exactly on the requested seconds (lossy codecs like mp3/aac/opus can't be byte-copied at arbitrary offsets without snapping to the previous codec frame). Supports mp3, wav, m4a, aac, flac, ogg, opus. Use to split a longer narration into segments for per-chunk lipsync, extract a clip, or convert format.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "audio_url": { + "type": "string", + "format": "uri", + "description": "URL of the audio file to trim (mp3, wav, m4a, aac, flac, ogg, opus)" + }, + "start_s": { + "description": "Start time in seconds (inclusive). Default 0 (start of file).", + "type": "number", + "minimum": 0 + }, + "end_s": { + "type": "number", + "exclusiveMinimum": 0, + "description": "End time in seconds (exclusive). Must be greater than start_s." + }, + "output_format": { + "description": "Output container format. Defaults to the input file's format (preserves codec for fastest path).", + "type": "string", + "enum": [ + "mp3", + "wav", + "m4a", + "aac", + "flac", + "ogg", + "opus" + ] + } + }, + "required": [ + "audio_url", + "end_s" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_beat_sync", + "description": "Align video cuts or zoom pulses to musical beats. Caller supplies the music track via music_url (chain generate_music first if needed) and the BPM. Pure ffmpeg + ffprobe — no provider calls. Typical runtime 30-90s for h264 input; HEVC/4K + mode=both can take 90-150s. Returns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "description": "Source video URL." + }, + "music_url": { + "type": "string", + "format": "uri", + "description": "Music track URL. Chain generate_music first if you don't have one. Audio formats accepted by ffmpeg work (mp3, m4a, wav, etc.)." + }, + "bpm": { + "type": "integer", + "minimum": 30, + "maximum": 300, + "description": "Beats per minute of the music track. Required (intentionally — the hub-skill's silent-disable footgun when bpm is unknown is structurally impossible here). Range is wider than internal defaults (65-130 in plugin's vibe-to-BPM table) by design." + }, + "mode": { + "default": "auto", + "description": "`sync` re-times scene cuts to beats; `zoom` adds beat-aligned zoom pulses; `both` runs sync then zoom; `auto` picks based on scene-cut count + audio_mode. `none` is intentionally not exposed — caller uses edit_audio_mix for that case.", + "type": "string", + "enum": [ + "sync", + "zoom", + "both", + "auto" + ] + }, + "audio_mode": { + "default": "auto", + "description": "`replace` = music only; `mix` = duck music under original audio via sidechain compressor; `auto` = mix if input has audio, else replace.", + "type": "string", + "enum": [ + "replace", + "mix", + "auto" + ] + }, + "beat_intensity": { + "default": "medium", + "description": "Zoom-pulse density. low = every 4th beat, medium = every 2nd, high = every beat. No effect on `sync` mode.", + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + }, + "orig_video_url": { + "description": "Optional separate audio source for `mix` mode when video_url's audio was stripped by upstream beat effects. Falls back to video_url's audio track if omitted.", + "type": "string", + "format": "uri" + }, + "speed_clamp_min": { + "default": 0.8, + "description": "`sync` mode: minimum playback speed.", + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "speed_clamp_max": { + "default": 1.25, + "description": "`sync` mode: maximum playback speed.", + "type": "number", + "minimum": 1, + "maximum": 2 + }, + "zoom_scale": { + "default": 1.04, + "description": "`zoom` mode: scale factor at beat peaks.", + "type": "number", + "minimum": 1.01, + "maximum": 1.2 + }, + "zoom_window_s": { + "default": 0.12, + "description": "`zoom` mode: half-width of each zoom pulse window.", + "type": "number", + "minimum": 0.05, + "maximum": 0.5 + } + }, + "required": [ + "video_url", + "music_url", + "bpm" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_browser_frame", + "description": "Wrap a video inside a 1280×800 macOS Sonoma desktop + browser-window chrome (traffic lights, tab title, URL address bar, drop shadow). Use to give explainer recordings a polished 'watching a browser' look. Output is always 1280×800 regardless of input dimensions; the input is letterboxed/pillarboxed to fit the 1168×637 content area.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "pattern": "^https:\\/\\/.*", + "description": "Source video URL (must be https://). Audio preserved if present." + }, + "style": { + "default": "mac", + "description": "Output style. Currently only 'mac' (Sonoma desktop).", + "type": "string", + "enum": [ + "mac" + ] + }, + "url": { + "default": "", + "description": "URL string shown in the address bar. Plain text — not validated as a real URL. Truncated at 65 chars in the display.", + "type": "string", + "maxLength": 80 + }, + "tab_title": { + "default": "", + "description": "Tab title shown in the browser tab. Truncated at 30 chars in the display.", + "type": "string", + "maxLength": 40 + } + }, + "required": [ + "video_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_concat", + "description": "Join multiple video clips into a single video, played in sequence. Useful for multi-shot workflows where each clip was generated separately.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_urls": { + "minItems": 2, + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "List of video URLs to concatenate in order" + } + }, + "required": [ + "video_urls" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_pip", + "description": "Overlay one video on top of another as a picture-in-picture window. Supports rectangular or circular overlay shape. Use for tutorials, reaction videos, or avatar overlays.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "main_video_url": { + "type": "string", + "format": "uri", + "description": "URL of the background video" + }, + "overlay_video_url": { + "type": "string", + "format": "uri", + "description": "URL of the overlay video" + }, + "position": { + "default": "bottom_right", + "description": "Corner to place the overlay (ignored if `position_px` is supplied)", + "type": "string", + "enum": [ + "top_left", + "top_right", + "bottom_left", + "bottom_right" + ] + }, + "position_px": { + "description": "Pixel-precise placement on the main video. Overrides `position`. Use to escape the corner-margin model — e.g. `{x: 20, y: H-h-78}` to leave clearance for a Mac-dock area below the overlay.", + "type": "object", + "properties": { + "x": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "y": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "x", + "y" + ] + }, + "size": { + "description": "Size of overlay relative to main video (0.05–0.5). Defaults to 0.25 inside the worker when omitted. Mutually exclusive with `size_px` — leaving this off lets callers use `size_px` for absolute pixel sizing.", + "type": "number", + "minimum": 0.05, + "maximum": 0.5 + }, + "size_px": { + "description": "Absolute pixel diameter for the scaled overlay (square). Mutually exclusive with `size`. Use when the caller needs the overlay to land at a specific pixel size on the main video regardless of the overlay's intrinsic dimensions. Upper bound is also capped at min(main_w, main_h) at runtime once the main video is probed.", + "type": "integer", + "minimum": 32, + "maximum": 2160 + }, + "shape": { + "default": "rectangle", + "description": "Shape of the overlay window", + "type": "string", + "enum": [ + "rectangle", + "circle" + ] + }, + "stroke_width_px": { + "default": 0, + "description": "Outline thickness around a `circle` overlay, in pixels (0 = no stroke). Requires `shape: 'circle'`. Max 20.", + "type": "integer", + "minimum": 0, + "maximum": 20 + }, + "stroke_color": { + "default": "white", + "description": "Outline color when `stroke_width_px > 0`. Restricted palette to keep the YUV color math simple — file a follow-up if you need hex/RGB.", + "type": "string", + "enum": [ + "white", + "black", + "red", + "green", + "blue" + ] + } + }, + "required": [ + "main_video_url", + "overlay_video_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_text_overlay", + "description": "Burn text onto a video at a specified position and optional time range. Use for branding, CTAs, watermarks, or titles.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "description": "URL of the video" + }, + "text": { + "type": "string", + "description": "Text to display on the video" + }, + "position": { + "default": "bottom_center", + "description": "Where to place the text", + "type": "string", + "enum": [ + "top_left", + "top_center", + "top_right", + "center", + "bottom_left", + "bottom_center", + "bottom_right" + ] + }, + "font_size": { + "default": 48, + "description": "Font size in pixels", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "font_color": { + "default": "white", + "description": "Font color (ffmpeg color name or hex)", + "type": "string" + }, + "start_s": { + "description": "Show text starting at this second", + "type": "number" + }, + "end_s": { + "description": "Hide text after this second", + "type": "number" + } + }, + "required": [ + "video_url", + "text" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "edit_trim", + "description": "Trim a video to a specified time range and/or convert format. Use to extract a segment from a longer video.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "description": "URL of the video to trim" + }, + "start_s": { + "description": "Start time in seconds", + "type": "number" + }, + "end_s": { + "description": "End time in seconds", + "type": "number" + }, + "output_format": { + "default": "mp4", + "description": "Output container format", + "type": "string", + "enum": [ + "mp4", + "webm", + "mov" + ] + } + }, + "required": [ + "video_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "extract_frame", + "description": "Extract a single frame from a video as a PNG image. Uses fast input-side seek so extraction is near-instant even for long videos. The returned PNG URL can be used as reference_image in generate_image (e.g. capture_website → extract_frame → generate_image to generate a laptop mockup using the real website screenshot as reference).\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "video_url": { + "type": "string", + "format": "uri", + "description": "URL of the video to extract a frame from" + }, + "time_s": { + "description": "Timestamp in seconds to extract (default 0 = first frame)", + "type": "number", + "minimum": 0 + } + }, + "required": [ + "video_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_image", + "description": "Generate an image. One tool, multiple providers via the `provider` enum. **Default: nano-banana-pro** (Gemini 3 Pro — best balance of quality + character/style consistency). Alternatives: gemini-flash-image (fastest, lower fidelity); seedream (cinematic + 2K/4K); gpt-image-2 (best for editing an existing image, supports `quality` + `output_format`). When the user expresses preference for speed, cost, image editing, or 4K resolution, surface the provider choice and let them override — otherwise stick with the default.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "Which image model. Default nano-banana-pro. gpt-image-2: best for editing an existing image (supports `quality` + `output_format`). seedream: cinematic + 2K/4K. gemini-flash-image: fastest, lower fidelity. Surface this choice when the user shows a cost/speed/quality preference.", + "type": "string", + "enum": [ + "nano-banana-pro", + "gemini-flash-image", + "seedream", + "gpt-image-2" + ] + }, + "prompt": { + "type": "string", + "minLength": 1, + "description": "Description of the image. Be specific about subject, style, lighting." + }, + "reference_image": { + "description": "Optional reference image URL for style / character consistency. On gpt-image-2 this routes through the /v1/images/edits endpoint (image-to-image editing).", + "type": "string", + "format": "uri" + }, + "aspect_ratio": { + "description": "Output aspect ratio (default 1:1).", + "type": "string", + "enum": [ + "1:1", + "16:9", + "9:16", + "3:4", + "4:3" + ] + }, + "resolution": { + "description": "Output resolution (default 1K). 4K supported on seedream only.", + "type": "string", + "enum": [ + "1K", + "2K", + "4K" + ] + }, + "quality": { + "description": "Image quality, gpt-image-2 only. `low` is ~10–20s; `medium`/`auto` are slower. `high` is intentionally NOT exposed — it exceeds the proxy 180s read timeout. Rejected on other providers.", + "type": "string", + "enum": [ + "auto", + "low", + "medium" + ] + }, + "output_format": { + "description": "Output image format, gpt-image-2 only. Defaults to png upstream. Rejected on other providers.", + "type": "string", + "enum": [ + "png", + "jpeg", + "webp" + ] + } + }, + "required": [ + "prompt" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_keyframes_video", + "description": "Generate a video transitioning between two keyframe images. Uses pika-api/generate/2.2/pikaframes (model pika-2.2-pikaframes).\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "first_frame": { + "type": "string", + "format": "uri", + "description": "First keyframe image URL." + }, + "last_frame": { + "type": "string", + "format": "uri", + "description": "Last keyframe image URL." + }, + "prompt": { + "description": "Transition prompt — motion between frames.", + "type": "string" + }, + "negative_prompt": { + "description": "Negative prompt — what to avoid.", + "type": "string" + }, + "duration": { + "description": "Output duration in seconds (5 or 10; default 5).", + "anyOf": [ + { + "type": "number", + "const": 5 + }, + { + "type": "number", + "const": 10 + } + ] + }, + "resolution": { + "description": "Output resolution (default 720p).", + "type": "string", + "enum": [ + "720p", + "1080p" + ] + }, + "seed": { + "description": "Integer seed for reproducibility.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + } + }, + "required": [ + "first_frame", + "last_frame" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_lipsync", + "description": "Sync a face (portrait image) to an audio track — talking or singing head video. Providers: kling (/v1/videos/avatar/image2video, default) or pika (parrot audio-to-video).\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "kling (default) or pika", + "type": "string", + "enum": [ + "kling", + "pika" + ] + }, + "image": { + "type": "string", + "format": "uri", + "description": "Source portrait image URL." + }, + "audio": { + "type": "string", + "format": "uri", + "description": "Source audio URL to sync to." + }, + "prompt": { + "description": "Optional style/motion hint (parrot only).", + "type": "string" + } + }, + "required": [ + "image", + "audio" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_motion_control_video", + "description": "Transfer motion from a reference video onto a character image. Providers: pika (parrot-staging/animate, default) supports promptText + resolution. kling (/v1/videos/motion-control) uses the std/pro mode knob.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "pika (default) or kling", + "type": "string", + "enum": [ + "pika", + "kling" + ] + }, + "image": { + "type": "string", + "format": "uri", + "description": "Character image URL." + }, + "video": { + "type": "string", + "format": "uri", + "description": "Motion reference video URL." + }, + "prompt": { + "description": "Optional motion / style hint.", + "type": "string" + }, + "resolution": { + "description": "Output resolution (pika only; default 720p).", + "type": "string", + "enum": [ + "480p", + "720p", + "1080p" + ] + }, + "model": { + "description": "Kling model id (kling only; default kling-v3).", + "type": "string", + "enum": [ + "kling-v3", + "kling-v2-6" + ] + }, + "mode": { + "description": "Kling generation mode (kling only; default std).", + "type": "string", + "enum": [ + "std", + "pro" + ] + } + }, + "required": [ + "image", + "video" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_music", + "description": "Generate music with vocals via MiniMax music-2.5. Duration is determined by lyric length (typically 20-60s); pass multi-section [verse]/[chorus] lyrics to get longer tracks.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "default": "minimax-music", + "description": "Which music model. Default minimax-music (currently the only provider).", + "type": "string", + "enum": [ + "minimax-music" + ] + }, + "prompt": { + "type": "string", + "minLength": 1, + "description": "Style/genre/mood description." + }, + "lyrics": { + "description": "Song lyrics with optional [verse]/[chorus] section tags. Output length scales with lyric length; omit for a ~33s instrumental.", + "type": "string" + } + }, + "required": [ + "prompt" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_reference_video", + "description": "Generate a video from multi-modal reference inputs (images, videos, audio) used as character / style / motion anchors — NOT as start frames. **Default: kling** (omni-video, sound-capable, up to 7 image / 3 video / 8 audio refs). Specialist: seedance — cinematic look, supports `fast` / `seed` / `resolution` / `auto_duration`, accepts up to 9 image / 3 video / 3 audio refs (combined ≤ 12).\n\nReference assets in prompt via <<<image_1>>>/<<<video_1>>> (kling) or @Image1/@Video1/@Audio1 (seedance).\n\nSound defaults to on; pass sound=false to suppress. For text- or single-image-driven video use `generate_video` instead.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "Which video model. **Default: kling** — fastest + cheapest, omni-video supports sound + up to 7/3/8 refs. seedance for cinematic look, fast tier, seed control, auto duration.", + "type": "string", + "enum": [ + "kling", + "seedance" + ] + }, + "prompt": { + "type": "string", + "minLength": 1, + "description": "Description or motion direction. Use @Image1 / @Video1 / @Audio1 tokens (seedance) or <<<image_1>>> / <<<video_1>>> tokens (kling) to anchor specific references." + }, + "reference_images": { + "description": "Reference image URLs. **Multi-ref is supported — pass an array of character / style anchors (NOT start frames).** Per-provider caps: seedance up to 9 (jpeg/png/webp, ≤30MB each); kling up to 7. Requires at least one entry here or in reference_videos.", + "maxItems": 9, + "type": "array", + "items": { + "type": "string", + "format": "uri" + } + }, + "reference_videos": { + "description": "Reference video URLs (motion / style). Up to 3 videos. Requires at least one entry here or in reference_images.", + "maxItems": 3, + "type": "array", + "items": { + "type": "string", + "format": "uri" + } + }, + "reference_audio": { + "description": "Reference audio URLs. **Multi-ref is supported on both providers — pass an array.** Per-provider caps: seedance accepts up to 3 (combined ≤ 15s, mp3 / wav only); kling accepts up to 8 (mp3 / wav / m4a). Reference via @Audio1/@Audio2/@Audio3 tokens (seedance) or paired with the prompt (kling).", + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "format": "uri" + } + }, + "duration": { + "description": "Length in seconds (default 5). Per-provider valid values: kling 3-15 · seedance 4-15. For seedance auto-length, pass `auto_duration: true` instead — `duration` is then ignored.", + "type": "integer", + "minimum": 3, + "maximum": 20 + }, + "auto_duration": { + "description": "Let fal pick the best clip length automatically. seedance only — rejected on kling. When true, the `duration` field is ignored. Default false.", + "type": "boolean" + }, + "aspect_ratio": { + "description": "Output aspect ratio (default 16:9). `21:9`, `4:3`, `3:4`, `auto` are seedance-only — passing them on kling is rejected.", + "type": "string", + "enum": [ + "16:9", + "9:16", + "1:1", + "21:9", + "4:3", + "3:4", + "auto" + ] + }, + "sound": { + "description": "Enable ambient audio. If omitted, defaults to true on both providers. Pass sound=false explicitly to suppress audio.", + "type": "boolean" + }, + "seed": { + "description": "Reproducibility seed. seedance only (echoed in result). Rejected on kling.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "fast": { + "description": "Use the fast (cheaper, ~20% off) seedance tier. seedance only. Caps resolution at 720p — combining `fast=true` with `resolution=1080p` is rejected.", + "type": "boolean" + }, + "resolution": { + "description": "Output resolution. seedance only. Default 720p. 1080p requires `fast=false`.", + "type": "string", + "enum": [ + "480p", + "720p", + "1080p" + ] + }, + "shots": { + "description": "Multi-shot narrative segments. kling only. Up to 6 shots, each with its own prompt + duration. Sum of `shot.duration` MUST equal the top-level `duration`.", + "maxItems": 6, + "type": "array", + "items": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "minLength": 1 + }, + "duration": { + "type": "integer", + "minimum": 1, + "maximum": 15 + } + }, + "required": [ + "prompt", + "duration" + ] + } + }, + "prompt_adherence": { + "description": "How strictly to follow the prompt. loose ≈ 0.3 (more creative interpretation), balanced ≈ 0.5 (default), strict ≈ 0.85 (literal). Kling only.", + "type": "string", + "enum": [ + "loose", + "balanced", + "strict" + ] + }, + "negative_prompt": { + "description": "What the model should AVOID generating. Kling only. Common patterns: 'blur, distortion, low quality'; 'watermarks, text overlay, logos'; 'deformed anatomy, extra fingers'.", + "type": "string", + "maxLength": 2500 + }, + "watermark": { + "description": "Burn kling's watermark into the output. Default unset = kling applies its own (account-tier-dependent). Pass `false` for clean brand content. Kling only.", + "type": "boolean" + }, + "image_types": { + "description": "Per-image role for reference_images (parallel array, same length). \"first_frame\" → starting frame; \"end_frame\" → ending frame; \"reference\" or null → plain style/character reference. Kling only. kling-video-o1 limits first/end frame usage when >2 images total.", + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "enum": [ + "first_frame", + "end_frame", + "reference" + ] + }, + { + "type": "null" + } + ] + } + }, + "video_refer_types": { + "description": "Per-video role for reference_videos (parallel array, same length). \"feature\" = character/style reference; \"base\" = video to restyle/edit. Default: all feature. Kling only.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "feature", + "base" + ] + } + }, + "video_keep_sounds": { + "description": "Per-video flag: preserve original audio track (parallel array to reference_videos). Default: false for all. Kling only.", + "type": "array", + "items": { + "type": "boolean" + } + }, + "element_ids": { + "description": "Kling subject library element IDs to bind (up to 3). Create elements first with create_kling_element. Kling only.", + "minItems": 1, + "maxItems": 3, + "type": "array", + "items": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + } + }, + "kling_model": { + "description": "Omni model variant (default \"kling-v3-omni\"). Kling only.", + "type": "string", + "enum": [ + "kling-v3-omni", + "kling-video-o1" + ] + }, + "quality_mode": { + "description": "Output quality tier. \"std\" = 720p; \"pro\" = 1080p (default); \"4k\" = 4K. Kling only.", + "type": "string", + "enum": [ + "std", + "pro", + "4k" + ] + }, + "intelligence_shot": { + "description": "Let Kling auto-determine shot breakdown from the prompt (shot_type=intelligence). Mutually exclusive with shots[]. Kling only.", + "type": "boolean" + }, + "voice_ids": { + "description": "Cloned voice IDs from `clone_voice`. Pass voice_ids for previously-cloned voices (small payload — Kling stores the binary server-side); pass `reference_audio` for ad-hoc inline audio (forces base64 binary in the request body — large). **Use voice_ids whenever possible** to avoid request-size limits. Combined `len(voice_ids) + len(reference_audio)` is capped at 8 (Kling upstream cap). Kling only — rejected on seedance.", + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "callback_url": { + "description": "Webhook URL Kling will POST to when generation completes. Useful for fire-and-forget patterns where the caller doesn't want to poll. Must be https://. Kling only.", + "type": "string", + "format": "uri", + "pattern": "^https:\\/\\/.*" + }, + "external_task_id": { + "description": "Caller-supplied tracing identifier passed through to Kling (echoed back in upstream task records). Useful for correlating Pika job IDs with downstream systems. ≤64 chars. Kling only.", + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "required": [ + "prompt" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_slide_animation", + "description": "Generate an animated motion-graphics video from a text prompt. Claude writes the HTML composition (scene structure, typography, GSAP animations, shader transitions); the HyperFrames engine renders it deterministically to MP4. Use for: title cards, product launch teasers, animated logo reveals, explainer slides, text-on-screen intros, designed motion graphics, and any 'slide animation' / 'animated text' / 'animated slides' request. NOT for real-footage video (use `generate_video`), still-image-to-video transitions (use `generate_keyframes_video`), or assembling existing clips (use `edit_concat`). Default returns the HTML CDN URL only (~30s); set `record_video=true` to also render an MP4 via deterministic per-frame seek (~30–90s typical, longer for dense shader work).\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "maxLength": 1000, + "description": "Describe the slide deck — style, content, mood, audience. Example: \"A dark energetic product launch for a fitness app. Bold typography, neon accents.\"" + }, + "brief": { + "description": "Optional structured product context. Merged with prompt.", + "type": "object", + "properties": { + "product_name": { + "type": "string" + }, + "tagline": { + "type": "string" + }, + "key_features": { + "type": "array", + "items": { + "type": "string" + } + }, + "tone": { + "type": "string", + "enum": [ + "professional", + "technical", + "casual", + "playful" + ] + }, + "call_to_action": { + "type": "string" + } + } + }, + "aspect_ratio": { + "description": "Canvas aspect ratio (default \"9:16\").", + "type": "string", + "enum": [ + "9:16", + "16:9", + "1:1" + ] + }, + "record_video": { + "description": "When true, renders the animation to MP4 via the HyperFrames engine (deterministic per-frame seek, not real-time recording). Returns the video/mp4 URL as the primary result (html_url also included in structured output). Adds ~30–90s to total run time depending on animation length.", + "type": "boolean" + }, + "fps": { + "description": "Render frame rate (24, 30, or 60). Default 30. Only applies when record_video=true.", + "anyOf": [ + { + "type": "number", + "const": 24 + }, + { + "type": "number", + "const": 30 + }, + { + "type": "number", + "const": 60 + } + ] + }, + "quality": { + "description": "HyperFrames render quality preset. `draft` = fastest (lower bitrate), `standard` = default, `high` = max quality (slower). Only applies when record_video=true.", + "type": "string", + "enum": [ + "draft", + "standard", + "high" + ] + } + }, + "required": [ + "prompt" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_speech", + "description": "Generate speech from text. Default: minimax-tts (fast). elevenlabs: premium, more voice options.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "default": "minimax-tts", + "description": "minimax-tts or elevenlabs. Default minimax-tts.", + "type": "string", + "enum": [ + "minimax-tts", + "elevenlabs" + ] + }, + "text": { + "type": "string", + "minLength": 1, + "maxLength": 10000, + "description": "Text to speak. Max 10000 chars." + }, + "voice_id": { + "description": "Provider-specific voice ID.", + "type": "string" + }, + "language": { + "default": "en", + "description": "BCP-47 language tag (e.g. 'en', 'zh', 'es').", + "type": "string" + } + }, + "required": [ + "text" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "generate_video", + "description": "Generate a video from a text prompt or a single image. One tool, multiple providers via the `provider` enum. **Default: kling** — fastest + cheapest, supports ambient sound/music. Prefer kling unless the caller's request has a concrete reason that maps to a specialist.\n\nFor reference-based generation (multi-ref images / videos / audio as character or style anchors) use `generate_reference_video` instead.\n\nSpecialist alternatives: veo3 — use when the clip needs synthesized natural-language dialogue or voice-over (higher cost). sora — up to 20s, premium quality. seedance — cinematic look, supports `end_image` / `fast` / `seed` / `resolution`. minimax — Hailuo, image_to_video only. pika — parrot i2v.\n\nSound defaults to on for sound-capable providers (kling / veo3 / sora / seedance); pass sound=false to suppress. Passing sound=true on pika or minimax is rejected.\n\nTwo modes: text_to_video (default), image_to_video (requires `image`).\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "Which video model. **Default: kling** — always prefer kling unless the caller's request has a concrete reason that maps to a specialist (veo3 for natural-language dialogue / voice-over; sora for 15–20s premium clips; seedance for cinematic look, supports `end_image` / `fast` / `seed` / `resolution`). A generic request for any video with sound/music/ambient audio should use kling — kling supports sound natively.", + "type": "string", + "enum": [ + "kling", + "pika", + "veo3", + "sora", + "seedance", + "minimax" + ] + }, + "mode": { + "description": "text_to_video (default): from prompt only. image_to_video: animate a source image as the first frame (requires `image`). minimax requires image_to_video. For reference-based generation (multi-ref images / videos / audio) use the sibling tool `generate_reference_video`.", + "type": "string", + "enum": [ + "text_to_video", + "image_to_video" + ] + }, + "prompt": { + "type": "string", + "minLength": 1, + "description": "Description or motion direction for the video." + }, + "image": { + "description": "Source image URL. Required when mode=image_to_video.", + "type": "string", + "format": "uri" + }, + "duration": { + "description": "Length in seconds (default 5; **sora defaults to 4** since 5 isn't in its accepted set). Per-provider valid values: kling 5 or 10 · pika 5 or 10 · veo3 5–8 · seedance 3–12 · minimax 6 · **sora 4, 8, or 12 (no other values accepted)**.", + "type": "integer", + "minimum": 3, + "maximum": 20 + }, + "aspect_ratio": { + "description": "Output aspect ratio (default 16:9). `21:9`, `4:3`, `3:4`, `auto` are seedance-only — passing them on other providers is rejected.", + "type": "string", + "enum": [ + "16:9", + "9:16", + "1:1", + "21:9", + "4:3", + "3:4", + "auto" + ] + }, + "sound": { + "description": "Enable ambient audio. If omitted, defaults to true on providers that support sound (kling, veo3, sora, seedance) and to false on providers that don't (pika, minimax). Pass sound=false explicitly to suppress audio on a sound-capable provider; passing sound=true on pika or minimax is rejected.", + "type": "boolean" + }, + "end_image": { + "description": "Optional end-frame image URL for image_to_video transitions (start→end morph). seedance only. Rejected on other providers.", + "type": "string", + "format": "uri" + }, + "seed": { + "description": "Reproducibility seed. seedance only (echoed in result). Rejected on other providers.", + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "fast": { + "description": "Use the fast (cheaper, ~20% off) seedance tier. seedance only. Caps resolution at 720p — combining `fast=true` with `resolution=1080p` is rejected.", + "type": "boolean" + }, + "resolution": { + "description": "Output resolution. seedance only. Default 720p. 1080p requires `fast=false`. Other providers ignore this.", + "type": "string", + "enum": [ + "480p", + "720p", + "1080p" + ] + }, + "shots": { + "description": "Multi-shot narrative segments (kling t2v only; rejected on kling i2v). Up to 6 shots, each with its own prompt + duration. Sum of `shot.duration` MUST equal the top-level `duration`.", + "maxItems": 6, + "type": "array", + "items": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "minLength": 1 + }, + "duration": { + "type": "integer", + "minimum": 1, + "maximum": 15 + } + }, + "required": [ + "prompt", + "duration" + ] + } + }, + "prompt_adherence": { + "description": "How strictly to follow the prompt. loose ≈ 0.3 (more creative interpretation), balanced ≈ 0.5 (default, kling's natural feel), strict ≈ 0.85 (literal, every detail honored). Kling only.", + "type": "string", + "enum": [ + "loose", + "balanced", + "strict" + ] + }, + "negative_prompt": { + "description": "What the model should AVOID generating. Common patterns to combine: general quality → 'blur, distortion, low quality'; clean look → 'watermarks, text overlay, logos'; character work → 'deformed anatomy, extra fingers'; single-shot → 'cuts, scene transitions'. Kling only — leave empty unless you have a specific artifact to suppress.", + "type": "string", + "maxLength": 2500 + }, + "watermark": { + "description": "Burn kling's watermark into the output. Default unset = kling applies its own (account-tier-dependent). Pass `false` for clean brand content. Kling only.", + "type": "boolean" + } + }, + "required": [ + "prompt" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_avatar_url", + "description": "Return the user's onboarding-generated avatar image URL. Use as the `image` param for generate_lipsync or as `reference_image` for generate_image. Returns null when the user has no avatar on file — call `identity_set_avatar` to create one.", + "inputSchema": { + "type": "object", + "properties": {}, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_memory_append", + "description": "Write a new entry to the agent's daily memory file in memory/YYYY-MM-DD.md. Use this to record important facts, user preferences, or session learnings.", + "inputSchema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The memory content to store" + }, + "source": { + "default": "agent", + "description": "Source of the memory (e.g. 'agent', 'user', 'skill')", + "type": "string" + } + }, + "required": [ + "text" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_memory_search", + "description": "Search long-term memory (MEMORY.md and memory/*.md) by keyword. Returns matching lines up to top_k results.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Keyword or phrase to search for in memory" + }, + "top_k": { + "default": 5, + "description": "Maximum number of matching lines to return (default 5)", + "type": "number" + } + }, + "required": [ + "query" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_persona_read", + "description": "Read a persona file from the agent workspace (IDENTITY.md, SOUL.md, or STYLE.md).", + "inputSchema": { + "type": "object", + "properties": { + "file": { + "default": "identity", + "description": "Which persona file to read: \"identity\", \"soul\", or \"style\"", + "type": "string", + "enum": [ + "identity", + "soul", + "style" + ] + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_set_avatar", + "description": "Generate the user's 3D avatar from a reference photo and persist it. Use when `identity_avatar_url` returned null. `image_url` MUST be a Pika-CDN URL — call `upload_asset` first if you have raw bytes. Returns the final avatar URL on success. Errors: 409 if avatar is already set, 422 if the image was rejected (no face / NSFW / decode), 423 if a generation is already in progress, 502 on transient infra issues.", + "inputSchema": { + "type": "object", + "properties": { + "image_url": { + "type": "string", + "format": "uri", + "description": "Pika-CDN URL of the user's reference photo (from upload_asset)." + } + }, + "required": [ + "image_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_set_voice", + "description": "Clone the user's voice from an audio sample and persist it. Use when `identity_voice_id` returned null. `audio_url` MUST be a Pika-CDN URL — call `upload_asset` first if needed. Provide a 10-30s clean voice recording. ElevenLabs is the default; choose `minimax` if ElevenLabs is unavailable. Returns JSON `{voice_id, platform}` on success. Errors: 409 if voice is already set, 422 if the audio was rejected (low quality / wrong format), 423 if a clone is already in progress, 502 on transient infra issues.", + "inputSchema": { + "type": "object", + "properties": { + "audio_url": { + "type": "string", + "format": "uri", + "description": "Pika-CDN URL of a 10-30s clean voice sample (from upload_asset)." + }, + "platform": { + "description": "Voice platform. Default: elevenlabs (better quality on short samples).", + "default": "elevenlabs", + "type": "string", + "enum": [ + "elevenlabs", + "minimax" + ] + } + }, + "required": [ + "audio_url" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_voice_id", + "description": "Return the user's registered voice ID (raw string). Pass the result as `voice_id` to generate_speech. Returns null when the user has no routable voice on file — call `identity_voice_info` if you also need the platform, or `identity_set_voice` to create one.", + "inputSchema": { + "type": "object", + "properties": {}, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_voice_info", + "description": "Return JSON `{voice_id, platform, sample_url}` for the user's registered voice. Use when you need to dispatch generate_speech against the correct provider (generate_speech defaults to minimax-tts, but ElevenLabs voice IDs must route to the elevenlabs adapter). `sample_url` is a short-lived download URL when the current voice is backed by a workspace fallback sample. Returns null fields when the user has no routable voice/sample on file.", + "inputSchema": { + "type": "object", + "properties": {}, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_voice_sample_url", + "description": "Return a short-lived download URL (raw string) for the workspace voice sample backing the user's current routable voice. Use as the `audio` param for transcription / preview flows. Returns \"null\" when no workspace fallback sample is on file (e.g. the voice was cloned via `identity_set_voice` from a Pika-CDN upload — that recording isn't stored in the workspace). Pair with `identity_voice_info` when you also need `{voice_id, platform}`.", + "inputSchema": { + "type": "object", + "properties": {}, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "identity_whoami", + "description": "Show the currently authenticated user.", + "inputSchema": { + "type": "object", + "properties": {}, + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "render_html_animation", + "description": "Render a HyperFrames-format HTML composition to MP4/WebM/MOV via the HyperFrames engine (deterministic per-frame Chrome BeginFrame seek + ffmpeg). Pure rendering primitive — no LLM step. Caller supplies the composition HTML (must include `<div id=\"stage\" data-width=\"W\" data-height=\"H\">` and clip children with `data-start`/`data-duration`/`data-track-index`); worker scaffolds a temp project and runs the CLI. Output dimensions come from the HTML's `data-width`/`data-height`; duration is derived from track timing — neither is a tool parameter.\n\nFor Claude-driven slide-animation generation, use `generate_slide_animation` instead. This tool is for callers that have already authored the composition (hand-written, ported from Remotion, or generated by a plugin-layer skill).\n\n`format=mov` and `format=webm` produce transparent overlays for compositing into other workflows.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "html": { + "type": "string", + "minLength": 1, + "maxLength": 5242880, + "description": "HyperFrames composition HTML. Must include `<div id=\"stage\" data-width=\"W\" data-height=\"H\">` with timed clip children. Asset references must be absolute URLs (no relative paths). Max 5 MB." + }, + "fps": { + "description": "Render frame rate (24, 30, or 60). Default 30.", + "anyOf": [ + { + "type": "number", + "const": 24 + }, + { + "type": "number", + "const": 30 + }, + { + "type": "number", + "const": 60 + } + ] + }, + "quality": { + "description": "Render quality preset. `draft` = fastest (lower bitrate), `standard` = default, `high` = max quality (slower). Default standard.", + "type": "string", + "enum": [ + "draft", + "standard", + "high" + ] + }, + "format": { + "description": "Output container. `mov` and `webm` produce transparent overlays for compositing into other workflows. Default mp4.", + "type": "string", + "enum": [ + "mp4", + "webm", + "mov" + ] + } + }, + "required": [ + "html" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "search_music", + "description": "Search a public music catalogue. Default: apple (Apple Music). Returns song metadata + 30s preview URLs.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "default": "apple", + "description": "Music catalogue provider. Default: `apple` (Apple Music). Returns song metadata + 30s preview URLs.", + "type": "string", + "enum": [ + "apple" + ] + }, + "query": { + "type": "string", + "minLength": 1, + "description": "Search query (e.g. 'radiohead creep' or 'lofi hip hop')." + }, + "limit": { + "default": 5, + "description": "Max number of results to return.", + "type": "integer", + "minimum": 1, + "maximum": 25 + } + }, + "required": [ + "query" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "search_skill", + "description": "Search for Pika skills by natural language query. Skills are reusable methodologies (e.g. 'make a singing video', 'create product showcase') that guide the agent through multi-step workflows.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Natural language query to search for skills" + }, + "top_k": { + "default": 5, + "description": "Maximum number of results to return (default 5)", + "type": "number" + } + }, + "required": [ + "query" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "task_cancel", + "description": "Cancel a running Pika task. Returns the terminal state; rejects if the task has already completed.", + "inputSchema": { + "type": "object", + "properties": { + "task_id": { + "type": "string" + } + }, + "required": [ + "task_id" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "task_status", + "description": "Poll the status of a long-running Pika task. Call this tool in a TIGHT LOOP (no Bash, no sleep) until status is `done`, `failed`, or `cancelled`. Each call blocks server-side up to ~20s waiting for a state change, so client-side sleeping is redundant and breaks the collapsed-tool-call view in the UI. When `done`, the `result` field holds the skill output (e.g. video URL).", + "inputSchema": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "task_id returned from any previous Pika tool invocation." + } + }, + "required": [ + "task_id" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "transcribe_audio", + "description": "Transcribe audio to text using OpenAI Whisper. Accepts audio URLs directly OR video URLs (audio track is extracted server-side via ffmpeg). Returns text + optional auto-detected language.\n\nReturns the result inline when complete (typically 30s–5min). Progress updates are sent as notifications during execution. If the server budget expires before completion, falls back to `{ task_id, status }` — then call `task_status(task_id)` in a tight loop (no Bash, no sleep) until done.", + "inputSchema": { + "type": "object", + "properties": { + "provider": { + "description": "Transcription provider. Currently only `whisper` is supported (default).", + "type": "string", + "enum": [ + "whisper" + ] + }, + "audio": { + "type": "string", + "format": "uri", + "description": "Audio or video URL (MP3, WAV, M4A, MP4, WebM, etc.)." + }, + "language": { + "description": "Optional BCP-47 language hint. Auto-detects if omitted.", + "type": "string" + }, + "timestamps": { + "description": "If true, return per-segment timestamps (start/end seconds + text) alongside the transcript. Raises an error if Whisper returns no segments. Useful for locating sentence boundaries within a longer audio file.", + "type": "boolean" + } + }, + "required": [ + "audio" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + }, + { + "name": "upload_asset", + "description": "Get a presigned URL for uploading a file (image, audio, video). Returns { presigned_url, public_url, expires_at }.", + "inputSchema": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "minLength": 1, + "description": "Original filename." + }, + "mime_type": { + "type": "string", + "minLength": 1, + "description": "MIME type (e.g. 'image/png', 'audio/mpeg')." + }, + "size_bytes": { + "type": "integer", + "minimum": 1, + "maximum": 524288000, + "description": "File size in bytes. Max 500 MB." + } + }, + "required": [ + "filename", + "mime_type", + "size_bytes" + ], + "$schema": "http://json-schema.org/draft-07/schema#" + } + } +]