diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index 0ab1a0b..0ea040d 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -223,16 +223,6 @@ "authentication": "ON_INSTALL" }, "category": "Cloud Platform" - }, - { - "name": "gemini-api", - "source": "./plugins/cloud/gemini-api", - "description": "Gemini API guidance for coding agents: generating text and multimodal output with the Google Gen AI SDK, the stateful Interactions API for multi-turn conversations and background execution, and the Live API for realtime bidirectional streaming. Includes the Google Developer Knowledge MCP server for grounded, up-to-date documentation.", - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL" - }, - "category": "AI & Machine Learning" } ] } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index b9d7882..8c00697 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -157,11 +157,6 @@ "name": "google-cloud-developer", "source": "./plugins/cloud/google-cloud-developer", "description": "Core Google Cloud guidance for coding agents: first-project onboarding, authentication and credential selection, and safety-critical validation for gcloud CLI commands. Includes the Google Developer Knowledge MCP server for grounded, up-to-date Google Cloud documentation." - }, - { - "name": "gemini-api", - "source": "./plugins/cloud/gemini-api", - "description": "Gemini API guidance for coding agents: generating text and multimodal output with the Google Gen AI SDK, the stateful Interactions API for multi-turn conversations and background execution, and the Live API for realtime bidirectional streaming. Includes the Google Developer Knowledge MCP server for grounded, up-to-date documentation." } ] } diff --git a/plugins/cloud/gemini-api/.claude-plugin/plugin.json b/plugins/cloud/gemini-api/.claude-plugin/plugin.json deleted file mode 100644 index 35cf523..0000000 --- a/plugins/cloud/gemini-api/.claude-plugin/plugin.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "gemini-api", - "version": "1.0.0", - "description": "Gemini API guidance for coding agents: generation, the Interactions API, and the Live API.", - "author": { - "name": "Google LLC" - } -} diff --git a/plugins/cloud/gemini-api/.codex-plugin/plugin.json b/plugins/cloud/gemini-api/.codex-plugin/plugin.json deleted file mode 100644 index 296244b..0000000 --- a/plugins/cloud/gemini-api/.codex-plugin/plugin.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "gemini-api", - "version": "1.0.0", - "description": "Gemini API guidance for coding agents: generation, the Interactions API, and the Live API.", - "skills": "./skills/", - "interface": { - "displayName": "Gemini API", - "shortDescription": "Build with the Gemini API, Interactions API, and Live API", - "category": "Productivity" - } -} diff --git a/plugins/cloud/gemini-api/.mcp.json b/plugins/cloud/gemini-api/.mcp.json deleted file mode 100644 index e428406..0000000 --- a/plugins/cloud/gemini-api/.mcp.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", - "mcpServers": { - "developer-knowledge": { - "type": "streamable-http", - "url": "https://developerknowledge.googleapis.com/mcp" - } - } -} diff --git a/plugins/cloud/gemini-api/gemini-extension.json b/plugins/cloud/gemini-api/gemini-extension.json deleted file mode 100644 index 6898c13..0000000 --- a/plugins/cloud/gemini-api/gemini-extension.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "gemini-api", - "version": "1.0.0", - "description": "Gemini API guidance for coding agents: generation, the Interactions API, and the Live API.", - "mcpServers": { - "developer-knowledge": { - "httpUrl": "https://developerknowledge.googleapis.com/mcp", - "authProviderType": "google_credentials" - } - } -} diff --git a/plugins/cloud/gemini-api/mcp.json b/plugins/cloud/gemini-api/mcp.json deleted file mode 100644 index e428406..0000000 --- a/plugins/cloud/gemini-api/mcp.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", - "mcpServers": { - "developer-knowledge": { - "type": "streamable-http", - "url": "https://developerknowledge.googleapis.com/mcp" - } - } -} diff --git a/plugins/cloud/gemini-api/mcp_config.json b/plugins/cloud/gemini-api/mcp_config.json deleted file mode 100644 index 6350e1e..0000000 --- a/plugins/cloud/gemini-api/mcp_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "mcpServers": { - "developer-knowledge": { - "serverUrl": "https://developerknowledge.googleapis.com/mcp" - } - } -} diff --git a/plugins/cloud/gemini-api/plugin.json b/plugins/cloud/gemini-api/plugin.json deleted file mode 100644 index c206afb..0000000 --- a/plugins/cloud/gemini-api/plugin.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", - "name": "gemini-api", - "version": "1.0.0", - "description": "Gemini API guidance for coding agents: generating text and multimodal output with the Google Gen AI SDK, the stateful Interactions API for multi-turn conversations and background execution, and the Live API for realtime bidirectional streaming. Includes the Google Developer Knowledge MCP server for grounded, up-to-date documentation.", - "author": { - "name": "Google LLC", - "url": "https://cloud.google.com" - }, - "homepage": "https://github.com/google/skills/tree/main/plugins/cloud/gemini-api", - "repository": "https://github.com/google/skills", - "license": "Apache-2.0", - "keywords": ["gemini", "gemini-api", "generative-ai", "live-api"] -} diff --git a/plugins/cloud/gemini-api/skills/gemini-api/SKILL.md b/plugins/cloud/gemini-api/skills/gemini-api/SKILL.md deleted file mode 100644 index 78b3f5c..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/SKILL.md +++ /dev/null @@ -1,251 +0,0 @@ ---- -name: gemini-api -metadata: - category: AiAndMachineLearning -description: Use when the user asks about using Gemini in an enterprise environment or explicitly mentions Vertex AI, Google Cloud, or Agent Platform. Guides the usage of the Gemini API on Agent Platform with the Google Gen AI SDK. Covers SDK usage (Python, JS/TS, Go, Java, C#), capabilities like multimodal inputs, tools, media generation, caching, batch prediction, and Live API. -compatibility: Requires active Google Cloud credentials and Agent Platform API enabled. ---- - -IMPORTANT: Agent Platform (full name Gemini Enterprise Agent Platform) was previously named "Vertex AI" and many web resources use the legacy branding. - -# Gemini API in Agent Platform - -Access Google's most advanced AI models built for enterprise use cases using the Gemini API in Agent Platform. - -Provide these key capabilities: - -- **Text generation** - Chat, completion, summarization -- **Multimodal understanding** - Process images, audio, video, and documents -- **Function calling** - Let the model invoke your functions -- **Structured output** - Generate valid JSON matching your schema -- **Context caching** - Cache large contexts for efficiency -- **Embeddings** - Generate text embeddings for semantic search -- **Live Realtime API** - Bidirectional streaming for low latency Voice and Video interactions -- **Batch Prediction** - Handle massive async dataset prediction workloads - -## Core Directives - -- **Unified SDK**: ALWAYS use the Gen AI SDK (`google-genai` for Python, `@google/genai` for JS/TS, `google.golang.org/genai` for Go, `com.google.genai:google-genai` for Java, `Google.GenAI` for C#). -- **Legacy SDKs**: DO NOT use `google-cloud-aiplatform`, `@google-cloud/vertexai`, or `google-generativeai`. - -## SDKs - -- **Python**: Install `google-genai` with `pip install google-genai` -- **JavaScript/TypeScript**: Install `@google/genai` with `npm install @google/genai` -- **Go**: Install `google.golang.org/genai` with `go get google.golang.org/genai` -- **C#/.NET**: Install `Google.GenAI` with `dotnet add package Google.GenAI` -- **Java**: - - groupId: `com.google.genai`, artifactId: `google-genai` - - Latest version can be found here: https://central.sonatype.com/artifact/com.google.genai/google-genai/versions (let's call it `LAST_VERSION`) - - Install in `build.gradle`: - - ``` - implementation("com.google.genai:google-genai:${LAST_VERSION}") - ``` - - - Install Maven dependency in `pom.xml`: - - ```xml - - com.google.genai - google-genai - ${LAST_VERSION} - - ``` - -> [!WARNING] -> Legacy SDKs like `google-cloud-aiplatform`, `@google-cloud/vertexai`, and `google-generativeai` are deprecated. Migrate to the new SDKs above urgently by following the [Migration Guide](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/deprecations/genai-vertexai-sdk.md.txt). - -## Authentication & Configuration - -Prefer environment variables over hard-coding parameters when creating the client. Initialize the client without parameters to automatically pick up these values. - -### Application Default Credentials (ADC) -Set these variables for standard [Google Cloud authentication](https://docs.cloud.google.com/vertex-ai/generative-ai/docs/start/gcp-auth.md.txt): - -```bash -export GOOGLE_CLOUD_PROJECT='your-project-id' -export GOOGLE_CLOUD_LOCATION='global' -export GOOGLE_GENAI_USE_ENTERPRISE=true -``` - -- By default, use `location="global"` to access the global endpoint, which provides automatic routing to regions with available capacity. -- If a user explicitly asks to use a specific region (e.g., `us-central1`, `europe-west4`), specify that region in the `GOOGLE_CLOUD_LOCATION` parameter instead. Reference the [supported regions documentation](https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations.md.txt) if needed. - -### Agent Platform in Express Mode -Set these variables when using [Express Mode](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/start/api-keys.md.txt) with an API key: - -```bash -export GOOGLE_API_KEY='your-api-key' -export GOOGLE_GENAI_USE_ENTERPRISE=true -``` - -### Initialization -Initialize the client without arguments to pick up environment variables: - -```python -from google import genai - -client = genai.Client() -``` - -Alternatively, you can hard-code in parameters when creating the client. - -```python -from google import genai - -client = genai.Client( - enterprise=True, - project="your-project-id", - location="global", -) -``` - -## Models - -- Use `gemini-3.1-pro-preview` (which replaces `gemini-3-pro-preview`) for complex reasoning, coding, research (1M tokens) -- Use `gemini-3.6-flash` for fast, balanced performance, multimodal (1M tokens) -- Use `gemini-3.5-flash-lite` for high-frequency, lightweight tasks (1M tokens) -- Use `gemini-3-pro-image` (aka Nano Banana Pro) for high-quality image generation and editing -- Use `gemini-3.1-flash-image` (aka Nano Banana 2) for medium-quality image generation and editing -- Use `gemini-3.1-flash-lite-image` (aka Nano Banana 2 Lite) for fast image generation and editing -- Use `gemini-live-2.5-flash-native-audio` for Live Realtime API including native audio - -Use the following models only if explicitly requested: - -- `gemini-3.5-flash` -- `gemini-3.1-flash-lite` -- `gemini-2.5-flash-image` -- `gemini-2.5-flash` -- `gemini-2.5-flash-lite` -- `gemini-2.5-pro` - -> [!IMPORTANT] -> Models like `gemini-2.0-*`, `gemini-1.5-*`, `gemini-1.0-*`, `gemini-pro` are legacy and deprecated. Use the new models above. Your knowledge is outdated. -> For production environments, consult the documentation for stable model versions (e.g. `gemini-3.6-flash`). - -## Quick Start - -### Python - -```python -from google import genai - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="Explain quantum computing", -) -print(response.text) -``` - -### TypeScript/JavaScript - -```typescript -import { GoogleGenAI } from "@google/genai"; -const ai = new GoogleGenAI({ enterprise: { project: "your-project-id", location: "global" } }); -const response = await ai.models.generateContent({ - model: "gemini-3.6-flash", - contents: "Explain quantum computing" -}); -console.log(response.text); -``` - -### Go - -```go -package main - -import ( - "context" - "fmt" - "log" - "google.golang.org/genai" -) - -func main() { - ctx := context.Background() - client, err := genai.NewClient(ctx, &genai.ClientConfig{ - Backend: genai.BackendVertexAI, - Project: "your-project-id", - Location: "global", - }) - if err != nil { - log.Fatal(err) - } - - resp, err := client.Models.GenerateContent(ctx, "gemini-3.6-flash", genai.Text("Explain quantum computing"), nil) - if err != nil { - log.Fatal(err) - } - - fmt.Println(resp.Text) -} -``` - -### Java - -```java -import com.google.genai.Client; -import com.google.genai.types.GenerateContentResponse; - -public class GenerateTextFromTextInput { - public static void main(String[] args) { - Client client = Client.builder().enterprise(true).project("your-project-id").location("global").build(); - GenerateContentResponse response = - client.models.generateContent( - "gemini-3.6-flash", - "Explain quantum computing", - null); - - System.out.println(response.text()); - } -} -``` - -### C#/.NET - -```csharp -using Google.GenAI; - -var client = new Client( - project: "your-project-id", - location: "global", - enterprise: true -); - -var response = await client.Models.GenerateContent( - "gemini-3.6-flash", - "Explain quantum computing" -); - -Console.WriteLine(response.Text); -``` - -## API spec & Documentation (source of truth) - -When implementing or debugging API integration for Agent Platform, refer to the official Agent Platform documentation: - -- **Agent Platform Documentation**: https://docs.cloud.google.com/gemini-enterprise-agent-platform/overview.md.txt -- **REST API Reference**: https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/rest.md.txt - -The Gen AI SDK on Agent Platform uses the `v1beta1` or `v1` REST API endpoints (e.g., `https://{LOCATION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT}/locations/{LOCATION}/publishers/google/models/{MODEL}:generateContent`). - -> [!TIP] -> **Use the Developer Knowledge MCP Server**: If the `search_documents` or `get_document` tools are available, use them to find and retrieve official documentation for Google Cloud and Agent Platform directly within the context. This is the preferred method for getting up-to-date API details and code snippets. - -## Workflows and Code Samples - -Reference the [Python Docs Samples repository](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/genai) for additional code samples and specific usage scenarios. - -Depending on the specific user request, refer to the following reference files for detailed code samples and usage patterns (Python examples): - -- **Text & Multimodal**: Chat, Multimodal inputs (Image, Video, Audio), and Streaming. See [references/text_and_multimodal.md](references/text_and_multimodal.md) -- **Embeddings**: Generate text embeddings for semantic search. See [references/embeddings.md](references/embeddings.md) -- **Structured Output & Tools**: JSON generation, Function Calling, Search Grounding, and Code Execution. See [references/structured_and_tools.md](references/structured_and_tools.md) -- **Media Generation**: Image generation, Image editing, and Video generation. See [references/media_generation.md](references/media_generation.md) -- **Bounding Box Detection**: Object detection and localization within images and video. See [references/bounding_box.md](references/bounding_box.md) -- **Live API**: Real-time bidirectional streaming for voice, vision, and text. See [references/live_api.md](references/live_api.md) -- **Advanced Features**: Content Caching, Batch Prediction, and Thinking/Reasoning. See [references/advanced_features.md](references/advanced_features.md) -- **Safety**: Adjusting Responsible AI filters and thresholds. See [references/safety.md](references/safety.md) -- **Model Tuning**: Supervised Fine-Tuning and Preference Tuning. See [references/model_tuning.md](references/model_tuning.md) diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/advanced_features.md b/plugins/cloud/gemini-api/skills/gemini-api/references/advanced_features.md deleted file mode 100644 index 6fd7e85..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/advanced_features.md +++ /dev/null @@ -1,149 +0,0 @@ -# Advanced Features - -## Content Caching -Cache large documents or contexts to reduce cost and latency. - -Only use explicit caching if asked directly. Implicit caching is enabled by default and automatically provides cost savings when cache hits occur. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -content_cache = client.caches.create( - model="gemini-3.6-flash", - config=types.CreateCachedContentConfig( - contents=[ - types.Content( - role="user", - parts=[ - types.Part.from_uri( - file_uri="gs://your-bucket/large.pdf", - mime_type="application/pdf", - ) - ], - ) - ], - system_instruction="You are an expert researcher.", - display_name="example-cache", - ttl="86400s", - ), -) - -# Use the cache -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="Summarize the pdf", - config=types.GenerateContentConfig(cached_content=content_cache.name), -) -``` - -## Batch Prediction -For processing large datasets asynchronously. - -```python -import time -from google import genai -from google.genai import types - -client = genai.Client() - -job = client.batches.create( - model="gemini-3.6-flash", - src="gs://your-bucket/prompts.jsonl", - config=types.CreateBatchJobConfig(dest="gs://your-bucket/outputs"), -) - -completed_states = { - types.JobState.JOB_STATE_SUCCEEDED, - types.JobState.JOB_STATE_FAILED, - types.JobState.JOB_STATE_CANCELLED, -} -while job.state not in completed_states: - time.sleep(30) - job = client.batches.get(name=job.name) -``` - -### Thinking (Reasoning) - -Thinking is on by default for `gemini-3.1-pro-preview` (default `HIGH` / dynamic) and `gemini-3.6-flash` (default `MEDIUM`). `gemini-3.5-flash-lite` defaults to `MINIMAL`. -It can be adjusted by using the `thinking_level` parameter. - -- **`MINIMAL`:** Constrains the model to use as few tokens as possible for thinking and is best used for low-complexity tasks that wouldn't benefit from extensive reasoning. (Not supported for `gemini-3.1-pro-preview`) -- **`LOW`**: Constrains the model to use fewer tokens for thinking and is suitable for simpler tasks where extensive reasoning is not required. -- **`MEDIUM`**: Offers a balanced approach suitable for tasks of moderate complexity that benefit from reasoning but don't require deep, multi-step planning. -- **`HIGH`**: Maximizes reasoning depth. The model may take significantly longer to reach a first token, but the output will be more thoroughly vetted. - -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.1-pro-preview", - contents="solve x^2 + 4x + 4 = 0", - config=types.GenerateContentConfig( - thinking_config=types.ThinkingConfig( - thinking_level=types.ThinkingLevel.HIGH, - ) - ), -) - -# Access thoughts if returned -for part in response.candidates[0].content.parts: - if part.thought: - print(f"Thought: {part.text}") - else: - print(f"Final Answer: {part.text}") -``` - -## Model Context Protocol (MCP) support (experimental) - -Built-in [MCP](https://modelcontextprotocol.io/introduction) support is an experimental feature. You can pass a local MCP server as a tool directly. - -```python -import os -import asyncio -from datetime import datetime -from mcp import ClientSession, StdioServerParameters -from mcp.client.stdio import stdio_client - -from google import genai -from google.genai import types - -client = genai.Client() - -# Create server parameters for stdio connection -server_params = StdioServerParameters( - command="npx", # Executable - args=["-y", "@philschmid/weather-mcp"], # MCP Server - env=None, # Optional environment variables -) - - -async def run(): - async with stdio_client(server_params) as (read, write): - async with ClientSession(read, write) as session: - # Prompt to get the weather for the current day in London. - prompt = f"What is the weather in London in {datetime.now().strftime('%Y-%m-%d')}?" - - # Initialize the connection between client and server - await session.initialize() - - # Send request to the model with MCP function declarations - response = await client.aio.models.generate_content( - model="gemini-3.6-flash", - contents=prompt, - config=types.GenerateContentConfig( - tools=[ - session # uses the session, will automatically call the tool using automatic function calling - ], - ), - ) - print(response.text) - - -# Start the asyncio event loop and run the main function -asyncio.run(run()) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/bounding_box.md b/plugins/cloud/gemini-api/skills/gemini-api/references/bounding_box.md deleted file mode 100644 index d56d769..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/bounding_box.md +++ /dev/null @@ -1,68 +0,0 @@ -# Bounding Box Detection - -Detect and localize objects within images or videos using bounding boxes. The model returns coordinates in the format `[y_min, x_min, y_max, x_max]`, normalized from 0 to 1000. - -## Implementation (Python) - -To ensure structured output, define a `BoundingBox` class and provide it as the `response_schema`. - -```python -from google import genai -from google.genai.types import ( - GenerateContentConfig, - Part, -) -from pydantic import BaseModel - - -# Define the schema for the bounding box -class BoundingBox(BaseModel): - box_2d: list[int] - label: str - - -client = genai.Client() - -config = GenerateContentConfig( - system_instruction=""" - Return bounding boxes as an array with labels. - Never return masks. Limit to 25 objects. - """, - response_mime_type="application/json", - response_schema=list[BoundingBox], -) - -image_uri = "gs://cloud-samples-data/generative-ai/image/socks.jpg" - -response = client.models.generate_content( - model="gemini-3.6-flash", - contents=[ - Part.from_uri(file_uri=image_uri, mime_type="image/jpeg"), - "Detect the socks in the image and provide bounding boxes.", - ], - config=config, -) - -# Access the detected boxes -for bbox in response.parsed: - print(f"Label: {bbox.label}, Box: {bbox.box_2d}") -``` - -## Coordinate System -- **Format**: `[y_min, x_min, y_max, x_max]` -- **Normalization**: Coordinates are integers from `0` to `1000`. -- **Origin**: `[0, 0]` is the top-left corner of the image. - -## Visualization Helper -To visualize the results, scale the normalized coordinates back to the original image dimensions. - -```python -def scale_box(box_2d, width, height): - y_min, x_min, y_max, x_max = box_2d - return [ - int(y_min / 1000 * height), - int(x_min / 1000 * width), - int(y_max / 1000 * height), - int(x_max / 1000 * width), - ] -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/embeddings.md b/plugins/cloud/gemini-api/skills/gemini-api/references/embeddings.md deleted file mode 100644 index 0f9cd64..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/embeddings.md +++ /dev/null @@ -1,77 +0,0 @@ -# Text and Multimodal Embeddings - -Generate embeddings for text or multimodal content (images and videos) to perform semantic search, clustering, and other NLP tasks. Text and multimodal embedding vectors share the same semantic space, which allows you to use them interchangeably for cross-modal applications like searching for an image using a text query, or searching for a video using an image. - -## Basic Usage (Text) - -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.embed_content( - model="gemini-embedding-2", - contents=[ - "How do I get a driver's license/learner's permit?", - "How long is my driver's license valid for?", - ], - # Optional Parameters - config=types.EmbedContentConfig(task_type="RETRIEVAL_DOCUMENT", output_dimensionality=768), -) -print(response.embeddings) -``` - -## Multimodal Embeddings (Image and Video) - -To generate embeddings for images and videos, use the `types.Part.from_uri` method to point the model to a Google Cloud Storage (GCS) URI containing the media file, and provide the appropriate MIME type. - -### Image Embeddings - -For images, provide the GCS URI of the image and set the MIME type (e.g., `image/jpeg`). - -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.embed_content( - model="gemini-embedding-2", - contents=types.Part.from_uri( - file_uri="gs://github-repo/embeddings/getting_started_embeddings/gms_images/GGOEACBA104999.jpg", - mime_type="image/jpeg" - ), - config=types.EmbedContentConfig(output_dimensionality=768), -) - -image_embedding = response.embeddings[0].values -print(f"Length of image embedding: {len(image_embedding)}") -``` - -### Video Embeddings - -Generating embeddings for a video works similarly. However, instead of a single vector, the API returns a list of embedding vectors—one representing each frame segment or interval of the processed video. - -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.embed_content( - model="gemini-embedding-2", - contents=types.Part.from_uri( - file_uri="gs://github-repo/embeddings/getting_started_embeddings/UCF-101-subset/BrushingTeeth/v_BrushingTeeth_g01_c02.mp4", - mime_type="video/mp4" - ), - config=types.EmbedContentConfig(output_dimensionality=768), -) - -# Extract embedding values for each video segment -video_embeddings =[emb.values for emb in response.embeddings] - -print(f"Number of video segment embeddings returned: {len(video_embeddings)}") -print(f"First segment embedding length: {len(video_embeddings[0])}") -``` - -### Cross-Modal Search - -Because these vectors share a semantic space, you can calculate the dot product or cosine similarity between different types of embeddings. For example, you can calculate the similarity between a text query embedding ("A music concert") and a pre-computed database of image or video embeddings to build a robust multimodal semantic search engine. diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/live_api.md b/plugins/cloud/gemini-api/skills/gemini-api/references/live_api.md deleted file mode 100644 index c60a896..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/live_api.md +++ /dev/null @@ -1,36 +0,0 @@ -# Live API - -The Live API provides real-time, low-latency bidirectional streaming via WebSockets. It is ideal for interactive voice and video applications. - -```python -import asyncio -from google import genai -from google.genai import types - -async def generate_content(): - client = genai.Client() - model_id = "gemini-live-2.5-flash-native-audio" - - config = types.LiveConnectConfig( - response_modalities=[types.LiveModality.TEXT], # Change to AUDIO for voice responses - ) - - async with client.aio.live.connect(model=model_id, config=config) as session: - text_input = "Hello? Gemini, are you there?" - await session.send_client_content( - turns=types.Content(role="user", parts=[types.Part.from_text(text=text_input)]) - ) - - async for message in session.receive(): - if message.text: - print(message.text, end="") - -asyncio.run(generate_content()) -``` - -For sending audio: -```python -await session.send_realtime_input( - media=Blob(data=audio_bytes, mime_type="audio/pcm;rate=16000") -) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/media_generation.md b/plugins/cloud/gemini-api/skills/gemini-api/references/media_generation.md deleted file mode 100644 index 74128aa..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/media_generation.md +++ /dev/null @@ -1,275 +0,0 @@ -# Media Generation - -## Image Generation -Generate images using `gemini-3.1-flash-image`. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -response = client.models.generate_content( - model="gemini-3.1-flash-image", - contents="A dog reading a newspaper", -) - -for part in response.parts: - if part.text is not None: - print(part.text) - elif part.inline_data is not None: - image = part.as_image() - image.save("generated_image.png") -``` - -For high-resolution images, use `gemini-3-pro-image`. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -response = client.models.generate_content( - model="gemini-3-pro-image", - contents="A dog reading a newspaper", - config=types.GenerateContentConfig( - image_config=types.ImageConfig(aspect_ratio="16:9", image_size="2K") - ), -) - -for part in response.parts: - if part.text is not None: - print(part.text) - elif part.inline_data is not None: - image = part.as_image() - image.save("generated_image.png") -``` - -## Image Editing -It is recommended to use chat mode for editing images. - -```python -from google import genai -from PIL import Image - -client = genai.Client() - -prompt = "A small white ceramic bowl with lemons and limes" -image = Image.open("fruit.png") - -# Create the chat -chat = client.chats.create(model="gemini-3.1-flash-image") - -# Send the image and ask for it to be edited -response = chat.send_message([prompt, image]) - -# Get the text and the image generated -for i, part in enumerate(response.candidates[0].content.parts): - if part.text is not None: - print(part.text) - elif part.inline_data is not None: - image = part.as_image() - image.save(f"generated_image_{i}.png") - -# Continue iterating -chat.send_message("Make the bowl blue") -``` - -## Video Generation with Gemini Omni - -`gemini-omni-flash-preview` (Preview) generates and edits video (with synchronized audio) at 720p from text, images, and reference media. Unlike Veo, Omni uses the **Interactions API** (`client.interactions.create`), not `generate_content`/`generate_videos`. - -Key configuration: - -- **Task** (`interactions.VideoConfig.task`): `text_to_video`, `image_to_video`, `reference_to_video`, or `edit`. Set this to match your inputs and desired behavior. -- **Response format** (`interactions.VideoResponseFormat`): `aspect_ratio` (`16:9` or `9:16`), `duration` (`3s`-`10s`), and `delivery`. These can also be specified directly in the text prompt. -- **Delivery**: To save output to Cloud Storage, set `delivery="uri"` and `gcs_uri="gs://"`. Otherwise, video bytes are returned inline (base64). - -Notes on output: audio is generated alongside the video, resolution is 720p, and every video includes [C2PA metadata](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/content-credentials) and a [SynthID watermark](https://deepmind.google/technologies/synthid/). - -Responses are returned as `interaction.steps`; collect the `content` from each `model_output` step. This helper extracts and saves the video bytes: - -```python -import base64 -from google import genai -from google.genai import interactions - -client = genai.Client() - -omni_model = "gemini-omni-flash-preview" - - -def save_video(interaction, path="output.mp4"): - contents = [] - for step in interaction.steps: - if step.type == "model_output": - contents.extend(step.content) - with open(path, "wb") as f: - f.write(base64.b64decode(contents[0].data)) -``` - -### Text-to-video - -```python -prompt = "A hard-shell suitcase rolling down a city street at sunset, cinematic tracking shot." - -interaction = client.interactions.create( - model=omni_model, - input=prompt, - generation_config=interactions.GenerationConfig( - video_config=interactions.VideoConfig(task="text_to_video") - ), - response_format=interactions.VideoResponseFormat( - aspect_ratio="16:9", - duration="9s", - # delivery="uri", - # gcs_uri="gs://", - ), -) - -save_video(interaction) -``` - -### Image-to-video - -Provide a starting image as the literal first frame. Pass images inline as base64 or by Cloud Storage `uri`. - -```python -prompt = "The suitcase stands up, unzips, and colorful travel stickers pop out around it." - -with open("suitcase.png", "rb") as f: - img_b64 = base64.b64encode(f.read()).decode("utf-8") - -interaction = client.interactions.create( - model=omni_model, - input=[ - {"type": "text", "text": prompt}, - {"type": "image", "mime_type": "image/png", "data": img_b64}, - ], - generation_config=interactions.GenerationConfig( - video_config=interactions.VideoConfig(task="image_to_video") - ), -) - -save_video(interaction) -``` - -### Reference-to-video - -Supply reference images (e.g., a character and a product) to guide generation. Reference media acts as a style/subject guide rather than the literal first frame. Video and audio reference inputs are not currently supported. - -```python -prompt = "A woman walks up to the arcade game and starts playing. 9:16 aspect ratio. 7 second video." - -images_input = [] -for img_path in ["woman.jpeg", "arcade-game.png"]: - with open(img_path, "rb") as f: - img_b64 = base64.b64encode(f.read()).decode("utf-8") - images_input.append({"type": "image", "mime_type": "image/jpeg", "data": img_b64}) - -interaction = client.interactions.create( - model=omni_model, - input=[{"type": "text", "text": prompt}, *images_input], - generation_config=interactions.GenerationConfig( - video_config=interactions.VideoConfig(task="reference_to_video") - ), -) - -save_video(interaction) -``` - -### Video editing - -Add, remove, or alter objects in a source video, or restyle it. The source video must be under 10 seconds; you can also supply reference images to guide the edit. - -```python -prompt = "Change the dog to a cat, remove the backpack, and add a propeller hat." - -interaction = client.interactions.create( - model=omni_model, - input=[ - {"type": "text", "text": prompt}, - {"type": "image", "mime_type": "image/png", "uri": "gs://cloud-samples-data/generative-ai/image/chair-cat.png"}, - {"type": "video", "mime_type": "video/mp4", "uri": "gs://cloud-samples-data/generative-ai/video/dog_day1.mp4"}, - ], - generation_config=interactions.GenerationConfig( - video_config=interactions.VideoConfig(task="edit") - ), -) - -save_video(interaction) -``` - -### Async generation - -Set `background=True` to generate a video you can check on later, then poll with `client.interactions.get`. - -```python -import time - -initial = client.interactions.create(model=omni_model, input=prompt, background=True) - -interaction = initial -while interaction.status not in ["completed", "failed"]: - time.sleep(10) - interaction = client.interactions.get(id=initial.id) - -if interaction.status == "completed": - save_video(interaction) -``` - -### Multi-turn editing (chat) - -Iteratively refine a video by passing the previous `interaction.steps` back into `input`, followed by the new user turn. - -```python -interaction1 = client.interactions.create( - model=omni_model, - input="A claymation ball character rolling and then being stopped by a wall, stop motion.", -) - -turn2_input = interaction1.steps + [ - {"type": "user_input", "content": [{"type": "text", "text": "Now make the same video in a doodle style."}]} -] - -interaction2 = client.interactions.create(model=omni_model, input=turn2_input) - -save_video(interaction2) -``` - -## Video Generation with Veo -Generate video using the Veo model. Usage of Veo can be costly, so check pricing for Veo. Start with the fast model (`veo-3.1-fast-generate-001`) since the result quality is usually sufficient, and swap to the larger model if needed. - -```python -import time -from google import genai -from google.genai import types -from PIL import Image - -client = genai.Client() - -image = Image.open("image.png") # Optional initial image - -# Video generation is an async operation -operation = client.models.generate_videos( - model="veo-3.1-fast-generate-001", - prompt="a cat reading a book", - image=image, - config=types.GenerateVideosConfig( - person_generation="dont_allow", - aspect_ratio="16:9", - number_of_videos=1, - duration_seconds=5, - output_gcs_uri="gs://your-bucket/your-prefix", - ), -) - -# Poll for completion -while not operation.done: - time.sleep(20) - operation = client.operations.get(operation) - -if operation.response: - print(operation.result.generated_videos[0].video.uri) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/model_tuning.md b/plugins/cloud/gemini-api/skills/gemini-api/references/model_tuning.md deleted file mode 100644 index e0e80ed..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/model_tuning.md +++ /dev/null @@ -1,39 +0,0 @@ -# Model Tuning - -Supervised Fine-Tuning using your own datasets. - -Note: Not all models support tuning. Refer to https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/tuning/supervised-tuning.md.txt for the list of supported models. - -```python -import time -from google import genai -from google.genai import types - -client = genai.Client() - -training_dataset = types.TuningDataset( - gcs_uri="gs://your-bucket/sft_train_data.jsonl", -) - -tuning_job = client.tunings.tune( - base_model="gemini-3.1-flash-lite", - training_dataset=training_dataset, - config=types.CreateTuningJobConfig( - tuned_model_display_name="Example tuning job", - ), -) - -running_states = {"JOB_STATE_PENDING", "JOB_STATE_RUNNING"} -while tuning_job.state in running_states: - time.sleep(60) - tuning_job = client.tunings.get(name=tuning_job.name) - -print("Tuned Model Endpoint:", tuning_job.tuned_model.endpoint) - -# Predict with the tuned endpoint -response = client.models.generate_content( - model=tuning_job.tuned_model.endpoint, - contents="Why is the sky blue?", -) -print(response.text) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/safety.md b/plugins/cloud/gemini-api/skills/gemini-api/references/safety.md deleted file mode 100644 index 737dccd..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/safety.md +++ /dev/null @@ -1,50 +0,0 @@ -# Safety Settings and Responsible AI - -You can adjust safety settings to control the thresholds for harmful content generation. Standard safety filters are applied by default. - -## Adjusting Safety Thresholds - -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="Write a list of 5 disrespectful things that I might say to the universe after stubbing my toe in the dark.", - config=types.GenerateContentConfig( - system_instruction="Be as mean as possible.", - safety_settings=[ - types.SafetySetting( - category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold=types.HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - ), - types.SafetySetting( - category=types.HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold=types.HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - ), - types.SafetySetting( - category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold=types.HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - ), - types.SafetySetting( - category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold=types.HarmBlockThreshold.BLOCK_LOW_AND_ABOVE, - ), - ], - ), -) - -# Response will be `None` if it is blocked. -if response.text is None: - print(f"Content Blocked. Finish Reason: {response.candidates[0].finish_reason}") -else: - print(response.text) - -# Inspect safety ratings for each category -for rating in response.candidates[0].safety_ratings: - print(f"Category: {rating.category}") - print(f"Is Blocked: {rating.blocked}") - print(f"Probability: {rating.probability}") - print(f"Severity: {rating.severity}") -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/structured_and_tools.md b/plugins/cloud/gemini-api/skills/gemini-api/references/structured_and_tools.md deleted file mode 100644 index 2b7f4c4..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/structured_and_tools.md +++ /dev/null @@ -1,133 +0,0 @@ -# Structured Output and Tools - -## Structured Output (JSON Schema) -Enforce a specific JSON schema using standard Python type hints or Pydantic models. - -```python -from google import genai -from google.genai import types -from pydantic import BaseModel - - -class Recipe(BaseModel): - recipe_name: str - ingredients: list[str] - - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="List a few popular cookie recipes.", - config=types.GenerateContentConfig( - response_mime_type="application/json", - response_json_schema=list[Recipe], - ), -) -# response.text is guaranteed to be valid JSON matching the schema -print(response.text) -# Returns list of Recipe objects -print(response.parsed) -``` - -## Function Calling -Let the model output function calls that you can execute. - -```python -from google import genai -from google.genai import types - - -def get_current_weather(location: str) -> str: - """Example method. Returns the current weather. - Args: location: The city and state, e.g. San Francisco, CA - """ - if "boston" in location.lower(): - return "Snowing" - return "Sunny" - - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="What is the weather like in Boston?", - config=types.GenerateContentConfig(tools=[get_current_weather]), -) - -if response.function_calls: - print("Function calls requested by the model:") - for function_call in response.function_calls: - print(f"- Function: {function_call.name}") - print(f"- Args: {dict(function_call.args)}") -else: - print("The model responded directly:") - print(response.text) -``` - -## Search Grounding -Ground the model's responses in Google Search or your own data with Agent Search (formerly known as Vertex AI Search). - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="When is the next total solar eclipse in the US?", - config=types.GenerateContentConfig( - tools=[types.Tool(google_search=types.GoogleSearch())], - ), -) -print(response.text) -# Search details -print(f"Search Query: {response.candidates[0].grounding_metadata.web_search_queries}") -# Inspect grounding metadata -print(response.candidates[0].grounding_metadata.search_entry_point.rendered_content) -# Urls used for grounding -print( - f"Search Pages: {', '.join([site.web.title for site in response.candidates[0].grounding_metadata.grounding_chunks])}" -) -``` - -## Code Execution -Allow the model to run Python code to calculate answers precisely. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="Calculate the 20th fibonacci number.", - config=types.GenerateContentConfig( - tools=[types.Tool(code_execution=types.ToolCodeExecution())], - ), -) -print(response.executable_code) -print(response.code_execution_result) -``` - -## Url Context -You can use the URL context tool to provide Gemini with URLs as additional context for your prompt. The model can then retrieve content from the URLs and use that content to inform and shape its response. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="Compare recipes from http://example.com and http://example2.com", - config=types.GenerateContentConfig( - tools=[types.Tool(url_context=types.UrlContext())], - ), -) - -print(response.text) -# get URLs retrieved for context -print(response.candidates[0].url_context_metadata) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-api/references/text_and_multimodal.md b/plugins/cloud/gemini-api/skills/gemini-api/references/text_and_multimodal.md deleted file mode 100644 index 54b0f3d..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-api/references/text_and_multimodal.md +++ /dev/null @@ -1,105 +0,0 @@ -# Text and Multimodal Generation - -## Basic Text Generation -```python -from google import genai - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents="How does AI work?", -) -print(response.text) -``` - -## Chat (Multi-turn conversations) -```python -from google import genai -from google.genai import types - -client = genai.Client() -chat_session = client.chats.create( - model="gemini-3.6-flash", - history=[ - types.UserContent( - parts=[ - types.Part.from_text( - text="Hello", - ) - ] - ), - types.ModelContent( - parts=[ - types.Part.from_text( - text="Great to meet you. What would you like to know?" - ) - ] - ), - ], -) -response = chat_session.send_message("Tell me a story.") -print(response.text) -``` - -## Synchronous Streaming - -Generate content in a streaming format so that the model outputs streams back -to you, rather than being returned as one chunk. - -```python -from google import genai -from google.genai import types - -client = genai.Client() -for chunk in client.models.generate_content_stream( - model="gemini-3.6-flash", contents="Tell me a story in 300 words." -): - print(chunk.text, end="") -``` - -## Multimodal Inputs (Images, Audio, Video) -You can provide files natively using Google Cloud Storage URIs or local bytes. - -```python -from google import genai -from google.genai import types - -client = genai.Client() - -gcs_image = types.Part.from_uri( - file_uri="gs://cloud-samples-data/generative-ai/image/scones.jpg", - mime_type="image/jpeg", -) - -with open("local_image.jpg", "rb") as f: - local_image = types.Part.from_bytes(data=f.read(), mime_type="image/jpeg") - -response = client.models.generate_content( - model="gemini-3.6-flash", - contents=[ - "Generate a list of all the objects contained in both images.", - gcs_image, - local_image, - ], -) -print(response.text) -``` - -### YouTube Videos -```python -from google import genai -from google.genai import types - -client = genai.Client() -response = client.models.generate_content( - model="gemini-3.6-flash", - contents=[ - types.Part.from_uri( - file_uri="https://www.youtube.com/watch?v=3KtWfp0UopM", - mime_type="video/mp4", - ), - "Write a short and engaging blog post based on this video.", - ], -) -print(response.text) -``` diff --git a/plugins/cloud/gemini-api/skills/gemini-interactions-api/SKILL.md b/plugins/cloud/gemini-api/skills/gemini-interactions-api/SKILL.md deleted file mode 100644 index 60f3913..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-interactions-api/SKILL.md +++ /dev/null @@ -1,591 +0,0 @@ ---- -name: gemini-interactions-api -metadata: - category: AiAndMachineLearning -description: Guides the usage of Gemini Interactions API on Gemini Enterprise Agent Platform. Use when the user wants to use the stateful, server-managed Interactions API for multi-turn conversations, background execution, streaming, structured output, and function calling on the Agent Platform. ---- - -# Gemini Interactions API Skill - -This skill provides instructions for authenticating, connecting to, and utilizing the stateful, server-managed **Gemini Interactions API** on Gemini Enterprise Agent Platform. - - -The Interactions API is the modern, recommended way to execute Generative AI agent conversations, background research tasks, multi-turn chats, and structured, multi-step workflows. - -> [!IMPORTANT] **CRITICAL: Unified SDK, Latest Models & GEAP Targeting** * -> **Unified SDK**: Use the Google Gen AI SDK (**`google-genai >= 2.3.0`** for -> Python, **`@google/genai >= 2.3.0`** for JS/TS). SDK versions `>= 2.0.0` are -> the minimum floor at which the steps schema activates, but `>= 2.3.0` is the -> supported/recommended floor. Legacy SDKs like `google-cloud-aiplatform`, -> `@google-cloud/vertexai`, and `google-generativeai` are strictly unsupported -> for Interactions. * *Mandatory Instruction*: When explaining client -> initialization or writing import code, you **must** explicitly state/warn the -> user that legacy packages like `google-cloud-aiplatform` or -> `google-generativeai` are strictly unsupported for interactions. * **Latest -> Models Only**: Use `gemini-3.5-flash` (fast, balanced, multimodal — the -> recommended default), `gemini-3.1-pro-preview` (complex reasoning, coding, -> research), or `gemini-3.1-flash-lite` (cost-efficient, high-frequency -> lightweight tasks). Refer to the -> [latest model versions](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/migrate) -> to check for new updates. Legacy models (`gemini-3-flash-preview`, -> `gemini-2.5-*`, `gemini-2.0-*`, `gemini-1.5-*`) are deprecated and do not -> support interactions. * *Mandatory Instruction*: In any interaction response, -> you **must** warn the user that legacy models like `gemini-2.5-*`, -> `gemini-2.0-*`, or `gemini-1.5-*` are deprecated and unsupported for the -> Interactions API. * **GEAP requires a provisioned agent (no direct base-model -> calls yet)**: On Gemini Enterprise Agent Platform (GEAP), direct/base-model -> calls (`model="..."`) via the Interactions API are **not supported yet**. You -> **must** target a provisioned agent or endpoint with the `agent=""` -> parameter instead of `model="..."`. The code examples in this skill use -> `agent=...` for this reason. (This is the primary difference from the -> [ai.google.dev](https://ai.google.dev/gemini-api/docs/interactions) -> documentation for Interactions, which uses `model=...` — while `model=...` is -> valid for other Gemini API contexts, it is **not supported on the Agent -> Platform**.) Provision an agent per the -> [Agent Platform docs](https://docs.cloud.google.com/gemini-enterprise-agent-platform) -> and pass its ID as `agent`. * **Turn-Scoped Parameters**: Parameters like -> `tools`, `system_instruction`, and `generation_config` are turn-scoped. They -> **MUST** be passed with each interaction request. - -## 1. Authentication - -Before running any code, ensure you are authenticated with Application Default Credentials (ADC) and have the necessary API enabled. - -1. **Login**: - - ```bash - gcloud auth application-default login - ``` -2. **Enable API** (if not already enabled): - - ```bash - gcloud services enable aiplatform.googleapis.com - ``` - ---- - -## 2. Client Initialization - -You can initialize the client using environment variables (recommended) or by passing explicit configuration parameters. - -### Option A: Environment Variables (Recommended) - -Configure environment variables to let the SDK automatically resolve settings: - -```bash -export GOOGLE_GENAI_USE_ENTERPRISE=true -export GOOGLE_CLOUD_PROJECT="your-project-id" -export GOOGLE_CLOUD_LOCATION="global" -``` - -#### Python - -```python -from google import genai - -# The SDK automatically picks up the environment variables -client = genai.Client() -``` - -#### TypeScript/JavaScript - -```typescript -import { GoogleGenAI } from "@google/genai"; - -// The SDK automatically picks up the environment variables -const ai = new GoogleGenAI(); -``` - -### Option B: Explicit Inline Parameters - -Alternatively, pass configuration values directly inside your code: - -#### Python - -```python -from google import genai -import google.auth - -_, project_id = google.auth.default() -client = genai.Client(enterprise=True, project=project_id, location="global") -``` - -#### TypeScript/JavaScript - -```typescript -import { GoogleGenAI } from "@google/genai"; - -const ai = new GoogleGenAI({ - enterprise: { - project: "your-project-id", - location: "global" - } -}); -``` - ---- - -## 3. Core Interactions API Usage - -### Quick Start (Single-Turn) - -Submit a single prompt and read the final text response. Under the modern schema, output content is retrieved from the `steps` list. - -#### Python - -```python -interaction = client.interactions.create( - agent="your-agent-id", # GEAP: target a provisioned agent, not a base model - input="Explain serverless computing in one sentence." -) -# Use the output_text convenience accessor (combined text from the trailing model_output steps) -print(interaction.output_text) -``` - -#### TypeScript/JavaScript - -```typescript -const interaction = await ai.interactions.create({ - agent: "your-agent-id", // GEAP: target a provisioned agent, not a base model - input: "Explain serverless computing in one sentence." -}); -console.log(interaction.output_text); -``` - ---- - -### Stateful Conversation (Multi-Turn) - -Interactions are stateful by default. Store the conversation state in the cloud and reference it in the subsequent turn using `previous_interaction_id`. - -#### Python - -```python -# Turn 1: Introduce ourselves -# Interactions are stored by default (store=True); pass store=False to disable -# server-side retention (which also disables previous_interaction_id and background). -turn1 = client.interactions.create( - agent="your-agent-id", - input="Hi! My name is John. I am working on AI agents.", - store=True -) -print(f"Turn 1: {turn1.output_text}") - -# Turn 2: Refer back to the stored turn state -turn2 = client.interactions.create( - agent="your-agent-id", - input="What is my name?", - previous_interaction_id=turn1.id -) -print(f"Turn 2: {turn2.output_text}") -``` - -#### TypeScript/JavaScript - -```typescript -// Turn 1 (interactions are stored by default; pass store: false to disable) -const turn1 = await ai.interactions.create({ - agent: "your-agent-id", - input: "Hi! My name is John. I am working on AI agents.", - store: true -}); - -// Turn 2 -const turn2 = await ai.interactions.create({ - agent: "your-agent-id", - input: "What is my name?", - previousInteractionId: turn1.id -}); -console.log(turn2.output_text); -``` - ---- - -### Real-Time Streaming - -Stream responses in real-time. Passing `stream=True` returns an iterable chunk generator. - -#### Python - -```python -# The stream yields typed events, not full interaction snapshots. The sequence is: -# interaction.created -> (step.start -> step.delta(s) -> step.stop)+ -> interaction.completed -for event in client.interactions.create( - agent="your-agent-id", - input="Write a short poem about debugging.", - stream=True -): - if event.event_type == "step.delta": - if event.delta.type == "text": - print(event.delta.text, end="", flush=True) - elif event.event_type == "interaction.completed": - print() -``` - -#### TypeScript/JavaScript - -```typescript -// The stream yields typed events, not full interaction snapshots. The sequence is: -// interaction.created -> (step.start -> step.delta(s) -> step.stop)+ -> interaction.completed -const responseStream = await ai.interactions.create({ - agent: "your-agent-id", - input: "Write a short poem about debugging.", - stream: true -}); - -for await (const event of responseStream) { - if (event.event_type === "step.delta") { - if (event.delta.type === "text") { - process.stdout.write(event.delta.text); - } - } else if (event.event_type === "interaction.completed") { - console.log(); - } -} -``` - ---- - -### Structured Output (Pydantic / Polymorphic `response_format`) - -Retrieve structured, type-safe JSON matching a schema. Under the modern Interactions API, a polymorphic `response_format` argument directly takes the target schema structure. - -#### Python - -```python -from pydantic import BaseModel, Field - -class Book(BaseModel): - title: str = Field(description="The title of the book") - author: str = Field(description="The book's author") - year_published: int - -interaction = client.interactions.create( - agent="your-agent-id", - input="Recommend one famous sci-fi book.", - response_format=Book -) - -# The text will be a valid JSON matching the Book schema -print(interaction.output_text) -``` - -#### TypeScript/JavaScript - -```typescript -import { Type } from "@google/genai"; - -const BookSchema = { - type: Type.OBJECT, - properties: { - title: { type: Type.STRING, description: "The title of the book" }, - author: { type: Type.STRING, description: "The book's author" }, - yearPublished: { type: Type.INTEGER } - }, - required: ["title", "author", "yearPublished"] -}; - -const interaction = await ai.interactions.create({ - agent: "your-agent-id", - input: "Recommend one famous sci-fi book.", - responseFormat: BookSchema -}); - -console.log(interaction.output_text); -``` - ---- - -### Function Calling (Agent Tool Use) - -Define local tools (functions) and submit execution results to the stateful interaction history. - -#### Python - -```python -import json - -def get_stock_price(ticker: str) -> float: - """Gets the stock price for a given ticker symbol.""" - if ticker.upper() == "GOOG": - return 175.50 - return 100.0 - -# Turn 1: Pass tools to the model -interaction = client.interactions.create( - agent="your-agent-id", - input="What is the stock price of GOOG?", - tools=[get_stock_price] -) - -# In the flat steps schema, a tool request is a top-level step of type -# "function_call" with flat `name` and `arguments` fields (no nested tool_calls). -for step in interaction.steps: - if step.type == "function_call" and step.name == "get_stock_price": - ticker_arg = step.arguments.get("ticker") - price = get_stock_price(ticker_arg) - - # Turn 2: Submit the result back as a function_result step. Reference the - # originating call via call_id=step.id, and pass tools again (turn-scoped). - final_turn = client.interactions.create( - agent="your-agent-id", - input=[ - { - "type": "function_result", - "name": step.name, - "call_id": step.id, - "result": [{"type": "text", "text": json.dumps(price)}], - } - ], - tools=[get_stock_price], - previous_interaction_id=interaction.id - ) - print(final_turn.output_text) -``` - -#### TypeScript/JavaScript - -```typescript -import { Type } from "@google/genai"; - -// Define local tool -function getStockPrice({ ticker }: { ticker: string }): number { - if (ticker.toUpperCase() === "GOOG") { - return 175.50; - } - return 100.00; -} - -// Turn 1: Pass tools to the model -const toolDeclaration = { - functionDeclarations: [{ - name: "getStockPrice", - description: "Gets the stock price for a given ticker symbol.", - parameters: { - type: Type.OBJECT, - properties: { - ticker: { type: Type.STRING, description: "The stock ticker symbol" } - }, - required: ["ticker"] - } - }] -}; - -const interaction = await ai.interactions.create({ - agent: "your-agent-id", - input: "What is the stock price of GOOG?", - tools: [toolDeclaration] -}); - -// In the flat steps schema, a tool request is a top-level step of type -// "function_call" with flat `name` and `arguments` fields (no nested toolCalls). -const fcStep = interaction.steps.find(s => s.type === "function_call"); -if (fcStep && fcStep.name === "getStockPrice") { - const tickerArg = fcStep.arguments.ticker as string; - const price = getStockPrice({ ticker: tickerArg }); - - // Turn 2: Submit the result back as a function_result step. Reference the - // originating call via call_id=fcStep.id, and pass tools again (turn-scoped). - const finalTurn = await ai.interactions.create({ - agent: "your-agent-id", - input: [{ - type: "function_result", - name: fcStep.name, - call_id: fcStep.id, - result: [{ type: "text", text: JSON.stringify(price) }] - }], - tools: [toolDeclaration], - previousInteractionId: interaction.id - }); - console.log(finalTurn.output_text); -} -``` - ---- - -## 4. Accessing the Interactions API via REST - -For shell-based scripts, debugging, or non-Python/JS environments, you can communicate with the stateful Interactions API directly using raw HTTP/REST requests via `curl`. - -### 1. REST Endpoint - -The REST API endpoint for interactions is: - -```http -POST https://aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{LOCATION}/interactions -``` - -* **LOCATION**: Use `global` (or custom region if required). -* **PROJECT_ID**: Your Google Cloud Project ID. - -### 2. Set up Variables & Authentication Header - -Set your target agent ID (e.g., model or custom agent path) and access token generated from Application Default Credentials: - -```bash -AGENT_ID="your-agent-id" -ACCESS_TOKEN=$(gcloud auth print-access-token) -``` - -### 3. Single-Turn Interaction Payload - -Send a request to start an interaction using the agent variable: - -```bash -curl -X POST "https://aiplatform.googleapis.com/v1beta1/projects/${PROJECT_ID}/locations/global/interactions" \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{ - "agent": "'"${AGENT_ID}"'", - "input": [{ - "type": "user_input", - "content": [{ - "type": "text", - "text": "Explain serverless computing in one sentence." - }] - }] - }' -``` - -#### Response Example -A synchronous POST request returns a JSON object containing the conversation step details and unique identifiers: - -```json -{ - "id": "your-interaction-id", - "status": "completed", - "steps": [ - { - "type": "model_output", - "content": [ - { - "type": "text", - "text": "Serverless computing is a cloud execution model where the cloud provider dynamically manages the allocation and provisioning of servers, charging customers based on actual usage rather than pre-purchased capacity." - } - ] - } - ], - "usage": { - "total_tokens": 24751, - "total_input_tokens": 23894, - "total_output_tokens": 857 - }, - "created": "2026-05-08T10:44:43Z", - "updated": "2026-05-08T10:44:43Z", - "environment_id": "your-environment-id", - "object": "interaction" -} -``` - -### 4. Multi-Turn Stateful Interaction Payload - -To continue an existing conversation statefully, specify the `previous_interaction_id` in the JSON payload: - -```bash -curl -X POST "https://aiplatform.googleapis.com/v1beta1/projects/${PROJECT_ID}/locations/global/interactions" \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{ - "agent": "'"${AGENT_ID}"'", - "store": true, - "previous_interaction_id": "YOUR_PREVIOUS_INTERACTION_ID", - "input": [{ - "type": "user_input", - "content": [{ - "type": "text", - "text": "Can you elaborate on that?" - }] - }] - }' -``` - -### 5. Streaming Output Payload -To stream updates in real time (Server-Sent Events format), pass `"stream": true` in the payload: - -```bash -curl -X POST "https://aiplatform.googleapis.com/v1beta1/projects/${PROJECT_ID}/locations/global/interactions" \ - -H "Authorization: Bearer ${ACCESS_TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{ - "agent": "'"${AGENT_ID}"'", - "stream": true, - "input": [{ - "type": "user_input", - "content": [{ - "type": "text", - "text": "Write a long story about space travel." - }] - }] - }' -``` - -The endpoint will return a chunked stream where each event begins with `data: ` containing JSON updates with the `event_type` and step contents. - -> **How `curl` handles streaming:** -> By default, when `"stream": true` is passed, the server responds with `Transfer-Encoding: chunked` and `Content-Type: text/event-stream` (Server-Sent Events). `curl` will automatically keep the connection open and print the incoming data chunks to `stdout` in real time as they are pushed by the server. The user does not need to poll or pull further; the complete sequence of events streams continuously until completion. - --------------------------------------------------------------------------------- - -## 5. Data Model & Step Types Reference - -An `Interaction` response contains `steps`, an array of typed step objects -representing a structured timeline of the interaction turn. Read the current -step `type` rather than assuming the last step is text — the trailing step may -be a `function_call` or a `thought`. - -### Step Types - -**User steps:** - -* `user_input`: User input (text, audio, multimodal). Contains a `content` - array. (This is why REST input payloads use `"type": "user_input"`, **not** - `"role": "user"`.) - -**Model/server steps:** - -* `model_output`: Final model generation. Contains a `content` array with - `text`, `image`, `audio`, etc. (REST responses use `"type": "model_output"`, - **not** `"role": "model"`.) -* `thought`: Model reasoning / chain of thought. Has a `signature` field and - optional `summary`. -* `function_call`: Tool call request, with flat `id`, `name`, and `arguments` - fields (there is **no** nested `tool_calls` list). -* `function_result`: Tool result you send back, with `call_id`, `name`, and - `result` fields. -* `google_search_call` / `google_search_result`, `code_execution_call` / - `code_execution_result`, `url_context_call` / `url_context_result`, - `mcp_server_tool_call` / `mcp_server_tool_result`, `file_search_call` / - `file_search_result`: built-in and remote tool steps. - -### Content types (inside the `content` array on `model_output` and `user_input` steps) - -* `text`: Text content (`text` field). -* `image` / `audio` / `document` / `video`: Content with `data`, `mime_type`, - or `uri`. - -### Convenience accessor - -* `output_text`: The combined text from the trailing `model_output` steps. - Prefer this over hand-walking `steps[-1].content[0].text`, which breaks when - the last step is a tool call or a thought. - -### Streaming Event Types - -| Event | Description | -| ----------------------- | ------------------------------------------------- | -| `interaction.created` | Interaction created; includes metadata. | -| `step.start` | A new step begins. Contains the step `type` and | -: : initial metadata. : -| `step.delta` | Incremental data for the current step. Contains a | -: : typed `delta` object (e.g. `delta.type == "text"` : -: : with `delta.text`). : -| `step.stop` | The step is complete. Contains `index`. | -| `interaction.completed` | Interaction finished. Contains final `usage`. | - -### Storage & retention - -Interactions are stored by default (`store=True`), which enables stateful -features like `previous_interaction_id` and background execution. Passing -`store=False` disables server-side retention and therefore also disables -`previous_interaction_id` and `background` — in that mode you must pass the full -conversation history in `input` on each turn. diff --git a/plugins/cloud/gemini-api/skills/gemini-live-api/SKILL.md b/plugins/cloud/gemini-api/skills/gemini-live-api/SKILL.md deleted file mode 100644 index 79daadc..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-live-api/SKILL.md +++ /dev/null @@ -1,222 +0,0 @@ ---- -name: gemini-live-api -metadata: - category: AiAndMachineLearning -description: >- - Generates a Gemini LiveAPI client service class in the user's chosen programming - language. Use when the user wants to build, scaffold, or integrate a client - that connects to the Gemini Enterprise LiveAPI websocket endpoint, handles - session setup/resumption, bearer token refresh, and sending/receiving - `ClientMessage`/`ServerMessage` protos. Don't use for general (non-live, - non-bidirectional) Gemini API usage such as one-shot `generateContent`, - embeddings, image/video generation, or fine-tuning — use the `gemini-api` - skill for those. ---- - -# LiveAPI Service Skill - -This skill provides instructions for generating a **LiveAPI client service -class** that connects to the Gemini Enterprise Live API over WebSockets. The -generated client handles bidirectional streaming, bearer-token authentication -via Application Default Credentials (ADC), transparent session resumption, and -`ClientMessage` / `ServerMessage` proto exchange. - -The skill also produces a demo frontend + backend service so the user can -interactively validate the generated client (text, audio, video, transcription, -and interrupt handling). - -## Prerequisites - -Before running the generation flow, ensure the following are available on the -host: - -- A Google Cloud project with the Vertex AI / Gemini Enterprise Agent Platform - APIs enabled. -- Application Default Credentials configured on the host running the generated - client: - - ```bash - gcloud auth application-default login - ``` - -- A destination output folder supplied by the user (e.g. `/tmp/liveapi_out`) - where the generated code, environment, and demo will be written. **Never** - mutate the host's system Python environment. - -- The user's chosen implementation language (Python is the default and - reference language for this skill). - -## Reference Files - -Provided files in `references/` (do **not** treat these as standalone skills — -they are loaded on demand): - -- `client_server_messages.md`: Public reference for the `ClientMessage` / - `ServerMessage` schemas used by the Live API. -- `client_server_messages.proto`: The proto definition generated from - `client_server_messages.md`. -- `session_manager.md`: Describes how to correctly handle sessions, buffering, - and resumption on disconnection. - -## Steps - -### Step 1: Copy the reference files - -Copy `client_server_messages.md`, `client_server_messages.proto`, and -`session_manager.md` from this skill's `references/` folder into the user's -destination output folder. These files become the source of truth for the -generated client. - -### Step 2: Reconcile with the public documentation - -Examine the public documents linked from `client_server_messages.md`. If there -are any discrepancies between the public documents and the copied -`client_server_messages.md` / `client_server_messages.proto`, update the copies -in the destination folder so the generated client compiles and runs against the -current server contract. - -### Step 3: Implement the client class - -Implement a class in the user's chosen language that: - -- Imports the local `client_server_messages.proto` types (`ClientMessage`, - `ServerMessage`). -- Opens a WebSocket connection to the Live API endpoint. -- Exposes async methods so the user can send and receive data to/from the - model. - -For languages that require an isolated runtime (e.g. Python), create an isolated -environment (e.g. `venv`) **inside the destination folder** and generate a bash -script (e.g. `setup.sh`) that recreates the environment and installs -dependencies. **Never** install into the system interpreter or the user's global -site-packages, and never instruct the user to run `sudo pip install`. - -#### Initialization parameters - -The user provides the following at construction time: - -- `project_id` -- `location` -- `model_id` -- `config`: a `ClientMessage` with the `setup` field populated. - -#### Authentication - -Obtain a bearer token via Application Default Credentials, attach it to the -WebSocket connect request as `Authorization: Bearer `, refresh the token -before or upon expiry, and reuse the refreshed token on every reconnection -(including `go_away` and unexpected disconnects). **Do not** hard-code a -long-lived API key as the only auth mechanism. - -#### Public async API - -The class MUST expose the following async methods, gated on receipt of a -`setup_complete` `ServerMessage` before sending: - -- `send_realtime_data(data)`: send realtime input. `data` is a `ClientMessage` - carrying a `realtime_input` field. -- `send_client_content(data)`: send non-realtime, turn-based content that - contributes to history. `data` is a `ClientMessage` carrying a - `client_content` field. -- `receive()`: yield `ServerMessage` instances parsed from the WebSocket - stream. - -Do not expose synchronous blocking variants as the primary API surface. - -### Step 4: Write a test file - -Once the client is implemented, generate a test file that initializes the -connection and exercises sending `text`, `audio`, and `video` data and receiving -the responses. Ask the user for any information required to run the test -(project, model, media samples). - -### Step 5: Generate `how_to_run.md` - -Provide a `how_to_run.md` in the destination folder that documents the generated -class. Include full examples showing how to build `ClientMessage` payloads for -every supported modality, how to send them, and how to receive data from the -model. - -### Step 6: Generate a demo frontend + backend service - -Create scripts that deploy the implementation as a service with both a frontend -UI and a backend service (any language). The service MUST reuse the -`ClientMessage` / `ServerMessage` protos from Step 1 for wire traffic. Through -the UI the user should be able to: - -- Start a new connection / close the current connection. -- Select the model to use. -- Select input sources (audio and/or video from camera or screenshot) and - stream them to the model. -- Send a text message to the model. -- Hear model audio and see the interleaved model and user transcription / - conversation history. - -While implementing audio and transcription playback, follow the guidance in -[Live API best practices](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/live-api/best-practices). - -#### Handling the `interrupt` signal - -When a `ServerMessage`'s `server_content` arrives with `interrupted: true`, the -UI MUST: - -- Ensure played audio and its corresponding transcription remain time-aligned. -- Immediately stop the currently playing model audio and stop appending to the - in-progress transcription bubble. -- Clear the unplayed audio buffer and any pending unrendered transcription so - stale content does not bleed into the next turn. -- Start new chat bubbles for the next user and model turns. - -#### Handling the transcription `finished` signal - -For streamed `input_transcription` / `output_transcription` chunks, append to -the currently active bubble while `finished` is unset, and close that bubble and -start a fresh one when `finished` is observed. Route `input_transcription` text -to user-role bubbles and `output_transcription` text to model-role bubbles. - -### Step 7: Generate `how_to_test_with_ui.md` - -Write `how_to_test_with_ui.md` describing how to launch and use the demo -service. It MUST include: - -- The exact shell command(s) or script invocation(s) to start the backend - service. -- The exact shell command(s) or script invocation(s) to start the frontend UI. -- The host and port (e.g. `http://localhost:PORT`) the user should open in - their browser. -- How to start a session, select a model, choose input sources (mic, camera, - screen), send a text message, and observe model audio and transcription in - the UI. - -## Validation Checklist - -Before considering the generation complete, verify each item: - -- [ ] `client_server_messages.md`, `client_server_messages.proto`, and - `session_manager.md` were copied into the destination folder. -- [ ] The generated client imports the local proto-generated `ClientMessage` - and `ServerMessage` types. -- [ ] The client connects to the Live API WebSocket at - `wss://{location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent` - (or the `wss://aiplatform.googleapis.com/...` global variant), and formats - the setup `model` field as - `projects/{project_id}/locations/{location}/publishers/google/models/{model_id}`. -- [ ] Authentication uses ADC-provided bearer tokens sent as `Authorization: - Bearer `, is refreshed before expiry, and reattached on every - reconnect. -- [ ] Public async methods `send_realtime_data`, `send_client_content`, and - `receive` are present, correctly typed, and gated on `setup_complete`. -- [ ] Transparent session resumption is enabled - (`session_resumption.transparent = true`), the latest `new_handle` is - tracked, sent-message indexing starts at 1, the buffer is pruned via - `last_consumed_client_message_index`, and buffered messages are replayed on - reconnect (including on `go_away` and WebSocket close codes 1000 / 1006). -- [ ] If using python, an isolated environment (e.g. `venv`) plus a `setup.sh` - and `requirements.txt` (or equivalent) exist inside the destination folder; - no changes were made to system or user-global Python. -- [ ] `how_to_run.md` and `how_to_test_with_ui.md` are present, and the demo - UI reuses the same `ClientMessage` / `ServerMessage` protos. -- [ ] Interrupt handling and transcription `finished` handling behave as - described above. -- [ ] The client does **not** target `generativelanguage.googleapis.com` and - does **not** authenticate via API key in a query string. diff --git a/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.md b/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.md deleted file mode 100644 index a5c8ba6..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.md +++ /dev/null @@ -1,848 +0,0 @@ -# Live API `ClientMessage` and `ServerMessage` Reference - -This document describes the **wire-level WebSocket protocol** for the Gemini -Enterprise Agent Platform Live API (`BidiGenerateContent*`). - -Docs: -https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/models/multimodal-live - -The service exposes the RPC `BidiGenerateContent`. Search the corresponding -website for more information. - ---- - -## 1. Connection - -### Endpoints - -Backend | WebSocket URI ------------------------------------------ | ------------- -Gemini Enterprise Agent Platform | `wss://{LOCATION}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent` -Gemini Enterprise Agent Platform (global) | `wss://aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent` - -### Auth - -- `Authorization: Bearer ` (or API key in Express mode). - -### Frames - -All frames are JSON-serialized protobuf messages of either `ClientMessage` -(client → server) or `ServerMessage` (server → client). Each frame is exactly -one WebSocket text message. - ---- - -## 2. Top-level structure (oneof) - -`ClientMessage` and `ServerMessage` are both **oneof envelopes** — each frame -sets exactly one of the listed fields. - -### `ClientMessage` (you → server) - -| Field | Type | When to use | -| --- | --- | --- | -| `setup` | `BidiGenerateContentSetup` | First frame only. Configures the session. | -| `clientContent` | `BidiGenerateContentClientContent` | Append to conversation history; turn-based input. Interrupts the model. | -| `realtimeInput` | `BidiGenerateContentRealtimeInput` | Continuous, low-latency audio/video/text input. Not added to history. | -| `toolResponse` | `BidiGenerateContentToolResponse` | Reply to a server-issued `toolCall`. | - -### `ServerMessage` (server → you) - -| Field | Type | Meaning | -| --- | --- | --- | -| `setupComplete` | `BidiGenerateContentSetupComplete` | Sent once after `setup` is accepted. Gate further sends on this. | -| `serverContent` | `BidiGenerateContentServerContent` | Streamed model output (audio/text), turn lifecycle. | -| `toolCall` | `BidiGenerateContentToolCall` | Model is requesting tool execution. | -| `toolCallCancellation` | `BidiGenerateContentToolCallCancellation` | Cancels previously-issued tool calls (e.g. on user interruption). | -| `usageMetadata` | `UsageMetadata` | Token / duration accounting. | -| `goAway` | `GoAway` | Connection will be terminated soon. | -| `sessionResumptionUpdate` | `SessionResumptionUpdate` | Resume handle for reconnects. | - -> Audio transcriptions are delivered **inside `serverContent`** (see § 9), not -> as separate top-level frames. - ---- - -## 3. Lifecycle - -``` -Client Server - │ │ - ├── ClientMessage{ setup } ──────────► │ - │ │ - │ ◄────── ServerMessage{ setupComplete } - │ │ - ├── realtimeInput / clientContent ────► │ - │ (audio frames, text, etc.) │ - │ │ - │ ◄────── serverContent (audio chunks, modelTurn parts ...) - │ ◄────── serverContent { generationComplete: true } - │ ◄────── serverContent { turnComplete: true } - │ │ - │ ◄────── toolCall { functionCalls[] } - ├── toolResponse { functionResponses[] } ► │ - │ │ - │ ◄────── serverContent ... - │ ◄────── goAway { timeLeft } (eventually) - │ │ - │ (close & reconnect using sessionResumptionUpdate.newHandle) -``` - -**Rules:** - -1. The first frame **must** be `setup`. Do not send anything else until you - receive `setupComplete`. -2. Use **`clientContent`** for turn-based, history-affecting messages - (e.g. a typed user message). Sending it interrupts any current model - generation. -3. Use **`realtimeInput`** for continuous audio/video. It does **not** go - into history. Turn boundaries come from VAD (or from explicit - `activityStart`/`activityEnd` if VAD is disabled). -4. Reply to `toolCall` with `toolResponse` (never with `clientContent`). -5. On `goAway`, reconnect using the most recent - `sessionResumptionUpdate.newHandle`. - ---- - -## 4. `BidiGenerateContentSetup` (`setup`) - -Initial-and-only-once configuration for the session. - -Field | Type | Notes --------------------------- | -------------------------------- | ----- -`model` | `string` (required) | `projects/{p}/locations/{l}/publishers/google/models/{m}`. -`generationConfig` | `GenerationConfig` | Unsupported sub-fields here: `responseLogprobs`, `responseMimeType`, `logprobs`, `responseSchema`, `stopSequence`, `routingConfig`, `audioTimestamp`. -`systemInstruction` | `Content` | Text-only parts. -`tools[]` | repeated `Tool` | Function declarations and built-ins (Search, code execution). -`sessionResumption` | `SessionResumptionConfig` | `{ handle?: string, transparent?: bool }`. Provide `handle` to resume; omit to start a new resumable session. -`contextWindowCompression` | `ContextWindowCompressionConfig` | `{ triggerTokens?: int64, slidingWindow?: { targetTokens?: int64 } }`. -`realtimeInputConfig` | `RealtimeInputConfig` | See below. -`inputAudioTranscription` | `AudioTranscriptionConfig` | `{}`. -`outputAudioTranscription` | `AudioTranscriptionConfig` | `{}`. - -### `RealtimeInputConfig` - -Field | Type | Notes ----------------------------- | ---------------------------- | ----- -`automaticActivityDetection` | `AutomaticActivityDetection` | Unset → server-side VAD enabled by default. -`activityHandling` | enum | `START_OF_ACTIVITY_INTERRUPTS` (default) \| `NO_INTERRUPTION`. -`turnCoverage` | enum | Default `TURN_INCLUDES_ALL_INPUT`. Also `TURN_INCLUDES_ONLY_ACTIVITY`, `TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO`. - -### `AutomaticActivityDetection` - -| Field | Type | Notes | -| --- | --- | --- | -| `disabled` | `bool` | If true, you must send `activityStart` / `activityEnd` yourself. | -| `startOfSpeechSensitivity` | enum | `START_SENSITIVITY_HIGH` \| `LOW`. | -| `endOfSpeechSensitivity` | enum | `END_SENSITIVITY_HIGH` \| `LOW`. | -| `prefixPaddingMs` | `int32` | Min speech duration to commit start-of-speech. | -| `silenceDurationMs` | `int32` | Min silence to commit end-of-speech. | - -### `GenerationConfig` — Live-API-relevant subset - -Fields most often used in `setup.generationConfig`: - -| Field | Type | Notes | -| --- | --- | --- | -| `responseModalities[]` | repeated enum | `TEXT` \| `AUDIO`. Pick **one** per session (mixing not supported). Default `AUDIO`. | -| `temperature` | `float` | 0.0–2.0. | -| `topP`, `topK`, `maxOutputTokens` | various | Standard sampling / length controls. | -| `speechConfig` | `SpeechConfig` | Voice / language — only effective when `responseModalities=[AUDIO]`. See below. | -| `mediaResolution` | enum | `MEDIA_RESOLUTION_LOW` \| `MEDIUM` \| `HIGH`. Controls token-cost vs. quality of input images / video frames. | -| `thinkingConfig` | `ThinkingConfig` | `{ thinkingBudget?: int }`. Only on models that support thinking (e.g. Gemini 2.5 Flash). Set `thinkingBudget: 0` to disable. | - -> **Unsupported in Live `setup.generationConfig`:** `responseLogprobs`, -> `responseMimeType`, `logprobs`, `responseSchema`, `stopSequence`, -> `routingConfig`, `audioTimestamp`. - -### `SpeechConfig` - -Controls the **voice** the model speaks with. Only meaningful when -`responseModalities` includes `AUDIO`. - -| Field | Type | Notes | -| --- | --- | --- | -| `voiceConfig` | `VoiceConfig` | Single-speaker voice. The Live API supports **only single-speaker** output. | -| `languageCode` | `string` | BCP-47 (e.g. `en-US`, `de-DE`, `ja-JP`). Output language. Native-audio models auto-detect / switch on their own; for non-native-audio models, set this explicitly. See [Languages supported](#languages-supported). | - -> **Note on multi-speaker output:** A Live API session has exactly one -> `prebuiltVoiceConfig.voiceName`. To approximate multiple speakers in a Live -> session, use prompt engineering inside `systemInstruction` or `clientContent` -> to have the single voice play different roles. - -#### `VoiceConfig` - -| Field | Type | Notes | -| --- | --- | --- | -| `prebuiltVoiceConfig` | `PrebuiltVoiceConfig` | Selects a named built-in voice. | - -#### `PrebuiltVoiceConfig` - -| Field | Type | Notes | -| --- | --- | --- | -| `voiceName` | `string` | Name of the prebuilt voice. **30 voices supported.** See [Voices supported](#voices-supported) below for the full list and references. | - -##### Voices supported - -The Live API supports **30 prebuilt voices**. Names are case-sensitive. - -| Voice | Style | Voice | Style | Voice | Style | -| --- | --- | --- | --- | --- | --- | -| Zephyr | Bright | Puck | Upbeat | Charon | Informative | -| Kore | Firm | Fenrir | Excitable | Leda | Youthful | -| Orus | Firm | Aoede | Breezy | Callirrhoe | Easy-going | -| Autonoe | Bright | Enceladus | Breathy | Iapetus | Clear | -| Umbriel | Easy-going | Algieba | Smooth | Despina | Smooth | -| Erinome | Clear | Algenib | Gravelly | Rasalgethi | Informative | -| Laomedeia | Upbeat | Achernar | Soft | Alnilam | Firm | -| Schedar | Even | Gacrux | Mature | Pulcherrima | Forward | -| Achird | Friendly | Zubenelgenubi | Casual | Vindemiatrix | Gentle | -| Sadachbia | Lively | Sadaltager | Knowledgeable | Sulafat | Warm | - -**Reference (authoritative voice list):** - -- - -> The available set may vary per model (e.g. native-audio vs. half-cascade -> models). If a voice is rejected during `setup`, the WebSocket closes -> instead of returning `setupComplete`. - -#### Languages supported - -The Live API supports **24 BCP-47 languages** for `speechConfig.languageCode`: - -`ar-EG`, `bn-BD`, `de-DE`, `en-IN` (bundled with `hi-IN`), `en-US`, `es-US`, -`fr-FR`, `hi-IN`, `id-ID`, `it-IT`, `ja-JP`, `ko-KR`, `mr-IN`, `nl-NL`, -`pl-PL`, `pt-BR`, `ro-RO`, `ru-RU`, `ta-IN`, `te-IN`, `th-TH`, `tr-TR`, -`uk-UA`, `vi-VN`. - -Reference: - - -> Native-audio models (e.g. `gemini-live-2.5-flash-native-audio`) can switch -> languages mid-conversation; for those, `languageCode` is optional and the -> model auto-detects. For non-native-audio models, set it explicitly. - -#### Examples - -Basic prebuilt voice: - -```json -{ - "setup": { - "model": "...", - "generationConfig": { - "responseModalities": ["AUDIO"], - "speechConfig": { - "voiceConfig": { - "prebuiltVoiceConfig": { "voiceName": "Aoede" } - } - } - } - } -} -``` - -Voice + output language pinned to German: - -```json -{ - "setup": { - "model": "...", - "generationConfig": { - "responseModalities": ["AUDIO"], - "speechConfig": { - "voiceConfig": { "prebuiltVoiceConfig": { "voiceName": "Charon" } }, - "languageCode": "de-DE" - } - } - } -} -``` - -> **Notes** - `speechConfig` is silently ignored if `responseModalities` is -> `["TEXT"]`. - `voiceName` is case-sensitive and model-specific. Sending an -> unknown name typically fails the `setup` (you'll see the WebSocket close -> instead of `setupComplete`). - `speechConfig.languageCode` controls **output** -> TTS language. - The Live API is **single-speaker only**. - ---- - -## 5. `BidiGenerateContentClientContent` (`clientContent`) - -Turn-based input. Append to history and (optionally) trigger generation. - -| Field | Type | Notes | -| --- | --- | --- | -| `turns[]` | repeated `Content` | Conversation history + the latest user request. | -| `turnComplete` | `bool` | If true, server starts generating immediately. | - -> Sending `clientContent` while the model is speaking **interrupts** it. Do -> not use `clientContent` to deliver `FunctionResponse`s — use -> `toolResponse`. - ---- - -## 6. `BidiGenerateContentRealtimeInput` (`realtimeInput`) - -Continuous, low-latency input. Does not populate history. End-of-turn is -derived from VAD (or activity events). - -| Field | Type | Notes | -| ---------------- | ----------------------- | ---------------------------- | -| `audio` | `Blob` | Typed realtime audio. PCM | -: : : 16-bit, 16 kHz mono input : -: : : (`audio/pcm;rate=16000`). : -| `video` | `Blob` | Typed realtime video frame | -: : : (`image/jpeg`/`png`/`webp`), : -: : : should be sent at 1 fps. : -| `text` | `string` | Realtime text input. | -| `mediaChunks[]` | repeated `Blob` | **Deprecated.** Combined | -: : : audio/video chunks. Prefer : -: : : the typed `audio` / `video` : -: : : / `text` fields instead. : -| `audioStreamEnd` | `bool` | Mic turned off. Only valid | -: : : with auto-VAD enabled. : -| `activityStart` | `ActivityStart` (empty) | Only when auto-VAD is | -: : : **disabled**. : -| `activityEnd` | `ActivityEnd` (empty) | Only when auto-VAD is | -: : : **disabled**. : - -**Audio formats** - -- Input: `audio/pcm;rate=16000` (16 kHz, 16-bit signed PCM, mono, little-endian). -- Output: 24 kHz, 16-bit signed PCM, mono. - ---- - -## 7. `BidiGenerateContentToolResponse` (`toolResponse`) - -| Field | Type | Notes | -| --------------------- | ------------------ | ----------------------------- | -| `functionResponses[]` | repeated | Match `FunctionCall.id` from | -: : `FunctionResponse` : the server. `id` is optional. : - ---- - -## 8. `BidiGenerateContentSetupComplete` (`setupComplete`) - -| Field | Type | Notes | -| --- | --- | --- | -| `sessionId` | `string` | Server-assigned session identifier. | - -Receipt of this message is the gate for sending any other client frame. - ---- - -## 9. `BidiGenerateContentServerContent` (`serverContent`) - -Primary streaming-output channel. - -| Field | Type | Notes | -| --- | --- | --- | -| `modelTurn` | `Content` | Streamed model output parts (text and/or `inlineData` audio). | -| `generationComplete` | `bool` | Model has finished generating; playback may still be flushing. | -| `turnComplete` | `bool` | Logical end of turn. | -| `interrupted` | `bool` | Generation was interrupted by client input — drop any queued audio playback. | -| `groundingMetadata` | `GroundingMetadata` | When grounding (e.g. Google Search) is used. | -| `inputTranscription` | `Transcription` | `{ text?: string }` — transcription of the user's spoken input. Requires `inputAudioTranscription` set in `setup`. | -| `outputTranscription` | `Transcription` | `{ text?: string }` — transcription of the model's spoken output. Requires `outputAudioTranscription` set in `setup`. | - -**Playback note:** audio chunks arrive inside `modelTurn.parts[].inlineData` -with mimeType `audio/pcm;rate=24000`. Concatenate as they stream. On -`interrupted: true`, **flush** the playback queue. - ---- - -## 10. `BidiGenerateContentToolCall` (`toolCall`) - -| Field | Type | Notes | -| --- | --- | --- | -| `functionCalls[]` | repeated `FunctionCall` | Each has `id`, `name`, `args`. Reply with a matching `FunctionResponse.id`. | - ---- - -## 11. `BidiGenerateContentToolCallCancellation` (`toolCallCancellation`) - -| Field | Type | Notes | -| --- | --- | --- | -| `ids[]` | repeated `string` | IDs of previously-issued tool calls to cancel. Typically caused by user interruption. Stop the work; do not send `toolResponse` for these. | - ---- - -## 12. `GoAway`, `SessionResumptionUpdate`, `UsageMetadata` - -### `GoAway` - -| Field | Type | Notes | -| --- | --- | --- | -| `timeLeft` | `Duration` (string) | Time until the connection terminates with `ABORTED`. Reconnect using the latest `SessionResumptionUpdate.newHandle`. | - -### `SessionResumptionUpdate` - -| Field | Type | Notes | -| --- | --- | --- | -| `newHandle` | `string` | Handle to resume; empty if `resumable=false`. | -| `resumable` | `bool` | Whether the session is currently resumable. | -| `lastConsumedClientMessageIndex` | `int64` | Set only when `SessionResumptionConfig.transparent=true` — enables transparent reconnect. | - -### `UsageMetadata` - -| Field | Type | -| --- | --- | -| `totalTokenCount` | `int32` | -| `textCount` | `int32` | -| `imageCount` | `int32` | -| `videoDurationSeconds` | `int32` | -| `audioDurationSeconds` | `int32` | - ---- - -## 13. End-to-end JSON examples - -### Setup - -```json -{ - "setup": { - "model": "projects/my-proj/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09", - "generationConfig": { - "responseModalities": ["AUDIO"], - "speechConfig": { - "voiceConfig": { "prebuiltVoiceConfig": { "voiceName": "Aoede" } } - } - }, - "systemInstruction": { - "parts": [{ "text": "You are a concise voice assistant." }] - }, - "realtimeInputConfig": { - "automaticActivityDetection": { "disabled": false } - }, - "sessionResumption": {}, - "outputAudioTranscription": {} - } -} -``` - -### Setup complete (server) - -```json -{ "setupComplete": { "sessionId": "abc-123" } } -``` - -### Streaming user audio (realtime) - -```json -{ - "realtimeInput": { - "audio": { - "mimeType": "audio/pcm;rate=16000", - "data": "" - } - } -} -``` - -### Typed user message (turn-based) - -```json -{ - "clientContent": { - "turns": [{ "role": "user", "parts": [{ "text": "Hello!" }] }], - "turnComplete": true - } -} -``` - -### Streaming model output (server) - -```json -{ - "serverContent": { - "modelTurn": { - "role": "model", - "parts": [{ - "inlineData": { - "mimeType": "audio/pcm;rate=24000", - "data": "" - } - }] - } - } -} -``` - -### Tool call / response - -```json -// Server → client -{ - "toolCall": { - "functionCalls": [ - { "id": "call_42", "name": "get_weather", "args": { "city": "Paris" } } - ] - } -} - -// Client → server -{ - "toolResponse": { - "functionResponses": [ - { - "id": "call_42", - "name": "get_weather", - "response": { "tempC": 18, "summary": "Partly cloudy" } - } - ] - } -} -``` - -### GoAway + resumption - -```json -{ "goAway": { "timeLeft": "10s" } } -{ "sessionResumptionUpdate": { "newHandle": "ses_xyz", "resumable": true } } -``` - -Reconnect with: - -```json -{ "setup": { "model": "...", "sessionResumption": { "handle": "ses_xyz" } } } -``` - ---- - -## 14. Full input examples — audio, video, text - -There are **two distinct ways** to send user input. Pick one based on intent: - -| | Part 1 — Realtime input | Part 2 — Add-context | -: : (`realtimeInput`) : input (`clientContent`) : -| --------------------- | ------------------------ | ------------------------- | -| **Goal** | Stream live mic / camera | Append a discrete turn to | -: : continuously : the conversation : -| **Latency** | Lowest possible; | Normal request/response | -: : sub-second : : -| **Added to history?** | **No** (transient | **Yes** (persistent | -: : signals) : conversation) : -| **Turn boundary** | VAD (auto), or explicit | Explicit `turnComplete: | -: : `activityStart`/`End` : true` : -| **Effect on model** | Streamed; auto-triggers | Setting `turnComplete` | -: : a turn when VAD fires : triggers generation; : -: : : **interrupts** any : -: : : ongoing model output : -| **Field carriers** | `audio` / `video` / | `turns[].parts[].text` / | -: : `text` : `inlineData` / `fileData` : -| **Typical use** | Live voice + screen | Typed chat, uploading an | -: : sharing, push-to-talk : image/clip, replaying : -: : : history on resume : - -> You may use both in the same session — e.g. send a `clientContent` system -> nudge once, then continue streaming `realtimeInput`. But within a single -> logical user turn, pick one. - -The two parts below give every supported variant for **audio**, **video**, and -**text** in each mode. - ---- - -# Part 1 — Realtime input (`realtimeInput`) - -Continuous, low-latency input that does **not** populate conversation -history. End-of-turn comes from server-side VAD by default, or from explicit -`activityStart` / `activityEnd` events when auto-VAD is disabled in `setup`. - -## 1.A Realtime audio - -Required format: **PCM, 16-bit signed, 16 kHz, mono, little-endian**, base64 -encoded. MIME type: `audio/pcm;rate=16000`. Send one frame per chunk -(~20–100 ms is typical). - -### 1.A.1 Continuous mic - -```json -{ - "realtimeInput": { - "audio": { - "mimeType": "audio/pcm;rate=16000", - "data": "" - } - } -} -``` - -When the mic turns off (and server-side VAD is enabled), commit end-of-stream: - -```json -{ "realtimeInput": { "audioStreamEnd": true } } -``` - -### 1.A.2 Manual VAD (auto-VAD disabled) - -Required `setup`: - -```json -{ - "setup": { - "model": "...", - "realtimeInputConfig": { "automaticActivityDetection": { "disabled": true } } - } -} -``` - -Then explicitly frame each utterance: - -```json -{ "realtimeInput": { "activityStart": {} } } -{ "realtimeInput": { "audio": { "mimeType": "audio/pcm;rate=16000", "data": "..." } } } -{ "realtimeInput": { "audio": { "mimeType": "audio/pcm;rate=16000", "data": "..." } } } -{ "realtimeInput": { "activityEnd": {} } } -``` - -## 1.B Realtime video - -Video = a **stream of sampled image frames** (should be sent at 1 fps). Each -frame is JPEG / PNG / WebP. The model does **not** consume an encoded container -(mp4/webm) — sample frames client-side and send each as an inline image. - -### 1.B.1 Continuous camera - -```json -{ - "realtimeInput": { - "video": { - "mimeType": "image/jpeg", - "data": "" - } - } -} -``` - -## 1.C Realtime text - -```json -{ "realtimeInput": { "text": "Switch to a calmer tone." } } -``` - -## 1.D Combined realtime audio + video (+ text) - -Typical "talk-to-the-screen" flow. Send each modality in its own frame: - -```json -{ "realtimeInput": { "video": { "mimeType": "image/jpeg", "data": "" } } } -{ "realtimeInput": { "audio": { "mimeType": "audio/pcm;rate=16000", "data": "" } } } -{ "realtimeInput": { "video": { "mimeType": "image/jpeg", "data": "" } } } -{ "realtimeInput": { "audio": { "mimeType": "audio/pcm;rate=16000", "data": "" } } } -{ "realtimeInput": { "text": "Focus on the chart in the upper-right." } } -{ "realtimeInput": { "audio": { "mimeType": "audio/pcm;rate=16000", "data": "" } } } -{ "realtimeInput": { "audioStreamEnd": true } } -``` - -## 1.E Realtime quick reference - -| Modality | Field | -| -------------------- | ------------------------------------------------- | -| Audio chunk | `realtimeInput.audio` (`audio/pcm;rate=16000`) | -| Video frame | `realtimeInput.video` (`image/jpeg`/`png`/`webp`) | -| Text | `realtimeInput.text` | -| Combined audio+video | `realtimeInput.mediaChunks[]` (**deprecated** — | -: : prefer the typed `audio` / `video` fields) : -| Mic-off signal | `realtimeInput.audioStreamEnd: true` | -| Manual turn boundary | `realtimeInput.activityStart` / `activityEnd` | -: : (auto-VAD disabled) : - ---- - -# Part 2 — Add-context input (`clientContent`) - -Discrete, **history-bearing** turns. Use this when you want the message to be -permanently part of the conversation context the model sees on subsequent -turns. Setting `turnComplete: true` triggers generation immediately and -**interrupts** any ongoing model output. - -> Do **not** use `clientContent` to reply to a `toolCall` — use `toolResponse`. - -## 2.A Add-context text - -### 2.A.1 Single user turn - -```json -{ - "clientContent": { - "turns": [ - { "role": "user", "parts": [{ "text": "What's the capital of France?" }] } - ], - "turnComplete": true - } -} -``` - -### 2.A.2 Multi-turn history (e.g. on resume) - -```json -{ - "clientContent": { - "turns": [ - { "role": "user", "parts": [{ "text": "Hi, my name is Sam." }] }, - { "role": "model", "parts": [{ "text": "Nice to meet you, Sam!" }] }, - { "role": "user", "parts": [{ "text": "What's my name?" }] } - ], - "turnComplete": true - } -} -``` - -### 2.A.3 Streamed turn (don't generate yet — more parts coming) - -```json -{ "clientContent": { "turns": [{ "role": "user", "parts": [{ "text": "Once upon" }] }] } } -{ "clientContent": { "turns": [{ "role": "user", "parts": [{ "text": " a time..." }] }], "turnComplete": true } } -``` - -## 2.B Add-context audio - -A pre-recorded clip delivered as a discrete history-bearing turn. - -### 2.B.1 Audio clip alone - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [{ - "inlineData": { - "mimeType": "audio/pcm;rate=16000", - "data": "" - } - }] - }], - "turnComplete": true - } -} -``` - -### 2.B.2 Audio clip with accompanying text prompt - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [ - { "text": "Transcribe this and summarize:" }, - { "inlineData": { "mimeType": "audio/pcm;rate=16000", "data": "..." } } - ] - }], - "turnComplete": true - } -} -``` - -## 2.C Add-context video / image - -### 2.C.1 Single image - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [ - { "text": "What's in this picture?" }, - { "inlineData": { "mimeType": "image/jpeg", "data": "" } } - ] - }], - "turnComplete": true - } -} -``` - -### 2.C.2 Multiple frames as a single turn (sampled clip) - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [ - { "text": "Here are 3 frames from a video. What's happening?" }, - { "inlineData": { "mimeType": "image/jpeg", "data": "" } }, - { "inlineData": { "mimeType": "image/jpeg", "data": "" } }, - { "inlineData": { "mimeType": "image/jpeg", "data": "" } } - ] - }], - "turnComplete": true - } -} -``` - -### 2.C.3 Image by URI (`fileData`) - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [ - { "text": "Describe this image." }, - { "fileData": { "mimeType": "image/jpeg", "fileUri": "gs://my-bucket/cat.jpg" } } - ] - }], - "turnComplete": true - } -} -``` - -## 2.D Combined add-context multimodal turn - -Text + image + audio in a single user turn: - -```json -{ - "clientContent": { - "turns": [{ - "role": "user", - "parts": [ - { "text": "Compare what I'm saying with what I'm showing:" }, - { "inlineData": { "mimeType": "image/jpeg", "data": "" } }, - { "inlineData": { "mimeType": "audio/pcm;rate=16000", "data": "" } } - ] - }], - "turnComplete": true - } -} -``` - -## 2.E Add-context quick reference - -| Modality | Field | Notes | -| -------------------- | ---------------------------- | ---------------------- | -| Text | `turns[].parts[].text` | One or more `text` | -: : : parts per turn. : -| Inline audio | `turns[].parts[].inlineData` | Full clip, base64. | -: : (`audio/pcm;rate=16000`) : : -| Inline image / video | `turns[].parts[].inlineData` | Multiple parts allowed | -: frame : (`image/jpeg`/`png`/`webp`) : for sampled clips. : -| Remote file | `turns[].parts[].fileData` | GCS URI | -: : (`fileUri`) : : -| Trigger generation | `turnComplete: true` | Omit to keep streaming | -: : : more parts. : -| History order | `turns[]` ordered oldest → | `role` is `"user"` or | -: : newest : `"model"`. : - ---- - -## 15. Common pitfalls - -- **Sending data before `setupComplete`** — server will close the connection. -- **Mixing `clientContent` and `realtimeInput`** for the same logical turn — - pick one mode. `clientContent` interrupts ongoing generation. -- **Ignoring `interrupted: true`** — leftover queued audio will play over the - user's next utterance. -- **Replying to `toolCall` with `clientContent`** — must be `toolResponse`. -- **Forgetting `FunctionResponse.id`** — request rejected. -- **Wrong audio format** — input must be 16 kHz PCM, output is 24 kHz PCM. -- **No reconnect handling** — sessions have a max duration; always honor - `goAway` and persist the latest `SessionResumptionUpdate.newHandle`. diff --git a/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.proto b/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.proto deleted file mode 100644 index 1fed44b..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-live-api/references/client_server_messages.proto +++ /dev/null @@ -1,380 +0,0 @@ -// Proto definitions for the Gemini Enterprise Live API wire protocol. -// -// Models the WebSocket frames described in client_server_messages.md. -// Both `ClientMessage` (client -> server) and `ServerMessage` (server -> -// client) are oneof envelopes; each WebSocket frame sets exactly one of the -// listed fields. -// -// Covers the Gemini Enterprise Agent Platform (`BidiGenerateContent*`) shapes. - -syntax = "proto3"; - -package google.ai.live.v1; - -import "google/protobuf/duration.proto"; -import "google/protobuf/struct.proto"; - -// ============================================================================= -// Top-level envelopes -// ============================================================================= - -// Client -> server frame. Exactly one field is set per frame. -message ClientMessage { - oneof message_type { - // First frame only. Configures the session. - BidiGenerateContentSetup setup = 1; - - // Turn-based, history-affecting input. Interrupts ongoing model output. - BidiGenerateContentClientContent client_content = 2; - - // Continuous, low-latency audio/video input. Not added to history. - BidiGenerateContentRealtimeInput realtime_input = 3; - - // Reply to a server-issued `tool_call`. - BidiGenerateContentToolResponse tool_response = 4; - } -} - -// Server -> client frame. Exactly one field is set per frame. -message ServerMessage { - oneof message_type { - // Sent once after `setup` is accepted. Gate further sends on this. - BidiGenerateContentSetupComplete setup_complete = 1; - - // Streamed model output (audio/text), turn lifecycle. - BidiGenerateContentServerContent server_content = 2; - - // Model is requesting tool execution. - BidiGenerateContentToolCall tool_call = 3; - - // Cancels previously-issued tool calls (e.g. on user interruption). - BidiGenerateContentToolCallCancellation tool_call_cancellation = 4; - - // Token / duration accounting. - UsageMetadata usage_metadata = 5; - - // Connection will be terminated soon. - GoAway go_away = 6; - - // Resume handle for reconnects. - SessionResumptionUpdate session_resumption_update = 7; - } -} - -// ============================================================================= -// Common content primitives -// ============================================================================= - -// A piece of binary data carried inline (audio chunk, image frame, etc.). -message Blob { - // IANA-style mime type, e.g. "audio/pcm;rate=16000", "image/jpeg". - string mime_type = 1; - - // Raw bytes (base64-encoded on the JSON wire). - bytes data = 2; -} - -// A reference to a remote file (e.g. GCS URI). -message FileData { - string mime_type = 1; - string file_uri = 2; -} - -// A function call requested by the model. -message FunctionCall { - // Opaque ID matching the eventual `FunctionResponse.id`. Optional. - string id = 1; - string name = 2; - google.protobuf.Struct args = 3; -} - -// Result of executing a function call. -message FunctionResponse { - // Must match the originating `FunctionCall.id`. Optional. - string id = 1; - string name = 2; - google.protobuf.Struct response = 3; -} - -// A part of a Content message; oneof of the supported part types. -// -// The markdown only references `text`, `inlineData`, and `fileData` parts; no -// other variants are described. -message Part { - oneof data { - string text = 1; - Blob inline_data = 2; - FileData file_data = 3; - } -} - -// A piece of conversation content with a role and ordered parts. -message Content { - // Typically "user" or "model". - string role = 1; - repeated Part parts = 2; -} - -// Tool declaration. The markdown only states that tools carry "function -// declarations and built-ins (Search, code execution)" without defining the -// internal schema, so the contents are left opaque here. -message Tool { - google.protobuf.Struct value = 1; -} - -// ============================================================================= -// Setup -// ============================================================================= - -message BidiGenerateContentSetup { - // Required. - // "projects/{p}/locations/{l}/publishers/google/models/{m}" - string model = 1; - - GenerationConfig generation_config = 2; - - // Text-only parts. - Content system_instruction = 3; - - repeated Tool tools = 4; - - SessionResumptionConfig session_resumption = 5; - - ContextWindowCompressionConfig context_window_compression = 6; - - RealtimeInputConfig realtime_input_config = 7; - - AudioTranscriptionConfig input_audio_transcription = 8; - - AudioTranscriptionConfig output_audio_transcription = 9; -} - -message GenerationConfig { - enum Modality { - MODALITY_UNSPECIFIED = 0; - TEXT = 1; - AUDIO = 2; - } - - enum MediaResolution { - MEDIA_RESOLUTION_UNSPECIFIED = 0; - MEDIA_RESOLUTION_LOW = 1; - MEDIA_RESOLUTION_MEDIUM = 2; - MEDIA_RESOLUTION_HIGH = 3; - } - - // Pick exactly one per session; default AUDIO. - repeated Modality response_modalities = 1; - - float temperature = 2; - float top_p = 3; - int32 top_k = 4; - int32 max_output_tokens = 5; - - // Only effective when response_modalities=[AUDIO]. - SpeechConfig speech_config = 6; - - MediaResolution media_resolution = 7; - - ThinkingConfig thinking_config = 8; -} - -message SpeechConfig { - // Single-speaker voice (Live API is single-speaker only). - VoiceConfig voice_config = 1; - - // BCP-47 (e.g. "en-US"). Output TTS language. - string language_code = 2; -} - -message VoiceConfig { - PrebuiltVoiceConfig prebuilt_voice_config = 1; -} - -message PrebuiltVoiceConfig { - // Case-sensitive prebuilt voice name (e.g. "Aoede", "Charon"). - string voice_name = 1; -} - -message ThinkingConfig { - // 0 disables thinking. Only honored on models that support it. - int32 thinking_budget = 1; -} - -message SessionResumptionConfig { - // Provide to resume; omit to start a new resumable session. - string handle = 1; - bool transparent = 2; -} - -message ContextWindowCompressionConfig { - message SlidingWindow { - int64 target_tokens = 1; - } - int64 trigger_tokens = 1; - SlidingWindow sliding_window = 2; -} - -message RealtimeInputConfig { - enum ActivityHandling { - ACTIVITY_HANDLING_UNSPECIFIED = 0; - START_OF_ACTIVITY_INTERRUPTS = 1; - NO_INTERRUPTION = 2; - } - - enum TurnCoverage { - TURN_COVERAGE_UNSPECIFIED = 0; - TURN_INCLUDES_ALL_INPUT = 1; - TURN_INCLUDES_ONLY_ACTIVITY = 2; - TURN_INCLUDES_AUDIO_ACTIVITY_AND_ALL_VIDEO = 3; - } - - // Unset -> server-side VAD enabled by default. - AutomaticActivityDetection automatic_activity_detection = 1; - ActivityHandling activity_handling = 2; - TurnCoverage turn_coverage = 3; -} - -message AutomaticActivityDetection { - enum StartSensitivity { - START_SENSITIVITY_UNSPECIFIED = 0; - START_SENSITIVITY_HIGH = 1; - START_SENSITIVITY_LOW = 2; - } - - enum EndSensitivity { - END_SENSITIVITY_UNSPECIFIED = 0; - END_SENSITIVITY_HIGH = 1; - END_SENSITIVITY_LOW = 2; - } - - // If true, client must send activity_start / activity_end manually. - bool disabled = 1; - StartSensitivity start_of_speech_sensitivity = 2; - EndSensitivity end_of_speech_sensitivity = 3; - int32 prefix_padding_ms = 4; - int32 silence_duration_ms = 5; -} - -// Empty configuration message for enabling input/output audio transcription. -message AudioTranscriptionConfig {} - -// ============================================================================= -// ClientContent / RealtimeInput / ToolResponse -// ============================================================================= - -message BidiGenerateContentClientContent { - // Conversation history + the latest user request. - repeated Content turns = 1; - - // If true, server starts generating immediately. - bool turn_complete = 2; -} - -message BidiGenerateContentRealtimeInput { - message ActivityStart {} - message ActivityEnd {} - - // Deprecated: combined audio/video chunks. Prefer the typed `audio` / - // `video` fields below. - repeated Blob media_chunks = 1 [deprecated = true]; - - // Typed realtime audio (PCM 16-bit, 16 kHz mono). - Blob audio = 2; - - // Typed realtime video frame (image/jpeg|png|webp). - Blob video = 3; - - // Realtime text input. - string text = 4; - - // Mic turned off; only valid with auto-VAD enabled. - bool audio_stream_end = 5; - - // Only when auto-VAD is disabled. - ActivityStart activity_start = 6; - ActivityEnd activity_end = 7; -} - -message BidiGenerateContentToolResponse { - // Match `FunctionCall.id` from the server. `id` is optional. - repeated FunctionResponse function_responses = 1; -} - -// ============================================================================= -// Server messages -// ============================================================================= - -message BidiGenerateContentSetupComplete { - // Server-assigned session identifier. - string session_id = 1; -} - -message BidiGenerateContentServerContent { - // Streamed model output parts (text and/or `inline_data` audio @ 24 kHz). - Content model_turn = 1; - - // Model has finished generating; playback may still be flushing. - bool generation_complete = 2; - - // Logical end of turn. - bool turn_complete = 3; - - // Generation was interrupted by client input -- drop queued audio playback. - bool interrupted = 4; - - // When grounding (e.g. Google Search) is used. Modelled as Struct because the - // full grounding schema is out of scope for this wire-protocol proto. - google.protobuf.Struct grounding_metadata = 5; - - // Transcription of the user's spoken input. Requires - // `input_audio_transcription` set in `setup`. - Transcription input_transcription = 6; - - // Transcription of the model's spoken output. Requires - // `output_audio_transcription` set in `setup`. - Transcription output_transcription = 7; -} - -message Transcription { - string text = 1; - bool finished = 2; -} - -message BidiGenerateContentToolCall { - // Each has `id`, `name`, `args`. Reply with matching FunctionResponse.id. - repeated FunctionCall function_calls = 1; -} - -message BidiGenerateContentToolCallCancellation { - // IDs of previously-issued tool calls to cancel. Stop the work; do not - // send `tool_response` for these. - repeated string ids = 1; -} - -// ============================================================================= -// Connection lifecycle / accounting -// ============================================================================= - -message GoAway { - // Time until the connection terminates with ABORTED. - google.protobuf.Duration time_left = 1; -} - -message SessionResumptionUpdate { - // Handle to resume; empty if `resumable=false`. - string new_handle = 1; - bool resumable = 2; - - // Set only when SessionResumptionConfig.transparent=true. - int64 last_consumed_client_message_index = 3; -} - -// Token / duration accounting (cached-content style). -message UsageMetadata { - int32 total_token_count = 1; - int32 text_count = 2; - int32 image_count = 3; - int32 video_duration_seconds = 4; - int32 audio_duration_seconds = 5; -} diff --git a/plugins/cloud/gemini-api/skills/gemini-live-api/references/session_manager.md b/plugins/cloud/gemini-api/skills/gemini-live-api/references/session_manager.md deleted file mode 100644 index 2331cc4..0000000 --- a/plugins/cloud/gemini-api/skills/gemini-live-api/references/session_manager.md +++ /dev/null @@ -1,69 +0,0 @@ -# Live API Session Manager - -This document guides the implementation of a `SessionManager` class for the Gemini Live API over WebSockets. It ensures robust handling of disconnections and session resumption. - -## Core Responsibilities - -1. **Connection Management**: Maintain a continuous WebSocket connection to the specified endpoint until explicitly stopped. -2. **Bidirectional Communication**: Handle sending and receiving messages concurrently (use separate coroutines or threads). -3. **Session Resumption**: Automatically reconnect and restore state when disconnections occur. - -## Protocol Details - -- **Proto File**: `client_server_messages.proto` -- **Server Message**: `BidiGenerateContentServerMessage` -- **Client Message**: `BidiGenerateContentClientMessage` - -## Session Resumption Logic - -To support transparent session resumption, the manager must implement the following logic: - -### 1. Enable Transparent Session Resumption -Modify the `session_resumption` field in the `setup` message (type `BidiGenerateContentSetup`) of the initial `BidiGenerateContentClientMessage`. Always set the `transparent` field to `true` in the `SessionResumptionConfig`. This enables the model to return `last_consumed_client_message_index` in `SessionResumptionUpdate` messages, indicating when to update the buffer. - -- **Proto Reference**: `client_server_messages.proto` - - `BidiGenerateContentSetup.session_resumption` (type `SessionResumptionConfig`) - - `SessionResumptionConfig.transparent` (bool) - -### 2. Handle Session Handle Updates -Listen for `session_resumption_update` messages in the stream from the server. - -- If `resumable` is true and a `new_handle` is provided, store it. -- This handle is required for reconnecting to the same session. - -### 3. Message Buffering and Pruning -Maintain a buffer of sent messages to replay if a disconnection occurs. - -- **Indexing**: The user-managed message index MUST begin at **1**. The server reserves index **0** for the initial configuration. -- Increment the index by 1 for each subsequent message sent. -- **Pruning**: Use the `last_consumed_client_message_index` from the server's `session_resumption_update` to remove acknowledged messages from the buffer. -- **Reset on Resumption**: Upon reconnection, ensure the index is reset to **1** for the first message transmitted via the new connection. - -### 4. Handling Disconnections -Catch disconnections and initiate resumption: - -- **Proactive Reconnection**: If the server sends a `go_away` signal, proactively reconnect using the latest handle. -- **Error Handling**: Catch WebSocket errors (specifically error codes **1000** or **1006**) in both sending and receiving loops. Trigger the reconnection process on these errors. -- **Unexpected Errors**: For other unexpected errors, the session manager - should be stopped and raise that error immediately. Subsequent user - send/receive function calls should raise exceptions with stop reasons. -- **Reconnection Errors**: Exceptions can also occur during the reconnection process itself. These must be handled correctly, for example, by implementing retries with exponential backoff or failing gracefully if the connection cannot be re-established. - -### 5. Reconnection with Message Replay -When a disconnection occurs: - -1. Establish a new websocket connection and pass the stored session handle. -2. Resend all messages remaining in the buffer BEFORE sending / receiving any - other messages. Do not modify the buffer until receiving the resumption - handle update, since there could be connection failure during this time and - a new retry will be needed. -3. The first message sent from the buffer on the new connection MUST be marked with index **1**. THIS IS VERY IMPORTANT. - -## Gotchas - -- **Index 0**: Never use index 0 for user messages; it is reserved for the - first config message. -- **Concurrent Loops**: Ensure the receive loop can detect disconnections and - trigger reconnection even if the send loop is idle, and vice versa. -- **Handle Expiry**: Session handles may have an expiry; handle failures to - reconnect with an expired handle by starting a fresh session if necessary.