mirror of
https://github.com/hairyf/skills.git
synced 2026-09-14 14:57:15 +08:00
feat(vision, imagine): add support for vision sessions, adjust imagine logic, and update documentation
This commit is contained in:
hairyf
@@ -12,4 +12,5 @@ node_modules
|
||||
temp
|
||||
__pycache__
|
||||
.imagine
|
||||
.vision
|
||||
.deps
|
||||
|
||||
@@ -12,8 +12,8 @@ node scripts/imagine.js "<prompt>" -e <image.png> -o <edited.png> # edit
|
||||
Follow this priority order unless the user specifies otherwise:
|
||||
|
||||
- **Images** — GPT Image 2 (`gpt-image-2`) → Nano Banana (`gemini-2.5-flash-image`) → SiliconFlow Qwen-Image (`Qwen/Qwen-Image`) → OpenAI GPT Image (`gpt-image-1`).
|
||||
- **Stickers / icons / logos** — Ideogram 3.0 (transparent) → GPT Image (`--background transparent -f png`) → FLUX.2 pro. For Ideogram, see `references/features-sticker.md`.
|
||||
- **Seamless textures** — Z-Image Turbo Tiling (fal.ai) → FLUX.2 pro → Nano Banana / GPT Image 2 with a "seamless tileable" prompt. See `references/features-texture.md`.
|
||||
- **Stickers / icons / logos** — Ideogram 3.0 (transparent) → GPT Image (`--background transparent -f png`) → FLUX.2 pro.
|
||||
- **Seamless textures** — Z-Image Turbo Tiling (fal.ai) → FLUX.2 pro → Nano Banana / GPT Image 2 with a "seamless tileable" prompt.
|
||||
|
||||
## Trigger scenarios
|
||||
|
||||
|
||||
+26
-57
@@ -3,82 +3,51 @@ name: imagine
|
||||
description: AI image, sticker, icon, and seamless-texture generation for agents. Use when the user asks to create, generate, edit, or iterate on images, photos, illustrations, posters, stickers, icons, logos, sprites, textures, or mockups. Provides a zero-dependency Node.js script that calls GPT Image 2, Nano Banana (Gemini), SiliconFlow Qwen-Image, or other image models and saves the result to disk.
|
||||
metadata:
|
||||
author: Hairy
|
||||
version: "2026.8.7"
|
||||
version: "2026.8.11"
|
||||
---
|
||||
|
||||
# Image Gen
|
||||
|
||||
Generate images, stickers, and seamless textures through a single zero-dependency Node.js script backed by multiple providers. The script speaks to each provider's API directly — no SDK or npm dependencies — and saves the generated image to disk.
|
||||
|
||||
## Model selection
|
||||
|
||||
Picked by the user for image generation, in priority order:
|
||||
|
||||
1. **GPT Image 2** (`gpt-image-2`, OpenAI) — best overall quality, 4K, excellent CJK/multilingual text rendering, up to 16 input images for edits.
|
||||
2. **Nano Banana** (`gemini-2.5-flash-image`, Google) — fast, free tier, native conversational image editing; newer `gemini-3.1-flash-image` / `gemini-3-pro-image` variants.
|
||||
3. **SiliconFlow Qwen-Image** (`Qwen/Qwen-Image`, SiliconFlow) — cheapest OpenAI-compatible option (~$0.042/image), strong Chinese text rendering, dedicated edit model `Qwen/Qwen-Image-Edit-2509`.
|
||||
4. **OpenAI GPT Image** (`gpt-image-1`, OpenAI) — previous-generation GPT Image model; still the most widely supported via third-party aggregators.
|
||||
|
||||
For stickers, icons, and textures use the dedicated models below; the general image models still work but the specialists produce better asset-ready output.
|
||||
|
||||
| Asset | Recommended models (priority order) | Notes |
|
||||
|-------|-------------------------------------|-------|
|
||||
| Images / photos / posters | GPT Image 2 → Nano Banana → SiliconFlow Qwen-Image → OpenAI GPT Image | Follow the user's order above |
|
||||
| Stickers / icons / logos | Ideogram 3.0 (transparent) → GPT Image (transparent PNG) → FLUX.2 pro | Transparent background built in |
|
||||
| Seamless textures / tiles | Z-Image Turbo Tiling → FLUX.2 pro → Nano Banana / GPT Image 2 | Z-Image Tiling is purpose-built for tileable textures |
|
||||
Generate images through `scripts/imagine.js`, a zero-dependency Node.js CLI (Node 18+, no npm install) backed by multiple providers. It saves images to disk and prints the absolute paths.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# Generate an image (defaults to gpt-image-2)
|
||||
# Generate an image
|
||||
node scripts/imagine.js "a retro synthwave poster, neon grid, 80s style" -o poster.png
|
||||
|
||||
# Sticker with transparent background
|
||||
node scripts/imagine.js "die-cut sticker of a cute sloth, white border" -m gpt-image-1.5 --background transparent -f png -o sloth.png
|
||||
|
||||
# Seamless texture via SiliconFlow (fast, cheap)
|
||||
node scripts/imagine.js "seamless tileable brick wall texture, top-down, uniform lighting" -m Qwen/Qwen-Image -s 1328x1328 -o brick.png
|
||||
|
||||
# Nano Banana with a landscape aspect ratio
|
||||
node scripts/imagine.js "a serene mountain lake at sunset" -m gemini-2.5-flash-image --aspect 16:9 -o lake.png
|
||||
|
||||
# Edit an existing image
|
||||
node scripts/imagine.js "add a red balloon in the sky" -e photo.png -o edited.png
|
||||
|
||||
# Two-step design in one conversation (design -> three views), e.g. Minecraft model design
|
||||
node scripts/imagine.js "生成一格Minecraft蜗牛，蜗牛壳是一个立方体，不要堆叠。蜗牛身体结构简单，有触角。蜗牛足够将整个身体缩回到蜗牛壳内部。请绘制 将宽高比设为 1:1" --session snail -o design.png
|
||||
node scripts/imagine.js "非常好，画出它的三视图" --session snail -o views.png
|
||||
# Multi-step session: keep context across calls (design first, then its three views)
|
||||
node scripts/imagine.js "a cute robot mascot, front view, flat vector style" --session mascot -o design.png
|
||||
node scripts/imagine.js "draw its three views (front, side, back) from the same design" --session mascot -o views.png
|
||||
```
|
||||
|
||||
## Setup
|
||||
## Model selection
|
||||
|
||||
Requires at least one API key (OpenAI, SiliconFlow, or Google AI Studio). On first run, follow [core-setup](references/core-setup.md) — including the required AGENTS.md merge. Once configured, the agent can generate images on demand.
|
||||
Priority for general images: **GPT Image 2 (`gpt-image-2`) → Nano Banana (`gemini-2.5-flash-image`) → SiliconFlow Qwen-Image (`Qwen/Qwen-Image`) → OpenAI GPT Image (`gpt-image-1`)**. When a provider has no API key, fall back to the next configured one.
|
||||
|
||||
## References
|
||||
| Asset | Recommended models (priority order) |
|
||||
|-------|-------------------------------------|
|
||||
| Images / posters | `gpt-image-2` → `gemini-2.5-flash-image` → `Qwen/Qwen-Image` → `gpt-image-1` |
|
||||
| Stickers / icons / logos | Ideogram 3.0 (transparent) → `gpt-image-1.5` / `gpt-image-2` with `--background transparent -f png` → FLUX.2 pro |
|
||||
| Seamless textures | Z-Image Turbo Tiling (fal.ai) → FLUX.2 pro → any strong model with a strict tiling prompt |
|
||||
|
||||
### Core
|
||||
Specialist models outside the script's providers (Ideogram, fal tiling) are called through their own APIs; `imagine.js` covers OpenAI, SiliconFlow, Gemini, and OpenAI-compatible relays.
|
||||
|
||||
| Topic | Description | Reference |
|
||||
|-------|-------------|-----------|
|
||||
| Usage | CLI arguments, providers, output handling, exit codes | [core-usage](references/core-usage.md) |
|
||||
| Setup | API keys, `.env` configuration, required AGENTS.md merge, verification | [core-setup](references/core-setup.md) |
|
||||
| How it works | Internals of imagine.js for patching and debugging | [core-script](references/core-script.md) |
|
||||
## Key workflows
|
||||
|
||||
### Models
|
||||
- **Transparent assets**: `--background transparent -f png` (never `jpeg`); phrase stickers as "die-cut sticker, thick white border, bold clean shapes".
|
||||
- **Seamless textures**: include "seamless, tileable, top-down, flat uniform lighting, no shadows, edges wrap perfectly".
|
||||
- **Text in images**: GPT Image 2 is best for CJK/multilingual text; keep text short and quoted.
|
||||
- **Aspect ratio**: Gemini via `--aspect 16:9`; OpenAI/SiliconFlow via `-s WxH` (Qwen native sizes like `1328x1328`, `1664x928`).
|
||||
- **Iterate cheaply**: draft with `Qwen/Qwen-Image` / Nano Banana or `-q low`, render finals on `gpt-image-2` with `-q high`; `--seed` for reproducible output.
|
||||
|
||||
| Topic | Description | Reference |
|
||||
|-------|-------------|-----------|
|
||||
| Image models | GPT Image 2, Nano Banana, SiliconFlow Qwen-Image, GPT Image — capabilities & parameters | [models-image](references/models-image.md) |
|
||||
## References — load on demand
|
||||
|
||||
### Features
|
||||
|
||||
| Topic | Description | Reference |
|
||||
|-------|-------------|-----------|
|
||||
| Stickers & icons | Die-cut stickers, transparent backgrounds, Ideogram / GPT Image / FLUX.2 | [features-sticker](references/features-sticker.md) |
|
||||
| Seamless textures | Tileable texture generation, Z-Image Turbo Tiling / FLUX.2 / prompt recipes | [features-texture](references/features-texture.md) |
|
||||
|
||||
### Best Practices
|
||||
|
||||
| Topic | Description | Reference |
|
||||
|-------|-------------|-----------|
|
||||
| Prompt recipes | Asset-ready prompts: style keywords, transparency, tiling, aspect ratios | [best-practices-prompts](references/best-practices-prompts.md) |
|
||||
| When | Reference |
|
||||
|------|-----------|
|
||||
| Full CLI arguments, session details, exit codes | [core-usage](references/core-usage.md) |
|
||||
| First run with no API key configured (env keys, AGENTS.md merge) | [core-setup](references/core-setup.md) |
|
||||
| Debugging the pipeline or editing the scripts | [core-script](references/core-script.md) |
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
---
|
||||
name: best-practices-prompts
|
||||
description: Asset-ready prompt recipes and generation best practices — style keywords, transparency, tiling, aspect ratios, and iteration workflow.
|
||||
---
|
||||
|
||||
# Prompt Recipes
|
||||
|
||||
Small prompt choices decide whether output is asset-ready or needs manual cleanup. These patterns apply across all supported models.
|
||||
|
||||
## Structure a generation prompt
|
||||
|
||||
`{subject} + {style} + {composition/view} + {lighting} + {background} + {technical notes}`
|
||||
|
||||
Example:
|
||||
> "a cozy reading nook with a cat, cozy flat illustration, soft warm light, plants around, pastel palette, high detail, 4k"
|
||||
|
||||
## Style keywords
|
||||
|
||||
| Desired look | Add to prompt |
|
||||
|--------------|---------------|
|
||||
| Flat vector / illustration | "flat vector illustration, bold clean shapes, limited color palette" |
|
||||
| Pixel art | "pixel art, 16-bit style, crisp pixels, limited palette" |
|
||||
| Watercolor | "watercolor illustration, soft edges, paper texture" |
|
||||
| Photorealistic | "photorealistic, natural lighting, shallow depth of field, shot on {camera}" |
|
||||
| 3D render | "3D render, soft studio lighting, octane render, clay material" |
|
||||
| Retro / poster | "retro synthwave poster, neon grid, grainy print texture, 80s style" |
|
||||
|
||||
## Transparency
|
||||
|
||||
- GPT Image: `--background transparent -f png` (or `webp`). Never request transparency with `jpeg`.
|
||||
- Ideogram: `background_transparent: true`.
|
||||
- Sticker phrasing: "die-cut sticker", "isolated on transparent background", "no background".
|
||||
- If a model ignores transparency, switch to Ideogram or `gpt-image-1.5`.
|
||||
|
||||
## Tiling / seamless
|
||||
|
||||
- Always include: "seamless", "tileable", "top-down", "flat uniform lighting", "no shadows", "edges wrap perfectly".
|
||||
- Avoid scene-like prompts (perspective, depth, vignettes) for textures.
|
||||
- Prefer a dedicated tiling model (Z-Image Turbo Tiling) over prompt-only approaches.
|
||||
|
||||
## Aspect ratios and sizes
|
||||
|
||||
- **OpenAI** — `1024x1024`, `1536x1024` (landscape), `1024x1536` (portrait); `gpt-image-2` accepts arbitrary `WIDTHxHEIGHT` (both divisible by 16, ratio within 1:3–3:1).
|
||||
- **SiliconFlow Qwen-Image** — use the model's native sizes: `1328x1328` (1:1), `1664x928` (16:9), `928x1664` (9:16), etc. Arbitrary sizes degrade quality.
|
||||
- **Gemini** — use `--aspect` (`1:1`, `16:9`, `9:16`, `4:3`, `3:4`), not pixel sizes.
|
||||
|
||||
## Text in images
|
||||
|
||||
- GPT Image 2 is best-in-class for CJK and multilingual text (posters, UI mockups, infographics).
|
||||
- Qwen-Image is strong at Chinese text and cheap for it.
|
||||
- Keep text short and quoted, e.g. `a poster that says "你好，世界"`.
|
||||
- For precise typography (logos, long copy), use Ideogram.
|
||||
|
||||
## Editing workflow
|
||||
|
||||
1. Generate a base image.
|
||||
2. Edit iteratively with `-e` and short deltas ("make the sky purple", "add a hat").
|
||||
3. Use reference images for consistency (up to 16 for GPT Image 2, up to 8 for FLUX.2).
|
||||
4. For inpainting, provide a mask (white = regenerate, black = keep) on OpenAI edits.
|
||||
|
||||
## Cost and iteration
|
||||
|
||||
- Draft with cheap/fast models (`Qwen-Image`, Nano Banana, `quality: low`), then render final assets on `gpt-image-2` (`quality: high`).
|
||||
- Use `--seed` to reproduce or iterate on a specific result (SiliconFlow / Gemini).
|
||||
- Generate variations in batches with `-n`/`--num-images` where supported, then pick the best.
|
||||
|
||||
## Key points
|
||||
|
||||
- Always state the style explicitly — models default to bland photorealism otherwise.
|
||||
- State the background explicitly ("transparent", "solid white", "blurred studio backdrop").
|
||||
- For assets, prefer dedicated specialists: Ideogram (stickers/logos), Z-Image Turbo Tiling (textures).
|
||||
- Fall back down the model priority order when a provider lacks a key.
|
||||
@@ -1,30 +1,34 @@
|
||||
---
|
||||
name: core-script
|
||||
description: How imagine.js works internally — provider routing, API payloads, response handling, and configuration variables.
|
||||
description: How imagine.js works internally — module layout, pipeline, session state, provider channels, and configuration.
|
||||
---
|
||||
|
||||
# How imagine.js works
|
||||
|
||||
A zero-dependency ESM script that generates images through multiple provider channels. The entry point is `scripts/imagine.js`; the implementation is split into small modules under `scripts/lib/`.
|
||||
A zero-dependency ESM script that generates images through multiple provider channels. The CLI entry is thin; provider logic and HTTP live in `scripts/lib/*.js` so each concern can be patched independently.
|
||||
|
||||
## Module layout
|
||||
## File layout
|
||||
|
||||
| File | Responsibility |
|
||||
|------|----------------|
|
||||
| `scripts/imagine.js` | Entry point — CLI parsing, provider/key checks, main flow, help |
|
||||
| `scripts/lib/env.js` | Zero-dependency `.env` loader (cwd + script dir) |
|
||||
| `scripts/lib/config.js` | Reads env into constants (keys, base URLs, proxy, model defaults, MIME maps) |
|
||||
| `scripts/lib/http.js` | Unified HTTP — native fetch or zero-dependency CONNECT proxy tunnel, JSON POST, multipart, downloads |
|
||||
| `scripts/lib/providers.js` | Provider routing + request builders (OpenAI, SiliconFlow, Gemini, relay) |
|
||||
| `scripts/lib/output.js` | Response normalization (`extractImages`) and output path resolution |
|
||||
```
|
||||
scripts/
|
||||
├── imagine.js # CLI entry: arg parsing, provider/key checks, main flow, help
|
||||
└── lib/
|
||||
├── env.js # Zero-dependency .env loader (cwd + script dir)
|
||||
├── config.js # Reads env into constants (keys, base URLs, proxy, model defaults, MIME maps)
|
||||
├── http.js # Unified HTTP — native fetch or zero-dependency CONNECT proxy tunnel, JSON POST, multipart, downloads
|
||||
├── providers.js # Provider routing + request builders (OpenAI, SiliconFlow, Gemini, relay)
|
||||
├── output.js # Response normalization (extractImages) and output path resolution
|
||||
└── session.js # Conversation state (--session): load/save/clear, latestImage, MIME sniffing
|
||||
```
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. **Parse args** — prompt, model, size, quality, background, format, edit/mask, seed, steps, aspect.
|
||||
1. **Parse args** — prompt, model, size, quality, background, format, edit/mask, seed, steps, aspect, session.
|
||||
2. **Route provider** — inferred from the model id (`gemini-*` → gemini; `Qwen/*`, `Kolors/*`, `Z-Image*`, `FLUX*` → siliconflow; `gpt-image-*` → openai), overridable with `--provider`.
|
||||
3. **Call the API** — one of three request builders below.
|
||||
4. **Extract images** — normalize provider responses into `{ buffer, ext }` items.
|
||||
5. **Save** — write each image to disk (creating parent directories), print absolute paths.
|
||||
3. **Load session** — when `--session` is passed, load prior state (or start fresh) and attach it as `opts.sessionHistory`; a provider/model mismatch resets the session with a warning.
|
||||
4. **Call the API** — one of three request builders; the session history is replayed per provider (see below).
|
||||
5. **Extract images** — normalize provider responses into `{ buffer, ext }` items.
|
||||
6. **Save & persist** — write each image to disk (creating parent directories), print absolute paths, then append the user turn + model turn (with image paths) to the session state.
|
||||
|
||||
## Session continuity (`--session`)
|
||||
|
||||
@@ -35,38 +39,34 @@ A zero-dependency ESM script that generates images through multiple provider cha
|
||||
"provider": "gemini",
|
||||
"model": "gemini-2.5-flash-image",
|
||||
"turns": [
|
||||
{ "role": "user", "text": "生成一格Minecraft蜗牛…", "images": [] },
|
||||
{ "role": "user", "text": "a cute robot mascot, front view…", "images": [] },
|
||||
{ "role": "model", "text": "", "images": ["/abs/path/design.png"] }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `loadSession` / `saveSession` / `clearSession` read, write, and delete the state file; `latestImage` returns the most recent generated image path; `mimeOf` guesses MIME types.
|
||||
- On a session call the provider receives `opts.sessionHistory` (the loaded state). After saving images, `imagine.js` appends the user turn and the model turn with the new absolute image paths, then persists the state.
|
||||
- `--session-reset` deletes the state file first; a provider/model mismatch between calls also resets the session with a warning.
|
||||
- Per-provider continuity: Gemini replays all turns as `contents` (native conversation); OpenAI/relay and SiliconFlow pass the latest image as the edit input (`image` field). Session calls without `--session` are untouched — stateless single generation remains the default.
|
||||
- `loadSession` / `saveSession` read and write the state file; `latestImage` returns the most recent generated image path; `mimeOf` guesses MIME types.
|
||||
- On a session call the provider receives `opts.sessionHistory`. After saving images, `imagine.js` appends the user turn and the model turn with the new absolute image paths, then persists the state.
|
||||
- A provider/model mismatch between calls resets the session with a warning.
|
||||
- Per-provider replay: Gemini replays all turns as `contents` (native conversation); OpenAI/relay and SiliconFlow pass the latest image as the edit input (`image` field). Without `--session`, behavior is stateless.
|
||||
|
||||
## Provider channels
|
||||
|
||||
### OpenAI (`/v1/images/generations`, `/v1/images/edits`)
|
||||
|
||||
- Create: JSON body `{ model, prompt, size?, quality?, background?, output_format?, n? }`.
|
||||
- Edit: `multipart/form-data` with `image` (+ optional `mask`), `prompt`, and the same option fields.
|
||||
- Response: `data[].b64_json` or `data[].url`.
|
||||
Create sends `{ model, prompt, size?, quality?, background?, output_format?, n? }`; edits are multipart with `image` (+ optional `mask`) and the same option fields. Responses are `data[].b64_json` or `data[].url`.
|
||||
|
||||
### SiliconFlow (`/v1/images/generations`, OpenAI-compatible)
|
||||
|
||||
- Body: `{ model, prompt, image_size, num_inference_steps, seed?, image? }`.
|
||||
- `batch_size` is only sent for Kolors models (the docs mark it Kolors-only).
|
||||
- Editing uses `Qwen/Qwen-Image-Edit-2509` with the `image` field as a base64 data URL.
|
||||
- Response: `images[].url` — **expires in one hour**, downloaded immediately.
|
||||
Body: `{ model, prompt, image_size, num_inference_steps, seed?, image? }`; `batch_size` is sent only for Kolors models. Editing uses `Qwen/Qwen-Image-Edit-2509` with the `image` field as a base64 data URL. Responses are `images[].url` — **expire in one hour**, downloaded immediately.
|
||||
|
||||
### Gemini / Nano Banana (`:generateContent`)
|
||||
|
||||
- Endpoint: `https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key=...`.
|
||||
- Body: `{ contents: [{ parts }], generationConfig: { response_modalities: ["IMAGE"], imageConfig: { aspectRatio? }, seed? } }`.
|
||||
- Editing passes the input image as an `inline_data` part before the text part.
|
||||
- Response: `candidates[].content.parts[].inlineData.data` (base64) or `fileData.fileUri` (downloaded with the API key header).
|
||||
Body: `{ contents: [{ parts }], generationConfig: { response_modalities: ["IMAGE"], imageConfig: { aspectRatio? }, seed? } }`. Editing passes the input image as an `inline_data` part before the text part. Responses are `candidates[].content.parts[].inlineData.data` (base64) or `fileData.fileUri` (downloaded with the API key header).
|
||||
|
||||
### Relay channel
|
||||
|
||||
Relays expose Nano Banana (and other models) through OpenAI-compatible `/v1/images/generations` with prefixed model names such as `google/gemini-2.5-flash-image` or `openai/gpt-image-2`. Prefixed names auto-route to the relay; `--provider relay` forces it.
|
||||
|
||||
## Configuration (env vars or .env)
|
||||
|
||||
@@ -81,21 +81,12 @@ A zero-dependency ESM script that generates images through multiple provider cha
|
||||
| `RELAY_API_KEY` | — | API key for the relay |
|
||||
| `HTTPS_PROXY` | — | Optional HTTP proxy (CONNECT tunnel) for blocked relay domains |
|
||||
|
||||
### Relay channel
|
||||
|
||||
Relays expose Nano Banana (and other models) through OpenAI-compatible `/v1/images/generations` with prefixed model names such as `google/gemini-2.5-flash-image` or `openai/gpt-image-2`. Requests go to `RELAY_BASE_URL` with `RELAY_API_KEY`. Prefixed model names auto-route to the relay; `--provider relay` forces it.
|
||||
|
||||
## Default models
|
||||
|
||||
| Provider | Default model |
|
||||
|----------|---------------|
|
||||
| openai | `gpt-image-2` |
|
||||
| siliconflow | `Qwen/Qwen-Image` |
|
||||
| gemini | `gemini-2.5-flash-image` |
|
||||
| relay | `google/gemini-2.5-flash-image` |
|
||||
`.env` files are loaded from the current working directory and the script's directory, and never override already-set environment variables.
|
||||
|
||||
## Key points
|
||||
|
||||
- Uses native `fetch` (or a built-in CONNECT tunnel when `HTTPS_PROXY` is set) — no npm dependencies; requires Node 18+.
|
||||
- Not bound to a single vendor; the same CLI covers all providers.
|
||||
- Multi-image responses (`data[]`, `images[]`, multiple Gemini candidates) are all saved.
|
||||
- Provider/model changes reset a session with a warning — keep the same model across a session.
|
||||
- SiliconFlow image URLs expire after one hour — downloads happen immediately.
|
||||
|
||||
@@ -1,81 +1,52 @@
|
||||
---
|
||||
name: core-setup
|
||||
description: First-time setup — choosing providers, obtaining API keys, configuring .env, and integrating into a project.
|
||||
description: First-time setup — choosing a provider, API keys, .env configuration, AGENTS.md merge, and verification.
|
||||
---
|
||||
|
||||
# Setup
|
||||
|
||||
Follow these steps when the skill is not yet configured (no API key detected).
|
||||
Follow these steps when no API key is configured.
|
||||
|
||||
## 1. Choose a provider
|
||||
|
||||
Ask the user which provider(s) they have keys for. Each maps to one `.env` variable:
|
||||
Ask which provider the user has a key for; each maps to one env var:
|
||||
|
||||
| Provider | Env var | Where to get the key |
|
||||
|----------|---------|----------------------|
|
||||
| Provider | Env var | Key source |
|
||||
|----------|---------|------------|
|
||||
| OpenAI (`gpt-image-2`, `gpt-image-1.5`, `gpt-image-1`) | `OPENAI_API_KEY` | https://platform.openai.com/api-keys |
|
||||
| SiliconFlow (`Qwen/Qwen-Image`, `Qwen/Qwen-Image-Edit-2509`, `Tongyi-MAI/Z-Image-Turbo`) | `SILICONFLOW_API_KEY` | https://cloud.siliconflow.cn/account/ak |
|
||||
| Google Gemini / Nano Banana (`gemini-2.5-flash-image`, `gemini-3.1-flash-image`, `gemini-3-pro-image`) | `GEMINI_API_KEY` | https://aistudio.google.com/apikey (free tier) |
|
||||
| SiliconFlow (`Qwen/Qwen-Image`, `Qwen/Qwen-Image-Edit-2509`) | `SILICONFLOW_API_KEY` | https://cloud.siliconflow.cn/account/ak |
|
||||
| Google Gemini / Nano Banana (`gemini-2.5-flash-image` etc.) | `GEMINI_API_KEY` | https://aistudio.google.com/apikey |
|
||||
|
||||
The agent should fall back to the next model in the priority order when a provider has no key configured. For example, if only `SILICONFLOW_API_KEY` is set, use `Qwen/Qwen-Image` for images even though GPT Image 2 is preferred.
|
||||
Fall back to the next model in the priority order when a provider has no key.
|
||||
|
||||
## 2. Configure .env
|
||||
|
||||
Write the key(s) to `scripts/.env` (next to imagine.js):
|
||||
Write the key(s) to `scripts/.env` (also loaded from cwd; existing env vars always win):
|
||||
|
||||
```bash
|
||||
OPENAI_API_KEY=sk-xxx
|
||||
SILICONFLOW_API_KEY=sk-xxx
|
||||
GEMINI_API_KEY=xxx
|
||||
RELAY_BASE_URL=https://api.ofox.ai/v1
|
||||
RELAY_API_KEY=sk-xxx
|
||||
|
||||
# Optional overrides
|
||||
# OPENAI_BASE_URL=https://api.openai.com/v1
|
||||
# SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1
|
||||
# HTTPS_PROXY=http://127.0.0.1:7890 # required when relay domains are blocked (e.g. in China)
|
||||
# Nano Banana via an OpenAI-compatible Chinese relay (no Google billing)
|
||||
# RELAY_BASE_URL=https://api.ofox.ai/v1
|
||||
# RELAY_API_KEY=sk-xxx
|
||||
# HTTPS_PROXY=http://127.0.0.1:7890 # required when relay domains are blocked
|
||||
```
|
||||
|
||||
`.env` is also loaded from the current working directory, and never overrides already-set environment variables.
|
||||
Prefixed model names (`google/gemini-2.5-flash-image`) auto-route to the relay; `--provider relay` forces it.
|
||||
|
||||
## 3. Merge AGENTS.md into the project (required)
|
||||
## 3. Merge AGENTS.md into the host project (required)
|
||||
|
||||
To make the project agent generate images automatically, **merge the content of `AGENTS.md` (in this skill directory) into the host project's `AGENTS.md` / `CLAUDE.md`**. Do not skip this step.
|
||||
|
||||
1. Read the skill's `AGENTS.md` (the "Image Generation Capability" section).
|
||||
2. If the project has an `AGENTS.md` / `CLAUDE.md`, check whether the section is already present — if so, skip the merge (idempotent).
|
||||
3. Otherwise, append the section to the existing file, or create a new `AGENTS.md` / `CLAUDE.md` at the project root.
|
||||
4. Tell the user the configuration is done — from now on they can just ask for images and the agent generates them automatically.
|
||||
Append this skill's `AGENTS.md` (the "Image Generation Capability" section) to the host project's `AGENTS.md` / `CLAUDE.md` if not already present. This is what lets the agent generate images automatically on request.
|
||||
|
||||
## 4. Verify
|
||||
|
||||
Run the script with the cheapest configured provider to confirm the key works before reporting success:
|
||||
|
||||
```bash
|
||||
node scripts/imagine.js "a small red square with a white border" -m Qwen/Qwen-Image -o verify.png
|
||||
```
|
||||
|
||||
## Key points
|
||||
|
||||
- The user's image model priority: GPT Image 2 → Nano Banana → SiliconFlow Qwen-Image → OpenAI GPT Image.
|
||||
- Only keys that are present are usable; fall back to a configured provider rather than failing.
|
||||
- The AGENTS.md merge is what enables automatic generation in the host project — the `.env` config alone is not enough.
|
||||
|
||||
## Using Nano Banana through a Chinese relay (no Google billing)
|
||||
|
||||
If native Gemini is unavailable (no Google payment method), use an OpenAI-compatible relay that sells Nano Banana access and accepts Alipay/WeChat:
|
||||
|
||||
```bash
|
||||
RELAY_BASE_URL=https://api.ofox.ai/v1
|
||||
RELAY_API_KEY=<relay-key>
|
||||
```
|
||||
|
||||
Then call `node imagine.js "..." -m google/gemini-2.5-flash-image`. Prefixed model names route to the relay automatically; use `--provider relay` to force it. When `GEMINI_API_KEY` is set, the native channel wins for bare `gemini-*` names — comment it out or pass `--provider relay`.
|
||||
|
||||
If the relay domain is unreachable directly (DNS pollution / SNI reset, common in China), route through the local proxy client (Clash/V2Ray etc. — the port is shown in the client's settings):
|
||||
|
||||
```bash
|
||||
HTTPS_PROXY=http://127.0.0.1:7890
|
||||
```
|
||||
|
||||
The script sends all HTTP traffic through the proxy via a zero-dependency CONNECT tunnel.
|
||||
- Only configured providers are usable — fall back rather than fail.
|
||||
- The AGENTS.md merge enables automatic generation; `.env` alone is not enough.
|
||||
|
||||
@@ -1,56 +1,32 @@
|
||||
---
|
||||
name: core-usage
|
||||
description: How to invoke imagine.js — prompts, providers, output files, editing, and exit codes.
|
||||
description: imagine.js CLI reference — generate, edit, model/provider selection, arguments, sessions, output, and exit codes.
|
||||
---
|
||||
|
||||
# Usage
|
||||
|
||||
Run the script with Node.js (18+). It sends the prompt to the selected provider, downloads the generated image, and saves it to disk. The provider is inferred from the model id, or forced with `--provider`.
|
||||
Run the script with Node.js (18+). The provider is inferred from the model id, or forced with `--provider`.
|
||||
Run `node scripts/imagine.js --help` for the full flag list.
|
||||
|
||||
## Generate
|
||||
## Generate and edit
|
||||
|
||||
```bash
|
||||
node imagine.js "a cute cat sticker, die-cut, white border" -o cat.png
|
||||
```
|
||||
|
||||
## Edit an existing image
|
||||
|
||||
```bash
|
||||
node imagine.js "add a red balloon" -e photo.png -o edited.png
|
||||
```
|
||||
|
||||
`-e/--edit` is supported by all three providers:
|
||||
|
||||
- OpenAI → `/v1/images/edits` (multipart, optional `--mask` for inpainting)
|
||||
- SiliconFlow → passes the image to `Qwen/Qwen-Image-Edit-2509` via the `image` field
|
||||
- Gemini → text + inline image parts in `generateContent`
|
||||
`-e/--edit` works on all providers.
|
||||
|
||||
## Provider and model selection
|
||||
|
||||
```bash
|
||||
# Explicit provider (overrides model-based inference)
|
||||
node imagine.js "..." --provider siliconflow -m Qwen/Qwen-Image
|
||||
|
||||
# Model-based inference
|
||||
node imagine.js "..." -m gpt-image-2 # -> openai (default)
|
||||
node imagine.js "..." -m gemini-2.5-flash-image # -> gemini
|
||||
node imagine.js "..." -m Qwen/Qwen-Image # -> siliconflow
|
||||
node imagine.js "..." -m gpt-image-2 # -> openai (default)
|
||||
node imagine.js "..." -m gemini-2.5-flash-image # -> gemini
|
||||
node imagine.js "..." -m Qwen/Qwen-Image # -> siliconflow
|
||||
node imagine.js "..." -m google/gemini-2.5-flash-image # -> relay (prefixed names auto-route)
|
||||
node imagine.js "..." --provider relay -m google/gemini-2.5-flash-image # force relay
|
||||
```
|
||||
|
||||
## OpenAI-compatible relays (e.g. Nano Banana via a Chinese relay)
|
||||
|
||||
Set `RELAY_API_KEY` + `RELAY_BASE_URL` to the relay endpoint. Prefixed model names (e.g. `google/gemini-2.5-flash-image`, `openai/gpt-image-2`, `bailian/qwen-image-3.0`) route to the relay automatically; `--provider relay` forces it:
|
||||
|
||||
```bash
|
||||
RELAY_API_KEY=<relay-key>
|
||||
RELAY_BASE_URL=https://api.ofox.ai/v1
|
||||
|
||||
node imagine.js "a cute dog" -m google/gemini-2.5-flash-image -o dog.png # -> relay (auto-detected)
|
||||
node imagine.js "a cute dog" -m google/gemini-2.5-flash-image --provider relay -o dog.png
|
||||
```
|
||||
|
||||
If the relay domain is unreachable on your network (DNS pollution / SNI reset), set `HTTPS_PROXY` — see core-setup.
|
||||
|
||||
## Arguments
|
||||
|
||||
| Argument | Description |
|
||||
@@ -58,52 +34,36 @@ If the relay domain is unreachable on your network (DNS pollution / SNI reset),
|
||||
| `<prompt>` | Text prompt describing the image to generate (required) |
|
||||
| `-o, --output <path>` | Output file or directory; defaults to `image-<timestamp>-<n>.<ext>` |
|
||||
| `-m, --model <id>` | Model id; defaults per provider (`gpt-image-2`, `Qwen/Qwen-Image`, `gemini-2.5-flash-image`) |
|
||||
| `--provider <name>` | Force provider: `openai`, `siliconflow`, `gemini` |
|
||||
| `--provider <name>` | Force provider: `openai`, `siliconflow`, `gemini`, `relay` |
|
||||
| `-s, --size <WxH>` | Image size, e.g. `1024x1024`, `1536x1024`, `1328x1328` |
|
||||
| `-q, --quality <lvl>` | `low` / `medium` / `high` (OpenAI GPT Image models) |
|
||||
| `--background <bg>` | `auto` / `transparent` / `opaque` (OpenAI; `transparent` requires `-f png` or `webp`) |
|
||||
| `-f, --format <fmt>` | `png` / `jpeg` / `webp` output format |
|
||||
| `-e, --edit <image>` | Edit an existing image (local path) |
|
||||
| `--mask <image>` | Mask for inpainting (OpenAI edits; white = regenerate, black = keep) |
|
||||
| `-n, --num-images <n>` | Number of images to generate (OpenAI / SiliconFlow Kolors) |
|
||||
| `--seed <n>` | Seed for reproducible output (SiliconFlow / Gemini) |
|
||||
| `--steps <n>` | Inference steps, default 20 (SiliconFlow) |
|
||||
| `--aspect <ratio>` | Aspect ratio for Gemini: `1:1`, `16:9`, `9:16`, `4:3`, `3:4`, `2:3`, `3:2`, `21:9` |
|
||||
| `-S, --session <name>` | Conversation continuity — reuse the previous generation in this session as edit input / conversation context (default: stateless, no continuity) |
|
||||
| `--session-reset` | Clear the session state for `--session <name>` before this run |
|
||||
| `-S, --session <name>` | Session continuity — preserve context across calls so later prompts can build on earlier results (default: stateless, no continuity) |
|
||||
|
||||
## Conversation continuity (`--session`)
|
||||
## Sessions (`--session`)
|
||||
|
||||
Some workflows need a follow-up generation that builds on the previous image (e.g. "generate the design, then draw its three views"). Pass the same `--session <name>` to both calls; omit it to keep the existing stateless behavior.
|
||||
For multi-step workflows that build on earlier results (e.g. design → three views), pass the same `--session <name>` to each call; omit it for stateless generation. State is stored in `.imagine/<name>.json` (or `IMAGINE_SESSION_DIR`).
|
||||
|
||||
```bash
|
||||
# First call: create the design
|
||||
node imagine.js "生成一格Minecraft蜗牛，蜗牛壳是一个立方体，不要堆叠。蜗牛身体结构简单，有触角。蜗牛足够将整个身体缩回到蜗牛壳内部。请绘制 将宽高比设为 1:1" --session snail -o design.png
|
||||
node imagine.js "a cute robot mascot, front view, flat vector style" --session mascot -o design.png
|
||||
|
||||
# Second call: continue the same conversation for the three views
|
||||
node imagine.js "非常好，画出它的三视图" --session snail -o views.png
|
||||
# Second call: continue the same session for the three views
|
||||
node imagine.js "draw its three views (front, side, back) from the same design" --session mascot -o views.png
|
||||
```
|
||||
|
||||
Session state is stored in `.imagine/<name>.json` (cwd), or `IMAGINE_SESSION_DIR/<name>.json` when that env var is set. Each call appends a user turn and the generated image paths to the state.
|
||||
- **Gemini / Nano Banana** — replays the whole conversation natively; later turns refine or re-draw earlier images.
|
||||
- **OpenAI / relays** — the latest session image is passed as the edit input.
|
||||
- **SiliconFlow** — the latest session image is passed via the `image` field; use an edit-capable model (`Qwen/Qwen-Image-Edit-2509`) for reliable continuity.
|
||||
|
||||
Provider behavior:
|
||||
A provider/model change between calls resets the session with a warning. `--session` combined with `--edit` uses the explicit `-e` image and still records the turn.
|
||||
|
||||
- **Gemini / Nano Banana** — replays the whole conversation (`contents`) natively, so later turns can refine or re-draw the earlier image.
|
||||
- **OpenAI / relays** — the latest session image is passed as the edit input to `/v1/images/edits` with the new prompt.
|
||||
- **SiliconFlow** — the latest session image is passed via the `image` field; use an edit-capable model (e.g. `Qwen/Qwen-Image-Edit-2509`) for reliable continuity.
|
||||
## Output and exit codes
|
||||
|
||||
`--session` with `--edit` uses the explicit `-e` image as input and still records the turn in the session. If the model/provider changes between calls of the same session name, the session is reset with a warning.
|
||||
|
||||
## Output
|
||||
|
||||
- The script prints the absolute path(s) of saved files to stdout.
|
||||
- If `-o` points to a directory (no extension), files are saved inside it as `image-<n>.<ext>`.
|
||||
- With `-n 2` (or a multi-image provider response), an index is appended before the extension.
|
||||
- Absolute path(s) of saved files are printed to stdout; `-o` may point to a directory.
|
||||
- SiliconFlow image URLs expire after one hour — the script downloads them immediately.
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| `0` | Success, image(s) saved and paths printed to stdout |
|
||||
| `1` | Missing prompt / API key, unknown option, or API failure (error printed to stderr) |
|
||||
- Exit `0` on success; `1` on missing prompt/key, unknown option, or API failure (message on stderr).
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
name: features-sticker
|
||||
description: Sticker, icon, and logo generation — die-cut stickers, transparent backgrounds, and the best models for asset-ready output.
|
||||
---
|
||||
|
||||
# Stickers & Icons
|
||||
|
||||
Stickers and icons need two things generic image models often get wrong: **clean shapes** and a **transparent background**. Specialist models or explicit parameters solve both.
|
||||
|
||||
## 1. Ideogram 3.0 (recommended for stickers/logos)
|
||||
|
||||
Ideogram is the strongest model for typography and clean graphic assets, and its transparent-background mode is purpose-built for stickers.
|
||||
|
||||
- **Direct API** — `POST https://api.ideogram.ai/generate`:
|
||||
```json
|
||||
{
|
||||
"image_request": {
|
||||
"prompt": "die-cut stickers of a sun, a cloud and a star, thick white outline",
|
||||
"model": "V_3",
|
||||
"background_transparent": true,
|
||||
"magic_prompt_option": "Auto",
|
||||
"aspect_ratio": "1:1"
|
||||
}
|
||||
}
|
||||
```
|
||||
Response: `data[0].url` (PNG with transparency).
|
||||
- **Transparent variant** — some hosts expose `ideogram-v3/generate-transparent` with just `{ "prompt": "..." }`.
|
||||
- **Style presets** — `AUTO`, `GENERAL`, `REALISTIC`, `DESIGN`; use `DESIGN` for logos and stickers.
|
||||
- **Also via fal / aggregators** — `fal-ai/ideogram/v3` supports `transparent_background`, `style_references`, and `character_references`.
|
||||
|
||||
## 2. GPT Image (`gpt-image-1.5` / `gpt-image-2`)
|
||||
|
||||
GPT Image models produce excellent die-cut stickers on demand.
|
||||
|
||||
```bash
|
||||
node scripts/imagine.js "die-cut sticker of a cute sloth, flat vector, thick white border" -m gpt-image-1.5 --background transparent -f png -o sloth.png
|
||||
```
|
||||
|
||||
- Transparent output requires `output_format: png` (or `webp`).
|
||||
- `gpt-image-1.5` is the reliable transparent-PNG choice; `gpt-image-2` exposes `background` on OpenAI native but some hosts omit it.
|
||||
- Keep sticker prompts short and shape-focused: "bold clean shapes", "thick black/white outline", "flat vector style", "no background".
|
||||
|
||||
## 3. FLUX.2 pro
|
||||
|
||||
- Best for photorealistic sticker *mockups* (a sticker sheet lying on a table) and consistent character stickers using reference images (up to 8 input images).
|
||||
- Endpoints: BFL `/flux-2-pro`, fal `fal-ai/flux-2-pro`, Together, Replicate.
|
||||
- Transparent output is not its strength — prefer Ideogram or GPT Image for cutout stickers.
|
||||
|
||||
## Prompt recipes
|
||||
|
||||
| Style | Prompt seed |
|
||||
|-------|-------------|
|
||||
| Classic die-cut | "die-cut sticker of {subject}, thick white border, bold clean shapes, flat vector" |
|
||||
| Cute / kawaii | "kawaii chibi {subject} sticker, glossy highlights, soft pastel colors, rounded shapes" |
|
||||
| Minimal icon | "minimal line icon of {subject}, single color, centered, transparent background" |
|
||||
| Logo | "flat logo mark of {subject}, DESIGN style, vector, transparent background" |
|
||||
| Emoji-style | "emoji-style {subject}, bold outlines, vibrant colors, sticker" |
|
||||
|
||||
## Key points
|
||||
|
||||
- Transparent background: `--background transparent -f png` (GPT Image) or `background_transparent: true` (Ideogram).
|
||||
- "Die-cut sticker" + "white/black outline" is the single most reliable sticker prompt phrase.
|
||||
- For consistent character stickers across a set, use FLUX.2 with character references.
|
||||
|
||||
<!--
|
||||
Source references:
|
||||
- https://ideogram.ai/api-transparent-backgrounds/
|
||||
- https://ideogram.ai/api-learn/
|
||||
- https://fal.ai/docs/model-api-reference/image-generation-api/ideogram-v3
|
||||
- https://developers.cloudflare.com/ai/models/openai/gpt-image-2/
|
||||
- https://docs.bfl.ai/quick_start/generating_images
|
||||
- https://www.together.ai/models/flux-2-pro
|
||||
-->
|
||||
@@ -1,77 +0,0 @@
|
||||
---
|
||||
name: features-texture
|
||||
description: Seamless texture and tile generation — Z-Image Turbo Tiling, FLUX.2, and prompt recipes for tileable game/3D/UI textures.
|
||||
---
|
||||
|
||||
# Seamless Textures
|
||||
|
||||
The hard requirement for textures is **seamless tiling**: edges must wrap so the texture repeats without visible seams. A purpose-built tiling model beats prompt tricks alone.
|
||||
|
||||
## 1. Z-Image Turbo Tiling (recommended for tileable textures)
|
||||
|
||||
Tongyi-MAI's Z-Image Turbo with toroidal-RoPE + circular VAE — generates pixel-wrap seamless repeating textures.
|
||||
|
||||
- **Endpoint** — `POST https://fal.run/fal-ai/z-image/turbo/tiling`, auth `Authorization: Key $FAL_KEY`.
|
||||
- **Key parameters**:
|
||||
- `tile_size` (latent): `64` = 512px, `128` = 1024px (default), `256` = 2048px
|
||||
- `tile_stride` (latent): `32` = 256px, `64` = 512px (default), `128` = 1024px
|
||||
- `tiling_mode`: `both` (omnidirectional, default), `horizontal`, `vertical`
|
||||
- `image_size`: `square_hd` (default), `square`, `portrait_4_3`, `portrait_16_9`, `landscape_4_3`, `landscape_16_9`, `auto`
|
||||
- `num_inference_steps`: 1–8 (default 8); `num_images`: 1–4; `output_format`: `jpeg`/`png`/`webp`
|
||||
- **Example**:
|
||||
```bash
|
||||
curl -X POST https://fal.run/fal-ai/z-image/turbo/tiling \
|
||||
-H "Authorization: Key $FAL_KEY" -H "Content-Type: application/json" \
|
||||
-d '{"prompt":"seamless tileable 4k brick wall texture, weathered, moss in mortar lines, flat uniform lighting, no shadows","image_size":"square_hd","tiling_mode":"both","num_inference_steps":8}'
|
||||
```
|
||||
- **Also on SiliconFlow** — `Tongyi-MAI/Z-Image-Turbo` is available OpenAI-compatible for general fast generation, but the seamless *tiling* mode is a fal-specific variant.
|
||||
|
||||
## 2. FLUX.2 pro
|
||||
|
||||
- High-detail, photorealistic textures; strong at material variation (wood, stone, fabric, metal).
|
||||
- Pair with a "seamless / tileable / flat top-down" prompt; FLUX is not tile-native so expect to check the seams.
|
||||
- Supports up to 8 reference images for style consistency.
|
||||
|
||||
## 3. Nano Banana / GPT Image 2 (prompt-based fallback)
|
||||
|
||||
When a tiling model is unavailable, use any strong image model with a strict tiling prompt:
|
||||
|
||||
```bash
|
||||
node scripts/imagine.js "seamless tileable dark wood floor texture, top-down flat view, uniform lighting, no shadows, edges wrap perfectly, 4k detail" -o wood.png
|
||||
```
|
||||
|
||||
- GPT Image 2: best for realistic PBR-style base maps with text-free material detail.
|
||||
- Nano Banana: free and fast for iterating on material looks.
|
||||
|
||||
## 4. 3D / game-asset pipelines
|
||||
|
||||
- **Seed3D 2.0** (ByteDance / Volcano Engine) — 3D geometry + texture generation in one model, production-ready for games.
|
||||
- **Ludo.ai** — game-asset REST API: sprites, icons, UI assets, textures, backgrounds, even 2D→3D with PBR maps.
|
||||
- **GenAI Model Generator** (Unreal plugin) — PBR map sets (base color, normal, roughness, metallic) via Google Gemini.
|
||||
|
||||
## Prompt recipes
|
||||
|
||||
| Material | Prompt seed |
|
||||
|----------|-------------|
|
||||
| Stone/brick | "seamless tileable {brick/stone} wall texture, weathered, uniform flat lighting, no shadows, edges wrap" |
|
||||
| Wood | "seamless tileable {oak/dark} wood floor texture, top-down, straight grain, flat lighting" |
|
||||
| Fabric | "seamless tileable {linen/denim} fabric texture, fine weave detail, even tone" |
|
||||
| Metal | "seamless brushed metal texture, horizontal grain, subtle reflections, flat lighting" |
|
||||
| Organic | "seamless moss/ground texture, top-down, natural color variation, no vignette" |
|
||||
|
||||
## Key points
|
||||
|
||||
- Use `tiling_mode: both` for general-purpose tiles; `horizontal`/`vertical` only for specific use cases.
|
||||
- Larger `tile_size` = higher resolution tile (up to 2048px) but slower.
|
||||
- Always say "seamless", "tileable", "top-down", "flat/uniform lighting", "no shadows" and "edges wrap perfectly".
|
||||
- For 3D workflows, generate base color first, then derive normal/roughness maps in the target engine or with PBR tools.
|
||||
|
||||
<!--
|
||||
Source references:
|
||||
- https://fal.ai/docs/model-api-reference/image-generation-api/z-image-turbo-tiling
|
||||
- https://fal.ai/models/fal-ai/z-image/turbo/tiling/api
|
||||
- https://docs.bfl.ai/quick_start/generating_images
|
||||
- https://www.aibase.com/news/27393
|
||||
- https://github.com/api-evangelist/ludo-ai
|
||||
- https://muddyterrain.com/docs/genai-modelgenerator/
|
||||
-->
|
||||
@@ -1,73 +0,0 @@
|
||||
---
|
||||
name: models-image
|
||||
description: Image generation models — GPT Image 2, Nano Banana (Gemini), SiliconFlow Qwen-Image, OpenAI GPT Image — capabilities, API parameters, and when to use each.
|
||||
---
|
||||
|
||||
# Image Models
|
||||
|
||||
The user's priority order for general image generation: **GPT Image 2 → Nano Banana → SiliconFlow Qwen-Image → OpenAI GPT Image**. All four can generate photos, illustrations, and posters; the differences are quality, text rendering, price, and latency.
|
||||
|
||||
## GPT Image 2 (`gpt-image-2`, OpenAI)
|
||||
|
||||
OpenAI's flagship image model (ChatGPT Images 2.0, released 2026-04). Best overall quality in the list.
|
||||
|
||||
- **Capabilities** — 4K output (up to `3840x2160`), photorealistic, accurate instruction following, excellent CJK and multilingual text rendering, strong at layouts/posters with text.
|
||||
- **Editing** — text + image editing, blends up to 16 input images (compose subjects, styles, references).
|
||||
- **Parameters** — `size` (`1024x1024`, `1024x1536`, `1536x1024`, `auto`, or arbitrary `WIDTHxHEIGHT` where both are divisible by 16 and aspect ratio is within 1:3–3:1), `quality` (`low`/`medium`/`high`/`auto`), `background` (`auto`/`opaque`/`transparent`), `output_format` (`png`/`jpeg`/`webp`), `output_compression` (0–100, JPEG).
|
||||
- **Pricing note** — premium tier; use `quality: low/medium` for drafts and `high` for final assets.
|
||||
- **Transparent caveat** — on some hosts (e.g. Cloudflare) `gpt-image-2` does not expose transparent backgrounds; use `gpt-image-1.5` when you need a transparent PNG.
|
||||
|
||||
## Nano Banana (`gemini-2.5-flash-image`, Google Gemini)
|
||||
|
||||
Google's fast native image model, plus newer variants:
|
||||
|
||||
| Brand | Model id | Notes |
|
||||
|-------|----------|-------|
|
||||
| Nano Banana | `gemini-2.5-flash-image` | GA, fast, good free tier |
|
||||
| Nano Banana 2 | `gemini-3.1-flash-image(-preview)` | Flash-speed quality, up to 4K |
|
||||
| Nano Banana Pro | `gemini-3-pro-image(-preview)` | Highest quality, advanced reasoning, 4K |
|
||||
|
||||
- **Endpoint** — `POST https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent` with `x-goog-api-key` header or `?key=` query param.
|
||||
- **Request** — `generationConfig.response_modalities: ["IMAGE"]`, and `generationConfig.imageConfig.aspectRatio`: `1:1` (default), `16:9`, `9:16`, `4:3`, `3:4`, `2:3`, `3:2`, `21:9` (note: REST uses camelCase — `imageConfig.aspectRatio`, not `aspect_ratio`).
|
||||
- **Editing** — pass the image as an `inline_data` part before the text part; great for conversational multi-turn edits ("make the sky purple", "now add birds").
|
||||
- **Response** — `candidates[0].content.parts[].inlineData.data` (base64) or `fileData.fileUri`.
|
||||
- **Free tier** — key from Google AI Studio; SynthID watermarking is applied automatically.
|
||||
- **OpenAI-compatible route** — aggregators expose it as `google/gemini-2.5-flash-image` on `/v1/images/generations`.
|
||||
|
||||
## SiliconFlow Qwen-Image (`Qwen/Qwen-Image`, SiliconFlow)
|
||||
|
||||
Alibaba's 20B MMDiT open model hosted on SiliconFlow with an OpenAI-compatible API. The cheapest option in the priority list.
|
||||
|
||||
- **Endpoint** — `POST https://api.siliconflow.cn/v1/images/generations`, `Authorization: Bearer <key>`.
|
||||
- **Model ids** — `Qwen/Qwen-Image` (generation), `Qwen/Qwen-Image-Edit-2509` (editing, `image` field), `Tongyi-MAI/Z-Image-Turbo` (fast generation).
|
||||
- **Recommended sizes** — `1328x1328` (1:1), `1664x928` (16:9), `928x1664` (9:16), `1472x1140` (4:3), `1140x1472` (3:4), `1584x1056` (3:2), `1056x1584` (2:3).
|
||||
- **Parameters** — `image_size`, `num_inference_steps` (default 20, range 1–100), `seed`, `negative_prompt`, `batch_size` (Kolors only).
|
||||
- **Pricing** — ~$0.042/image; strong Chinese and English text rendering.
|
||||
- **Response** — `images[].url`; the URL expires after one hour, so download it immediately.
|
||||
|
||||
## OpenAI GPT Image (`gpt-image-1`, OpenAI)
|
||||
|
||||
The previous-generation GPT Image model — the "GPT Image" the user already knows. Still the most widely available GPT Image variant on third-party platforms.
|
||||
|
||||
- **Capabilities** — photorealistic images, accurate text rendering, precise instruction following; the family also includes `gpt-image-1.5` and `gpt-image-1-mini`.
|
||||
- **Parameters** — `size` (`1024x1024`, `1536x1024`, `1024x1536`, `auto`), `quality` (`low`/`medium`/`high`/`auto`), `background` (`auto`/`transparent`/`opaque` — transparent requires `png`/`webp`), `output_format` (`png`/`jpeg`/`webp`), `style` (`vivid`/`natural`).
|
||||
- **Editing** — `/v1/images/edits` with multipart `image` (+ optional `mask`), `prompt`, and options.
|
||||
- **When to use** — when `gpt-image-2` is unavailable (region/aggregator limits) or when you need the reliably-supported transparent PNG path.
|
||||
|
||||
## Choosing
|
||||
|
||||
- **Best quality, text in image** → `gpt-image-2`
|
||||
- **Fast + free, iterative edits** → Nano Banana
|
||||
- **Cheap batch / Chinese text** → SiliconFlow Qwen-Image
|
||||
- **Fallback / widest compatibility** → `gpt-image-1`
|
||||
|
||||
<!--
|
||||
Source references:
|
||||
- https://developers.openai.com/api/docs/guides/tools-image-generation
|
||||
- https://developers.cloudflare.com/ai/models/openai/gpt-image-2/
|
||||
- https://ai.google.dev/gemini-api/docs/models/gemini-2.5-flash-image
|
||||
- https://docs.siliconflow.cn/en/api-reference/images/images-generations
|
||||
- https://www.siliconflow.com/blog/qwen-image-is-here-on-siliconflow-superior-text-rendering-precise-image-editing
|
||||
- https://fal.ai/learn/tools/what-is-gpt-image-2
|
||||
- https://itbrief.co.uk/story/openai-launches-chatgpt-images-2-0-with-api-access
|
||||
-->
|
||||
@@ -40,7 +40,7 @@ import {
|
||||
siliconflowGenerate,
|
||||
} from "./lib/providers.js";
|
||||
import { extractImages, resolveOutputPath } from "./lib/output.js";
|
||||
import { clearSession, loadSession, saveSession } from "./lib/session.js";
|
||||
import { loadSession, saveSession } from "./lib/session.js";
|
||||
|
||||
/**
|
||||
* Parse CLI arguments
|
||||
@@ -63,7 +63,6 @@ function parseArgs() {
|
||||
steps: 0,
|
||||
aspect: "",
|
||||
session: "",
|
||||
sessionReset: false,
|
||||
help: false,
|
||||
};
|
||||
const positional = [];
|
||||
@@ -121,9 +120,6 @@ function parseArgs() {
|
||||
case "--session":
|
||||
opts.session = argv[++i] || "";
|
||||
break;
|
||||
case "--session-reset":
|
||||
opts.sessionReset = true;
|
||||
break;
|
||||
case "-h":
|
||||
case "--help":
|
||||
opts.help = true;
|
||||
@@ -160,12 +156,10 @@ Options:
|
||||
--seed <n> Seed for reproducible output
|
||||
--steps <n> Inference steps (SiliconFlow, default 20)
|
||||
--aspect <ratio> Aspect ratio for Gemini: 1:1, 16:9, 9:16, 4:3, 3:4
|
||||
-S, --session <name> Conversation continuity: reuse the previous generation in
|
||||
this session as the edit input / conversation context.
|
||||
State is stored in .imagine/<name>.json
|
||||
(IMAGINE_SESSION_DIR overrides the directory). Omit to keep
|
||||
the default stateless behavior.
|
||||
--session-reset Clear the session state before this run
|
||||
-S, --session <name> Session continuity: keep context across calls so later
|
||||
prompts can build on earlier results. State is stored in
|
||||
.imagine/<name>.json (IMAGINE_SESSION_DIR overrides the
|
||||
directory). Omit to keep the default stateless behavior.
|
||||
-h, --help Show this help
|
||||
|
||||
Env vars (or .env next to the script / in cwd):
|
||||
@@ -225,10 +219,6 @@ async function main() {
|
||||
// Session continuity — load prior state (or start fresh) and attach it to opts.
|
||||
let sessionState = null;
|
||||
if (opts.session) {
|
||||
if (opts.sessionReset) {
|
||||
clearSession(opts.session);
|
||||
console.error(`提示: 已重置会话 "${opts.session}"。`);
|
||||
}
|
||||
sessionState = loadSession(opts.session);
|
||||
if (sessionState && (sessionState.provider !== provider || sessionState.model !== model)) {
|
||||
console.error(
|
||||
|
||||
@@ -38,14 +38,6 @@ export function saveSession(name, state) {
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a session state file.
|
||||
*/
|
||||
export function clearSession(name) {
|
||||
const file = sessionPath(name);
|
||||
if (fs.existsSync(file)) fs.unlinkSync(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the most recent generated image path in the session history.
|
||||
*/
|
||||
|
||||
@@ -13,6 +13,14 @@ node vision.js --base64 "<data URL>" "Describe this image"
|
||||
node vision.js --base64 - < shot.b64 "Describe this image" --coords center
|
||||
```
|
||||
|
||||
Multi-turn conversations (e.g. "compare with the previous image"): pass the
|
||||
same `-S <name>` to each call so earlier images and replies are replayed:
|
||||
|
||||
```
|
||||
node vision.js "shot1.png" "Describe this screenshot" -S ui
|
||||
node vision.js "shot2.png" "Compare with the previous screenshot" -S ui
|
||||
```
|
||||
|
||||
Keep vision replies concise (no filler). For UI/debug inspection — locating elements, click targets, layout — pass `--coords` to get precise pixel element coordinates.
|
||||
|
||||
## Trigger scenarios
|
||||
|
||||
+13
-2
@@ -3,14 +3,15 @@ name: vision
|
||||
description: Image recognition for agents without native vision support. Use when the user shares an image path or URL, the message contains saved image attachments, or the user asks to analyze, describe, or identify image content. Sends local images or remote URLs to an OpenAI-compatible vision model; --coords returns pixel element coordinates for UI/debug inspection.
|
||||
metadata:
|
||||
author: Hairy
|
||||
version: "2026.8.10"
|
||||
compatibility: Requires Node.js 18+ (native fetch) and network access to an OpenAI-compatible vision API.
|
||||
version: "2026.8.11"
|
||||
---
|
||||
|
||||
# Vision
|
||||
|
||||
The underlying model has no native image understanding. When an image arrives, **do not use the Read tool** — run `scripts/vision.js` to get a text description back.
|
||||
|
||||
Requires Node.js 18+ (native fetch) and network access to an OpenAI-compatible vision API.
|
||||
|
||||
## When to use
|
||||
|
||||
- User shares an image path (local file or remote URL)
|
||||
@@ -25,10 +26,20 @@ node scripts/vision.js --url "<image url>" "[question]"
|
||||
node scripts/vision.js "<image path>" "find the search button" --coords
|
||||
node scripts/vision.js --base64 "<base64 or data:image/...;base64,... URL>" "[question]"
|
||||
node scripts/vision.js --base64 - < shot.b64 "[question]" --coords center
|
||||
node scripts/vision.js "<image path>" "[question]" -S ui # multi-turn conversation
|
||||
node scripts/vision.js "<image 2>" "compare with the previous image" -S ui
|
||||
```
|
||||
|
||||
Run `node scripts/vision.js --help` for the full flag list.
|
||||
|
||||
## Session continuity
|
||||
|
||||
Pass the same `-S, --session <name>` to several calls to keep the conversation
|
||||
history — previous images and replies are replayed alongside the current
|
||||
question on every call. State is stored in `.vision/<name>.json` (cwd), or
|
||||
`VISION_SESSION_DIR/<name>.json` when that env var is set. See
|
||||
[core-usage](references/core-usage.md).
|
||||
|
||||
## Output contract
|
||||
|
||||
The reply is injected into the caller's context, so keep it compact:
|
||||
|
||||
@@ -19,6 +19,7 @@ scripts/
|
||||
├── prompts.js # System prompts + per-round contracts
|
||||
├── image.js # Format sniffing, size detection, base64 data URLs
|
||||
├── api.js # Payload building + OpenAI-compatible request
|
||||
├── session.js # Conversation state (--session): load/save/clear, replay
|
||||
└── coords.js # Parse/normalize/scale coordinates (shared)
|
||||
```
|
||||
|
||||
@@ -29,6 +30,38 @@ scripts/
|
||||
3. **Coarse-to-fine rounds** — round 1 asks the model to propose a precise point or a zoom region (`propose` contract); round 2 re-locates in the focused crop (`locate` contract, bbox or center per `--coords` format); `--rounds 3` adds a small-window verification round (`verify` contract).
|
||||
4. **Call the API per round** — POST to `{BASE_URL}/chat/completions` with a user message containing a base64 `image_url` part and a `text` part.
|
||||
5. **Normalize & remap** — each round's `## Coordinates` section is normalized to canonical JSON lines (tolerating malformed output), and every center point is scaled back to the ORIGINAL image pixels using that round's crop offset.
|
||||
6. **Session save** — when `--session` is passed, the user turn (question + image source) and the assistant turn (the printed reply) are appended to the state file.
|
||||
|
||||
## Session continuity (`--session`)
|
||||
|
||||
`scripts/lib/session.js` manages conversation state at `.vision/<name>.json`
|
||||
(or `VISION_SESSION_DIR`), mirroring the imagine skill's design:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "qwen-vl-max",
|
||||
"turns": [
|
||||
{ "role": "user", "text": "Describe this screenshot", "images": ["C:/abs/path/shot1.png"] },
|
||||
{ "role": "assistant", "text": "A login form with…", "images": [] }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- `isValidSessionName` / `sessionPath` / `loadSession` / `saveSession`
|
||||
validate the name (letters/digits/`-`/`_`), resolve, read, and write the
|
||||
state file; `trimTurns` caps the replayed history
|
||||
(default 10, `VISION_SESSION_MAX_TURNS`); `imageUrlFor` converts a stored
|
||||
source (data URL, http URL, or local path) into an API-ready image at replay
|
||||
time — missing local files yield `null` so the turn degrades to text-only.
|
||||
- On a session call, `vision.js` loads the state, replays the trimmed turns via
|
||||
`buildPayload(..., { history })`, and appends the new user + assistant turns
|
||||
after the reply succeeds. The current image + per-round instructions are
|
||||
always the last message, so contracts like `locate` only apply to the current
|
||||
task.
|
||||
- A model change between calls resets the session with a warning.
|
||||
- Image sources are stored verbatim: local paths are stored as absolute paths,
|
||||
URLs as URLs, and `--base64` input as a `data:image/...;base64,...` URL (the
|
||||
temp file itself is still cleaned up after the run).
|
||||
|
||||
## Configuration (env vars or .env)
|
||||
|
||||
@@ -37,6 +70,8 @@ scripts/
|
||||
| `DASHSCOPE_API_KEY` | — | Required. API key for the vision service |
|
||||
| `VISION_MODEL` | `qwen-vl-max` | Model name |
|
||||
| `DASHSCOPE_BASE_URL` | `https://dashscope.aliyuncs.com/compatible-mode/v1` | OpenAI-compatible endpoint |
|
||||
| `VISION_SESSION_DIR` | `.vision` | Directory for `--session` state files |
|
||||
| `VISION_SESSION_MAX_TURNS` | `10` | Max history turns replayed per session call |
|
||||
|
||||
`.env` files are loaded from the current working directory and the script's directory, and never override already-set environment variables.
|
||||
|
||||
@@ -49,6 +84,8 @@ scripts/
|
||||
- Debug mode is caller-driven: coordinates appear only with `--coords`; there is no keyword auto-detection.
|
||||
- Remote URLs are downloaded so resampling works everywhere; no caller-supplied dimensions needed.
|
||||
- `--base64` materializes the payload to a temp file in the OS temp dir (cleaned up on exit), keeping `--coords` resampling unchanged.
|
||||
- Session continuity is stateless by default — history is replayed only when
|
||||
`-S/--session` is passed; the same session name resumes the conversation.
|
||||
- Resampling uses `sharp` (installed on first `--coords` use into `scripts/.deps/`); the non-debug path stays zero-dependency.
|
||||
- The coordinate contract emits bbox by default and center points with `--coords center`; parsing derives the other representation (bbox ↔ center) automatically.
|
||||
- Model input limits come from a per-model table with `VISION_MAX_PIXELS` / `VISION_MAX_LONG_EDGE` env overrides, since the API does not expose them.
|
||||
|
||||
@@ -72,6 +72,7 @@ Unknown models fall back to a conservative 1568px / 1.15MP limit. Tune via env v
|
||||
| `--coords [center]` | Debug mode: append the `## Coordinates` section (bbox by default; center points with `--coords center`) in original pixels |
|
||||
| `--detail [n]` | Fuller detail output; optional token cap `n` (default 1600; compact mode caps at 1000) |
|
||||
| `--rounds N` | 1 = single locate, 2 = coarse-to-fine (default), 3 = + verification round |
|
||||
| `-S, --session <name>` | Session continuity — replay this session's previous turns (images + text) together with the current question (default: stateless, no continuity) |
|
||||
| `-h, --help` | Print usage and exit |
|
||||
|
||||
Supported local formats: jpg, jpeg, png, gif, webp, bmp.
|
||||
@@ -89,9 +90,41 @@ node vision.js "ui.png" "find the login button and its position" --coords
|
||||
node vision.js "chart.png" "extract all labels and values" --detail 2500
|
||||
```
|
||||
|
||||
## Session continuity (`--session`)
|
||||
|
||||
Some workflows need a follow-up question that builds on images seen earlier in
|
||||
the same conversation (e.g. "describe this screenshot", then "where is the
|
||||
search bar compared to the previous one?"). Pass the same `-S <name>` to both
|
||||
calls; omit it to keep the existing stateless behavior.
|
||||
|
||||
```bash
|
||||
# First call: recognize the first screenshot
|
||||
node vision.js "shot1.png" "Describe this screenshot" -S ui
|
||||
|
||||
# Second call: same session replays shot1 + its answer alongside shot2
|
||||
node vision.js "shot2.png" "Compare the layout with the previous screenshot" -S ui
|
||||
```
|
||||
|
||||
Session state is stored in `.vision/<name>.json` (cwd), or
|
||||
`VISION_SESSION_DIR/<name>.json` when that env var is set. Each call appends a
|
||||
user turn (question + image source) and an assistant turn (the reply) to the
|
||||
state.
|
||||
|
||||
How history is replayed:
|
||||
|
||||
- Every previous user turn's image is sent again (local files are re-read at
|
||||
call time; missing files degrade that turn to text-only instead of failing).
|
||||
- Only the most recent turns are replayed (default 10, override with
|
||||
`VISION_SESSION_MAX_TURNS`), so image-heavy histories don't overflow the
|
||||
model's context window. Stored turns are never deleted.
|
||||
- If the model changes between calls of the same session name, the session is
|
||||
reset with a warning (same behavior as imagine's provider/model check).
|
||||
- `--coords` rounds replay the same history; the final coordinates reply is
|
||||
stored as the assistant turn.
|
||||
|
||||
## Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| `0` | Success, description printed to stdout |
|
||||
| `0` | Success, description printed to stdout (and session saved when `--session` was passed) |
|
||||
| `1` | Missing API key, missing image argument, or API failure (error printed to stderr) |
|
||||
|
||||
@@ -15,23 +15,42 @@ import {
|
||||
* Build the API payload. Instructions are embedded in the user text part so
|
||||
* strict OpenAI-compatible endpoints that only accept user messages still work.
|
||||
*/
|
||||
export function buildPayload(imageUrl, prompt, { contract, format, detail, maxTokens, size }) {
|
||||
export function buildPayload(imageUrl, prompt, { contract, format, detail, maxTokens, size, history }) {
|
||||
const system = detail ? SYSTEM_DETAIL : SYSTEM_BRIEF;
|
||||
let instruction = system;
|
||||
if (contract === "locate") instruction += `\n${coordInstruction(size, format)}`;
|
||||
else if (contract === "propose") instruction += `\n${proposeInstruction(size)}`;
|
||||
else if (contract === "verify") instruction += `\n${verifyInstruction(size)}`;
|
||||
|
||||
// Replay prior turns first (session continuity); the current image+task is
|
||||
// always the last message so per-round instructions only apply to it.
|
||||
const messages = [];
|
||||
for (const turn of history || []) {
|
||||
const text = turn.text || "";
|
||||
const images = Array.isArray(turn.images) ? turn.images.filter(Boolean) : [];
|
||||
if (turn.role === "assistant") {
|
||||
if (text) messages.push({ role: "assistant", content: text });
|
||||
} else if (turn.role === "user" && (images.length || text)) {
|
||||
const content = images.map((image) => ({
|
||||
type: "image_url",
|
||||
image_url: { url: image },
|
||||
}));
|
||||
if (text) content.push({ type: "text", text });
|
||||
messages.push({ role: "user", content });
|
||||
}
|
||||
}
|
||||
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: imageUrl } },
|
||||
{ type: "text", text: `${instruction}\n\nTask: ${prompt}` },
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
model: config.model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: imageUrl } },
|
||||
{ type: "text", text: `${instruction}\n\nTask: ${prompt}` },
|
||||
],
|
||||
},
|
||||
],
|
||||
messages,
|
||||
stream: false,
|
||||
max_tokens: maxTokens,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* Session state management for vision conversations.
|
||||
* Mirrors the imagine skill's session design: state is persisted as JSON in
|
||||
* .vision/<name>.json (or VISION_SESSION_DIR) and replayed on the next call
|
||||
* with the same --session name.
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import { detectMime } from "./image.js";
|
||||
|
||||
// Session state directory; override with VISION_SESSION_DIR.
|
||||
const SESSION_DIR = process.env.VISION_SESSION_DIR || ".vision";
|
||||
|
||||
const NAME_RE = /^[A-Za-z0-9_-]+$/;
|
||||
|
||||
/**
|
||||
* Validate a session name (letters/digits/-/_) to avoid path traversal.
|
||||
*/
|
||||
export function isValidSessionName(name) {
|
||||
return NAME_RE.test(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the state file path for a session name.
|
||||
*/
|
||||
export function sessionPath(name) {
|
||||
if (!isValidSessionName(name)) {
|
||||
throw new Error(`Invalid session name "${name}": use letters, digits, "-" or "_".`);
|
||||
}
|
||||
return path.resolve(SESSION_DIR, `${name}.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a session state, or null when missing/corrupted.
|
||||
*/
|
||||
export function loadSession(name) {
|
||||
const file = sessionPath(name);
|
||||
if (!fs.existsSync(file)) return null;
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf-8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist session state, creating the directory as needed.
|
||||
*/
|
||||
export function saveSession(name, state) {
|
||||
const file = sessionPath(name);
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify(state, null, 2));
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the most recent N turns for replay (default 10, override with
|
||||
* VISION_SESSION_MAX_TURNS). Image-heavy histories are trimmed so replays do
|
||||
* not overflow the model's context window; stored turns are never deleted.
|
||||
*/
|
||||
export function trimTurns(turns, maxTurns) {
|
||||
const cap = Number.isFinite(maxTurns) && maxTurns > 0 ? maxTurns : 10;
|
||||
if (!Array.isArray(turns) || turns.length <= cap) return turns || [];
|
||||
return turns.slice(turns.length - cap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a stored image source into an API-ready image URL/data URL:
|
||||
* - data: URLs (stored base64 input) pass through as-is;
|
||||
* - http(s) URLs pass through as-is (the API fetches them);
|
||||
* - local paths are read at replay time; missing files return null so the
|
||||
* turn degrades to text-only instead of failing.
|
||||
*/
|
||||
export function imageUrlFor(source) {
|
||||
const s = String(source || "");
|
||||
if (!s) return null;
|
||||
if (/^data:image\//i.test(s)) return s;
|
||||
if (/^https?:\/\//i.test(s)) return s;
|
||||
try {
|
||||
const file = path.resolve(s);
|
||||
if (!fs.existsSync(file)) return null;
|
||||
const buf = fs.readFileSync(file);
|
||||
return `data:${detectMime(buf)};base64,${buf.toString("base64")}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,13 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { config } from "./lib/env.js";
|
||||
import { detectMime, resolveImage } from "./lib/image.js";
|
||||
import { buildPayload, requestVisionApi } from "./lib/api.js";
|
||||
import {
|
||||
imageUrlFor,
|
||||
isValidSessionName,
|
||||
loadSession,
|
||||
saveSession,
|
||||
trimTurns,
|
||||
} from "./lib/session.js";
|
||||
import {
|
||||
clampRegion,
|
||||
extractCoordinates,
|
||||
@@ -73,6 +80,14 @@ export {
|
||||
scaleRegion,
|
||||
windowAround,
|
||||
} from "./lib/coords.js";
|
||||
export {
|
||||
imageUrlFor,
|
||||
isValidSessionName,
|
||||
loadSession,
|
||||
saveSession,
|
||||
sessionPath,
|
||||
trimTurns,
|
||||
} from "./lib/session.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
@@ -92,10 +107,17 @@ Flags:
|
||||
--detail [n] Fuller detail output; token cap n (default 1600).
|
||||
--rounds N 1 = single locate, 2 = coarse-to-fine (default),
|
||||
3 = + verification round.
|
||||
-S, --session <name> Session continuity: keep context across calls so later
|
||||
questions can build on earlier images and replies.
|
||||
State is stored in .vision/<name>.json
|
||||
(VISION_SESSION_DIR overrides the directory). Omit to
|
||||
keep the default stateless behavior.
|
||||
-h, --help Show this help.
|
||||
|
||||
Examples:
|
||||
node vision.js shot.png "Describe this image"
|
||||
node vision.js shot.png "Describe this image" -S ui
|
||||
node vision.js shot2.png "Compare with the previous image" -S ui
|
||||
node vision.js ui.png "find the login button" --coords center
|
||||
node vision.js --base64 - "find the login button" --coords center`;
|
||||
}
|
||||
@@ -114,6 +136,7 @@ function parseArgs() {
|
||||
detail: false,
|
||||
maxTokens: 1000,
|
||||
rounds: 2,
|
||||
session: "",
|
||||
help: false,
|
||||
promptParts: [],
|
||||
};
|
||||
@@ -144,6 +167,8 @@ function parseArgs() {
|
||||
} else if (arg === "--rounds" && argv[i + 1] && /^\d+$/.test(argv[i + 1])) {
|
||||
opts.rounds = Math.max(1, Math.min(3, parseInt(argv[i + 1], 10)));
|
||||
i++;
|
||||
} else if (arg === "-S" || arg === "--session") {
|
||||
opts.session = argv[++i] || "";
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
opts.help = true;
|
||||
} else if (arg.startsWith("--")) {
|
||||
@@ -203,7 +228,7 @@ function readStdin() {
|
||||
/**
|
||||
* Decode a raw base64 string or data:image/...;base64,... URL into a temp
|
||||
* image file so the rest of the pipeline can treat it as a local file.
|
||||
* Returns the temp path.
|
||||
* Returns the temp path and the data URL (used as the session image source).
|
||||
*/
|
||||
function materializeBase64(payload) {
|
||||
let b64 = String(payload).trim();
|
||||
@@ -228,7 +253,7 @@ function materializeBase64(payload) {
|
||||
}[mime] || "img";
|
||||
const tmpPath = path.join(os.tmpdir(), `vision-base64-${process.pid}-${Date.now()}.${ext}`);
|
||||
fs.writeFileSync(tmpPath, buf);
|
||||
return tmpPath;
|
||||
return { tmpPath, dataUrl: `data:${mime};base64,${buf.toString("base64")}` };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -257,14 +282,49 @@ async function main() {
|
||||
|
||||
let base64TempPath = null;
|
||||
try {
|
||||
let currentSource = "";
|
||||
if (opts.isBase64) {
|
||||
const payload = opts.imageSource === "-" ? await readStdin() : opts.imageSource;
|
||||
base64TempPath = materializeBase64(payload);
|
||||
const materialized = materializeBase64(payload);
|
||||
base64TempPath = materialized.tmpPath;
|
||||
currentSource = materialized.dataUrl;
|
||||
opts.imageSource = base64TempPath;
|
||||
opts.isUrl = false;
|
||||
} else if (opts.isUrl) {
|
||||
currentSource = opts.imageSource;
|
||||
} else {
|
||||
currentSource = path.resolve(opts.imageSource);
|
||||
}
|
||||
const prompt = opts.promptParts.join(" ") || "Describe this image concisely.";
|
||||
|
||||
// Session continuity — load prior state (or start fresh) and build the
|
||||
// replay history. A model switch resets the session with a warning.
|
||||
let sessionState = null;
|
||||
if (opts.session) {
|
||||
if (!isValidSessionName(opts.session)) {
|
||||
console.error(
|
||||
`Error: invalid session name "${opts.session}" — use letters, digits, "-" or "_".`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
sessionState = loadSession(opts.session);
|
||||
if (sessionState && sessionState.model !== config.model) {
|
||||
console.error(
|
||||
`提示: 会话 "${opts.session}" 属于模型 ${sessionState.model}，本次是 ${config.model}，已重新开始会话。`,
|
||||
);
|
||||
sessionState = null;
|
||||
}
|
||||
if (sessionState) {
|
||||
const history = [];
|
||||
for (const turn of trimTurns(sessionState.turns || [])) {
|
||||
const images = (turn.images || []).map(imageUrlFor).filter(Boolean);
|
||||
history.push({ role: turn.role, text: turn.text || "", images });
|
||||
}
|
||||
opts.sessionHistory = history;
|
||||
}
|
||||
}
|
||||
|
||||
let output = "";
|
||||
if (opts.coords) {
|
||||
// Debug mode: coarse-to-fine localization.
|
||||
// Round 1 proposes a point or a zoom region, round 2 locates precisely
|
||||
@@ -284,6 +344,7 @@ async function main() {
|
||||
detail: opts.detail,
|
||||
maxTokens: opts.maxTokens,
|
||||
size: { width: meta.scaledW, height: meta.scaledH },
|
||||
history: opts.sessionHistory,
|
||||
}),
|
||||
);
|
||||
};
|
||||
@@ -337,7 +398,8 @@ async function main() {
|
||||
}
|
||||
|
||||
const coordsText = finalElements.map((el) => JSON.stringify(formatElement(el, opts.coordFormat))).join("\n");
|
||||
console.log(finalHead ? `${finalHead}\n\n## Coordinates\n${coordsText}` : coordsText);
|
||||
output = finalHead ? `${finalHead}\n\n## Coordinates\n${coordsText}` : coordsText;
|
||||
console.log(output);
|
||||
cleanup(meta2);
|
||||
} else {
|
||||
const { dataUrl } = await resolveImage(opts.imageSource, opts.isUrl);
|
||||
@@ -347,9 +409,23 @@ async function main() {
|
||||
detail: opts.detail,
|
||||
maxTokens: opts.maxTokens,
|
||||
size: null,
|
||||
history: opts.sessionHistory,
|
||||
}),
|
||||
);
|
||||
console.log(result);
|
||||
output = result;
|
||||
console.log(output);
|
||||
}
|
||||
|
||||
if (opts.session) {
|
||||
const state = sessionState || { model: config.model, turns: [] };
|
||||
state.turns.push({
|
||||
role: "user",
|
||||
text: prompt,
|
||||
images: currentSource ? [currentSource] : [],
|
||||
});
|
||||
state.turns.push({ role: "assistant", text: output, images: [] });
|
||||
const sessionFile = saveSession(opts.session, state);
|
||||
console.error(`会话 "${opts.session}" 已更新: ${sessionFile}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Vision failed:", err.message);
|
||||
|
||||
Reference in New Issue
Block a user