skills/qdrant__skills
1
0
Fork 0
mirror of https://github.com/qdrant/skills.git synced 2026-09-19 07:27:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Docker-based skill testing framework.

Browse Source
This commit is contained in:
István Zoltán Szabó
2026-07-28 16:13:23 +02:00
parent e24485f0e7
commit 4655665461
17 changed files with 2011 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.lycheeignore
+4
View File
@@ -1 +1,5 @@
https://docs.cursor.com/context/skills
# skill-test/README.md placeholder URLs (illustrative, not real endpoints)
https://skills.qdrant.tech/path/to/plugin.zip
https://skills.qdrant.tech/\.\.\./SKILL.md
skill-test/.dockerignore
+4
View File
@@ -0,0 +1,4 @@
.env
runs
.git
.DS_Store
skill-test/.env.example
+17
View File
@@ -0,0 +1,17 @@
# Copy to .env and fill in the values you want Docker to pass to Claude Code.
# Do not commit .env.
ANTHROPIC_API_KEY=
# Optional examples:
# ANTHROPIC_MODEL=sonnet
# ANTHROPIC_BASE_URL=
# ANTHROPIC_BETAS=
# Optional Bedrock/Vertex examples:
# CLAUDE_CODE_USE_BEDROCK=1
# AWS_ACCESS_KEY_ID=
# AWS_SECRET_ACCESS_KEY=
# AWS_REGION=
# CLAUDE_CODE_USE_VERTEX=1
# ANTHROPIC_VERTEX_PROJECT_ID=
skill-test/.gitignore
+8
View File
@@ -0,0 +1,8 @@
# Captured run outputs (generated, not source)
runs/
# Local credentials
.env
# OS
.DS_Store
skill-test/Dockerfile
+33
View File
@@ -0,0 +1,33 @@
ARG NODE_IMAGE=node:22-bookworm-slim
FROM ${NODE_IMAGE}
ARG CLAUDE_CODE_VERSION=latest
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
git \
jq \
python3 \
ripgrep \
tini \
&& rm -rf /var/lib/apt/lists/*
RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
RUN useradd --create-home --shell /bin/bash claude \
&& mkdir -p /workspace /runs \
&& chown -R claude:claude /workspace /runs
COPY container/run-claude-prompt.sh /usr/local/bin/run-claude-prompt
COPY container/run-claude-session.sh /usr/local/bin/run-claude-session
RUN chmod +x /usr/local/bin/run-claude-prompt /usr/local/bin/run-claude-session
USER claude
ENV HOME=/home/claude
WORKDIR /workspace
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["claude", "--version"]
skill-test/README.md
+312
View File
@@ -0,0 +1,312 @@
# Claude Code Skill Test Harness
This repo runs Claude Code inside a fresh Docker container for each prompt, captures
the answer, and avoids reusing local Claude state between tests.
## Why This Is Fresh
Each run starts a new container with a new container-local `HOME`. The host
`~/.claude` directory is not mounted. The container command also uses
`claude -p --no-session-persistence`, so Claude Code does not save a resumable
session for the prompt.
## Build
```bash
scripts/build-image.sh
```
The base image and package install come from the network, so transient Docker
Hub or npm timeouts can happen. The build script retries three times by default:
```bash
scripts/build-image.sh --attempts 5
```
If you already have a different Node image locally, or Docker Hub is struggling
with that exact tag, use another Debian-based Node image:
```bash
scripts/build-image.sh --node-image node:22-bookworm
```
To pin Claude Code:
```bash
scripts/build-image.sh --claude-code-version 2.1.89
```
## Auth
Use an API key for scripted runs:
Generate a Claude Platform API key at https://platform.claude.com/. Next, add this key to your local `.env` file:
```bash
cp .env.example .env
```
Then edit `.env` and set `ANTHROPIC_API_KEY`.
You can also skip `.env` and export credentials in your shell before running the
script.
If a run exits with `Not logged in · Please run /login`, the fresh container did
not receive usable credentials. Check that `.env` contains a non-empty
`ANTHROPIC_API_KEY`, or pass `--env-file /path/to/env`.
## Run A Smoke Test
```bash
scripts/run-claude-test.sh prompts/qdrant-smoke.md
```
Or build and run in one step:
```bash
scripts/run-claude-test.sh --build prompts/qdrant-smoke.md
```
Each run writes:
```text
runs/<run-id>/
metadata.json
prompt.md
readable.md
stderr.txt
stdout.txt
```
`readable.md` is generated automatically after each run. To regenerate it, or to
turn an older Claude Code `stream-json` output into a readable transcript:
```bash
scripts/render-claude-stdout.js runs/<run-id>
```
With no argument, it renders the newest run under `runs/`:
```bash
scripts/render-claude-stdout.js
```
To save the transcript:
```bash
scripts/render-claude-stdout.js runs/<run-id> --output runs/<run-id>/readable.md
```
## Run A JSON Test-Prompt
The prompt file may also be a JSON test-prompt (for example the files under
`evals/test-prompts/`) that carries the prompt plus scoring metadata:
```json
{
"name": "qdrant-hybrid-search",
"product_area": "hybrid search",
"skill_url": "https://skills.qdrant.tech/.../SKILL.md",
"prompt": "We run hybrid search (dense + sparse) inside one large collection ...",
"rubric": [ { "type": "must", "text": "..." } ]
}
```
Pass the `.json` file directly:
```bash
scripts/run-claude-test.sh ../evals/test-prompts/qdrant-hybrid-search.json
```
The runner validates that the file parses and has a non-empty string `prompt`
field, extracts that field, and sends only it to Claude Code. The run id is
derived from the test-prompt's `name` field (falling back to the file name if
`name` is missing), and the original JSON is copied to
`runs/<run-id>/test-prompt.json` so its `rubric`, `skill_url`, and
`product_area` are available for scoring alongside the transcript.
Reading a JSON test-prompt requires `jq` on the host (it extracts the `prompt`
field before the container starts). On macOS, install it with `brew install jq`.
The runner exits with a clear error if `jq` is missing.
## Run A Batch Of Test-Prompts
To run several test-prompts in one go, use the batch wrapper. Each argument is
either a file or a directory (every `*.json` inside it is run, sorted by name):
```bash
scripts/run-claude-test-batch.sh ../evals/test-prompts
```
Options placed before a literal `--` are forwarded verbatim to every underlying
`run-claude-test.sh` invocation:
```bash
scripts/run-claude-test-batch.sh --model sonnet --max-turns 20 -- \
../evals/test-prompts/qdrant-hybrid-search.json \
../evals/test-prompts/qdrant-tenant-scaling.json
```
Build the image once first (`scripts/build-image.sh`) rather than passing
`--build`, which would rebuild before every run. The batch continues past a
failing run, prints a pass/fail summary, and exits non-zero if any run failed.
## Test Local Skills
If you have a local skill directory containing `SKILL.md`:
```bash
scripts/run-claude-test.sh \
--skills-dir ../skills/qdrant-scaling \
prompts/qdrant-smoke.md
```
If you have a directory containing multiple skills, each child directory with a
`SKILL.md` is installed into the fresh container for that run.
## Test Plugin URLs
If `skills.qdrant.tech` provides a Claude Code plugin zip URL, pass it directly:
```bash
scripts/run-claude-test.sh \
--plugin-url https://skills.qdrant.tech/path/to/plugin.zip \
prompts/qdrant-smoke.md
```
Repeat `--plugin-url` for multiple plugin zips.
## Test Remote Skill Discovery
To test a prompt where the skill is not installed locally and Claude must reach
the remote URL itself:
```bash
scripts/run-claude-test.sh \
--permission-mode bypassPermissions \
--max-turns 20 \
prompts/qdrant-latency-remote-skill.md
```
This prompt contains:
```text
My search latency jumped from 80ms to 400ms p99 over the weekend. How do I figure out what changed? Use skills.qdrant.tech
```
Use `bypassPermissions` only in the disposable Docker container. It lets Claude
Code run commands such as `curl` to inspect `skills.qdrant.tech`; without that,
a non-interactive run may be unable to fetch the remote skill source and may
answer from general knowledge instead.
For an auditable transcript that can show whether Claude actually used a tool to
inspect the URL, add verbose output:
```bash
scripts/run-claude-test.sh \
--permission-mode bypassPermissions \
--max-turns 20 \
--extra-args "--verbose" \
prompts/qdrant-latency-remote-skill.md
```
## Interrogate Further
For an interactive same-instance investigation, start a disposable session:
```bash
scripts/run-claude-session.sh \
--skills-dir ../skills/qdrant-scaling \
prompts/qdrant-smoke.md
```
You can ask follow-up questions inside Claude Code. When you exit, the container
is removed, so the session does not leak into the next test.
For stricter auditability, create a second prompt and run it as a new test. To
preserve visible context, include the previous `stdout.txt` content in your
follow-up prompt file and run another fresh container.
## Permission Modes
Pass `--permission-mode MODE` to `run-claude-test.sh` or `run-claude-session.sh`
to set, for that single test run, which actions Claude Code may take without
stopping to ask you for approval. The runner validates the value and rejects
anything outside this list:
- `default` — Claude asks before each file edit, shell command, or network request; only reads run without a prompt. Shown as "Manual" in the CLI, and `manual` is an accepted alias.
- `acceptEdits` — Auto-approves file edits and common filesystem commands (`mkdir`, `touch`, `mv`, `cp`, etc.) inside the working directory; everything else still prompts.
- `plan` — Claude researches and proposes changes without editing anything; edits stay blocked until you approve a plan.
- `auto` — Runs without routine prompts while a background classifier blocks risky actions; requires a supported plan and model.
- `dontAsk` — Auto-denies anything not pre-approved, running only allow-listed tools and read-only commands, and never waits for input; best for unattended runs. In a non-interactive container there is no one to answer a permission prompt, so instead of stalling, `dontAsk` denies the call, hands the denial back to Claude, and lets the run continue to completion.
- `bypassPermissions` — Skips all permission checks so every tool call runs immediately. Use only inside an isolated container or VM.
Mode names are **case-sensitive**: pass them exactly as written above, for
example `dontAsk` (not `dontask` or `DontAsk`). The runner rejects any other
spelling.
For the full reference, see the Claude Code docs:
<https://code.claude.com/docs/en/permission-modes>.
## Useful Options
Flags may appear in any order relative to the prompt file, so
`run-claude-test.sh prompts/x.md --permission-mode plan` and
`run-claude-test.sh --permission-mode plan prompts/x.md` are equivalent. An
unexpected second positional argument is rejected rather than silently ignored.
```bash
scripts/run-claude-test.sh \
--model sonnet \
--max-turns 20 \
--max-budget-usd 1.00 \
--permission-mode auto \
prompts/qdrant-smoke.md
```
`auto` is the default permission mode here. Rather than hard-denying anything not
pre-approved the way `dontAsk` does, it lets Claude work through in-scope actions
without asking permission from the user while a background classifier still blocks
anything beyond the task's scope — a better fit for unattended runs that should get
real work done. For a stricter, fully locked-down run, pass
`--permission-mode dontAsk`, which only ever runs pre-approved tools.
Note one edge case for headless (`-p`) runs like these: in `auto` mode the run
**aborts** if the classifier blocks the same action 3 times in a row or 20 times
total, since there is no user to approve a fallback prompt. A test that repeatedly
attempts out-of-scope actions can therefore end early, where `dontAsk` would
deny each call and let the run continue to completion.
For tests that intentionally need Claude Code to execute commands or edit a
throwaway workspace, use a disposable workspace and pass a more permissive mode,
for example:
```bash
scripts/run-claude-test.sh \
--workspace ./fixtures/example-project \
--workspace-rw \
--permission-mode bypassPermissions \
prompts/my-agentic-test.md
```
## Choose Model Interactively
Instead of specifying a model name directly, use `--choose-model` to select from
a menu:
```bash
scripts/run-claude-test.sh --choose-model prompts/qdrant-smoke.md
```
This will prompt you:
```text
Select a Claude model:
1) haiku
2) sonnet
3) opus
#?
```
Type `1`, `2`, or `3` and press Enter. The test will run with your chosen model.
The selected model is recorded in the run's `metadata.json` for reference.
skill-test/container/run-claude-prompt.sh
Executable
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
set -Eeuo pipefail
PROMPT_FILE="${PROMPT_FILE:-/prompt.md}"
RUNS_DIR="${RUNS_DIR:-/runs}"
CLAUDE_RUN_ID="${CLAUDE_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}"
CLAUDE_WORKSPACE="${CLAUDE_WORKSPACE:-/workspace}"
CLAUDE_OUTPUT_FORMAT="${CLAUDE_OUTPUT_FORMAT:-text}"
CLAUDE_PERMISSION_MODE="${CLAUDE_PERMISSION_MODE:-auto}"
CLAUDE_MAX_TURNS="${CLAUDE_MAX_TURNS:-20}"
CLAUDE_MODEL="${CLAUDE_MODEL:-}"
CLAUDE_MAX_BUDGET_USD="${CLAUDE_MAX_BUDGET_USD:-}"
CLAUDE_PLUGIN_URLS="${CLAUDE_PLUGIN_URLS:-}"
CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}"
if [[ ! -f "$PROMPT_FILE" ]]; then
echo "Prompt file not found: $PROMPT_FILE" >&2
exit 64
fi
run_dir="$RUNS_DIR/$CLAUDE_RUN_ID"
mkdir -p "$run_dir" "$CLAUDE_WORKSPACE" "$HOME/.claude/skills"
cp "$PROMPT_FILE" "$run_dir/prompt.md"
install_skill_dir() {
local source_dir="$1"
local skill_name="$2"
if [[ -f "$source_dir/SKILL.md" ]]; then
mkdir -p "$HOME/.claude/skills/$skill_name"
cp -R "$source_dir/." "$HOME/.claude/skills/$skill_name/"
fi
}
if [[ -d /input-skills ]]; then
if [[ -f /input-skills/SKILL.md ]]; then
install_skill_dir /input-skills mounted-skill
else
shopt -s nullglob
for skill_dir in /input-skills/*; do
if [[ -d "$skill_dir" && -f "$skill_dir/SKILL.md" ]]; then
install_skill_dir "$skill_dir" "$(basename "$skill_dir")"
fi
done
fi
fi
args=(
-p
--no-session-persistence
--output-format "$CLAUDE_OUTPUT_FORMAT"
--permission-mode "$CLAUDE_PERMISSION_MODE"
)
if [[ "$CLAUDE_OUTPUT_FORMAT" == "stream-json" ]]; then
args+=(--verbose)
fi
if [[ -n "$CLAUDE_MAX_TURNS" ]]; then
args+=(--max-turns "$CLAUDE_MAX_TURNS")
fi
if [[ -n "$CLAUDE_MODEL" ]]; then
args+=(--model "$CLAUDE_MODEL")
fi
if [[ -n "$CLAUDE_MAX_BUDGET_USD" ]]; then
args+=(--max-budget-usd "$CLAUDE_MAX_BUDGET_USD")
fi
if [[ -d /input-plugin ]]; then
args+=(--plugin-dir /input-plugin)
fi
if [[ -n "$CLAUDE_PLUGIN_URLS" ]]; then
while IFS= read -r plugin_url; do
if [[ -n "$plugin_url" ]]; then
args+=(--plugin-url "$plugin_url")
fi
done <<< "$CLAUDE_PLUGIN_URLS"
fi
if [[ -n "$CLAUDE_EXTRA_ARGS" ]]; then
# Whitespace tokenization is intentional here so callers can pass advanced
# Claude Code flags without this wrapper needing to model every one.
read -r -a extra_args <<< "$CLAUDE_EXTRA_ARGS"
args+=("${extra_args[@]}")
fi
prompt="$(< "$PROMPT_FILE")"
started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
set +e
(
cd "$CLAUDE_WORKSPACE"
claude "${args[@]}" "$prompt"
) > "$run_dir/stdout.txt" 2> "$run_dir/stderr.txt"
status=$?
set -e
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
jq -n \
--arg run_id "$CLAUDE_RUN_ID" \
--arg started_at "$started_at" \
--arg finished_at "$finished_at" \
--arg output_format "$CLAUDE_OUTPUT_FORMAT" \
--arg permission_mode "$CLAUDE_PERMISSION_MODE" \
--arg max_turns "$CLAUDE_MAX_TURNS" \
--arg model "$CLAUDE_MODEL" \
--arg max_budget_usd "$CLAUDE_MAX_BUDGET_USD" \
--argjson exit_code "$status" \
'{
run_id: $run_id,
started_at: $started_at,
finished_at: $finished_at,
exit_code: $exit_code,
output_format: $output_format,
permission_mode: $permission_mode,
max_turns: $max_turns,
model: $model,
max_budget_usd: $max_budget_usd
}' > "$run_dir/metadata.json"
echo "Run directory: $run_dir"
echo "Exit code: $status"
exit "$status"
skill-test/container/run-claude-session.sh
Executable
+64
View File
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
set -Eeuo pipefail
PROMPT_FILE="${PROMPT_FILE:-}"
CLAUDE_WORKSPACE="${CLAUDE_WORKSPACE:-/workspace}"
CLAUDE_PERMISSION_MODE="${CLAUDE_PERMISSION_MODE:-auto}"
CLAUDE_MODEL="${CLAUDE_MODEL:-}"
CLAUDE_PLUGIN_URLS="${CLAUDE_PLUGIN_URLS:-}"
CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}"
mkdir -p "$CLAUDE_WORKSPACE" "$HOME/.claude/skills"
install_skill_dir() {
local source_dir="$1"
local skill_name="$2"
if [[ -f "$source_dir/SKILL.md" ]]; then
mkdir -p "$HOME/.claude/skills/$skill_name"
cp -R "$source_dir/." "$HOME/.claude/skills/$skill_name/"
fi
}
if [[ -d /input-skills ]]; then
if [[ -f /input-skills/SKILL.md ]]; then
install_skill_dir /input-skills mounted-skill
else
shopt -s nullglob
for skill_dir in /input-skills/*; do
if [[ -d "$skill_dir" && -f "$skill_dir/SKILL.md" ]]; then
install_skill_dir "$skill_dir" "$(basename "$skill_dir")"
fi
done
fi
fi
args=(--permission-mode "$CLAUDE_PERMISSION_MODE")
if [[ -n "$CLAUDE_MODEL" ]]; then
args+=(--model "$CLAUDE_MODEL")
fi
if [[ -d /input-plugin ]]; then
args+=(--plugin-dir /input-plugin)
fi
if [[ -n "$CLAUDE_PLUGIN_URLS" ]]; then
while IFS= read -r plugin_url; do
if [[ -n "$plugin_url" ]]; then
args+=(--plugin-url "$plugin_url")
fi
done <<< "$CLAUDE_PLUGIN_URLS"
fi
if [[ -n "$CLAUDE_EXTRA_ARGS" ]]; then
read -r -a extra_args <<< "$CLAUDE_EXTRA_ARGS"
args+=("${extra_args[@]}")
fi
if [[ -n "$PROMPT_FILE" && -f "$PROMPT_FILE" ]]; then
args+=("$(< "$PROMPT_FILE")")
fi
cd "$CLAUDE_WORKSPACE"
exec claude "${args[@]}"
skill-test/prompts/qdrant-latency-remote-skill.md
+3
View File
@@ -0,0 +1,3 @@
My search latency jumped from 80ms to 400ms p99 over the weekend. How do I figure out what changed?
Use skills.qdrant.tech.
skill-test/prompts/qdrant-migration-skill.md
+3
View File
@@ -0,0 +1,3 @@
I have two Qdrant instances:
cluster-A: <URL> API key: <API key> And cluster-B: <URL> API key: <API key>
I want you to migrate the collections from cluster-A to cluster-B. Use skills.qdrant.tech
skill-test/prompts/qdrant-migration.md
+3
View File
@@ -0,0 +1,3 @@
I have two Qdrant instances:
cluster-A: <URL> API key: <API key> And cluster-B: <URL> API key: <API key>
I want you to migrate the collections from cluster-A to cluster-B.
skill-test/prompts/qdrant-smoke.md
+7
View File
@@ -0,0 +1,7 @@
You are testing whether Qdrant-related Claude skills are available in this fresh container.
Please do three things:
1. State which Qdrant-related skill, plugin, or local instruction source appears to be loaded.
2. Explain how you would create a minimal Qdrant collection for text embeddings.
3. If no Qdrant skill appears to be loaded, say that plainly and answer from general knowledge only.
skill-test/scripts/build-image.sh
Executable
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}"
CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-latest}"
NODE_IMAGE="${CLAUDE_TEST_NODE_IMAGE:-node:22-bookworm-slim}"
BUILD_ATTEMPTS="${CLAUDE_TEST_BUILD_ATTEMPTS:-3}"
PULL_BASE="0"
usage() {
cat <<'USAGE'
Usage: scripts/build-image.sh [options]
Options:
--image NAME Docker image tag to build.
--claude-code-version VER Claude Code npm package version or "latest".
--node-image IMAGE Base Node image. Default: node:22-bookworm-slim.
--attempts N Retry docker build up to N times. Default: 3.
--pull Always attempt to pull a newer base image.
-h, --help Show this help.
Environment:
CLAUDE_TEST_IMAGE Default image tag.
CLAUDE_CODE_VERSION Default Claude Code package version.
CLAUDE_TEST_NODE_IMAGE Default base Node image.
CLAUDE_TEST_BUILD_ATTEMPTS Default retry count.
USAGE
}
require_value() {
local option="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
echo "Missing value for $option" >&2
exit 64
fi
}
while [[ $# -gt 0 ]]; do
case "$1" in
--image)
require_value "$1" "${2:-}"
IMAGE="$2"
shift 2
;;
--claude-code-version)
require_value "$1" "${2:-}"
CLAUDE_CODE_VERSION="$2"
shift 2
;;
--node-image)
require_value "$1" "${2:-}"
NODE_IMAGE="$2"
shift 2
;;
--attempts)
require_value "$1" "${2:-}"
BUILD_ATTEMPTS="$2"
shift 2
;;
--pull)
PULL_BASE="1"
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown option: $1" >&2
usage >&2
exit 64
;;
esac
done
if ! [[ "$BUILD_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then
echo "--attempts must be a positive integer" >&2
exit 64
fi
docker_args=(
build
--build-arg "NODE_IMAGE=$NODE_IMAGE"
--build-arg "CLAUDE_CODE_VERSION=$CLAUDE_CODE_VERSION"
-t "$IMAGE"
)
if [[ "$PULL_BASE" == "1" ]]; then
docker_args+=(--pull)
fi
docker_args+=("$REPO_ROOT")
for attempt in $(seq 1 "$BUILD_ATTEMPTS"); do
echo "Docker build attempt $attempt/$BUILD_ATTEMPTS using base $NODE_IMAGE"
if docker "${docker_args[@]}"; then
exit 0
fi
if [[ "$attempt" -lt "$BUILD_ATTEMPTS" ]]; then
sleep_seconds=$((attempt * 5))
echo "Build failed; retrying in ${sleep_seconds}s..." >&2
sleep "$sleep_seconds"
fi
done
echo "Docker build failed after $BUILD_ATTEMPTS attempts." >&2
exit 1
skill-test/scripts/render-claude-stdout.js
Executable
+440
View File
@@ -0,0 +1,440 @@
#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
const repoRoot = path.resolve(__dirname, "..");
function usage() {
console.log(`Usage: scripts/render-claude-stdout.js [RUN_DIR|STDOUT_FILE] [options]
Turns Claude Code stdout.txt into a readable transcript.
Arguments:
RUN_DIR|STDOUT_FILE A runs/<run-id> directory or stdout.txt file.
Defaults to the newest directory under runs/.
Options:
--output FILE Write transcript to FILE instead of stdout.
--show-tool-json Include full tool input JSON.
--show-raw Include unparsable raw lines.
-h, --help Show this help.
Examples:
scripts/render-claude-stdout.js
scripts/render-claude-stdout.js runs/20260616T125005Z-qdrant-latency-remote-skill
scripts/render-claude-stdout.js runs/20260616T125005Z-qdrant-latency-remote-skill/stdout.txt --output readable.txt
`);
}
function parseArgs(argv) {
const options = {
input: "",
output: "",
showToolJson: false,
showRaw: false,
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === "-h" || arg === "--help") {
usage();
process.exit(0);
}
if (arg === "--output") {
options.output = requireValue(arg, argv[i + 1]);
i += 1;
continue;
}
if (arg === "--show-tool-json") {
options.showToolJson = true;
continue;
}
if (arg === "--show-raw") {
options.showRaw = true;
continue;
}
if (arg.startsWith("-")) {
fail(`Unknown option: ${arg}`);
}
if (options.input) {
fail(`Only one input path is supported: got ${options.input} and ${arg}`);
}
options.input = arg;
}
return options;
}
function requireValue(option, value) {
if (!value) {
fail(`Missing value for ${option}`);
}
return value;
}
function fail(message) {
console.error(message);
process.exit(64);
}
function resolveInput(input) {
if (input) {
const absolute = path.resolve(input);
return stdoutPathFor(absolute);
}
const runsDir = path.join(repoRoot, "runs");
if (!fs.existsSync(runsDir)) {
fail("No input provided and runs/ does not exist.");
}
const newest = fs
.readdirSync(runsDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => {
const dir = path.join(runsDir, entry.name);
return { dir, mtimeMs: fs.statSync(dir).mtimeMs };
})
.sort((a, b) => b.mtimeMs - a.mtimeMs)[0];
if (!newest) {
fail("No input provided and runs/ has no run directories.");
}
return stdoutPathFor(newest.dir);
}
function stdoutPathFor(inputPath) {
if (!fs.existsSync(inputPath)) {
fail(`Input path not found: ${inputPath}`);
}
const stats = fs.statSync(inputPath);
if (stats.isDirectory()) {
const stdoutPath = path.join(inputPath, "stdout.txt");
if (!fs.existsSync(stdoutPath)) {
fail(`Run directory has no stdout.txt: ${inputPath}`);
}
return stdoutPath;
}
return inputPath;
}
function readJsonIfExists(filePath) {
if (!fs.existsSync(filePath)) {
return null;
}
try {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
} catch {
return null;
}
}
function oneLine(value) {
return String(value ?? "")
.replace(/\s+/g, " ")
.trim();
}
function indent(text, spaces = 2) {
const prefix = " ".repeat(spaces);
return String(text)
.split("\n")
.map((line) => (line ? `${prefix}${line}` : ""))
.join("\n");
}
function prettyJson(value) {
return JSON.stringify(value, null, 2);
}
function summarizeToolInput(input) {
if (!input || typeof input !== "object") {
return "";
}
const parts = [];
if (input.url) {
parts.push(`url=${input.url}`);
}
if (input.query) {
parts.push(`query=${input.query}`);
}
if (input.command) {
parts.push(`command=${input.command}`);
}
if (input.cmd) {
parts.push(`cmd=${input.cmd}`);
}
if (input.file_path) {
parts.push(`file=${input.file_path}`);
}
if (input.path) {
parts.push(`path=${input.path}`);
}
if (input.prompt) {
parts.push(`prompt=${oneLine(input.prompt).slice(0, 220)}`);
}
return parts.join("\n");
}
function getToolResultText(content) {
if (typeof content === "string") {
return content;
}
if (!Array.isArray(content)) {
return "";
}
return content
.map((item) => {
if (typeof item === "string") {
return item;
}
if (item && typeof item === "object") {
if (typeof item.content === "string") {
return item.content;
}
if (typeof item.text === "string") {
return item.text;
}
if (item.type === "tool_reference" && item.tool_name) {
return `tool reference: ${item.tool_name}`;
}
}
return "";
})
.filter(Boolean)
.join("\n");
}
function truncate(text, max = 1800) {
const value = String(text ?? "");
if (value.length <= max) {
return value;
}
return `${value.slice(0, max)}\n... [truncated ${value.length - max} chars]`;
}
function renderSystem(event, lines) {
if (event.subtype !== "init") {
lines.push(`## System: ${event.subtype || event.type}`);
return;
}
lines.push("## Claude Code Session");
lines.push("");
lines.push(`- Session: ${event.session_id || "unknown"}`);
lines.push(`- CWD: ${event.cwd || "unknown"}`);
lines.push(`- Model: ${event.model || "unknown"}`);
lines.push(`- Claude Code: ${event.claude_code_version || "unknown"}`);
lines.push(`- Permission mode: ${event.permissionMode || "unknown"}`);
lines.push(`- Auth source: ${event.apiKeySource || "unknown"}`);
if (Array.isArray(event.skills) && event.skills.length > 0) {
lines.push(`- Loaded skills: ${event.skills.join(", ")}`);
}
if (Array.isArray(event.plugins) && event.plugins.length > 0) {
lines.push(`- Plugins: ${event.plugins.join(", ")}`);
}
}
function renderAssistant(event, lines, options) {
const content = event.message && event.message.content;
if (!Array.isArray(content)) {
return;
}
for (const item of content) {
if (!item || typeof item !== "object") {
continue;
}
if (item.type === "text" && item.text) {
lines.push("");
lines.push("## Assistant");
lines.push("");
lines.push(item.text.trim());
continue;
}
if (item.type === "tool_use") {
lines.push("");
lines.push(`## Tool Use: ${item.name || "unknown"}`);
if (item.id) {
lines.push(`- ID: ${item.id}`);
}
const summary = summarizeToolInput(item.input);
if (summary) {
lines.push("");
lines.push(indent(summary));
}
if (options.showToolJson && item.input !== undefined) {
lines.push("");
lines.push("```json");
lines.push(prettyJson(item.input));
lines.push("```");
}
continue;
}
}
}
function renderUserToolResult(event, lines) {
const content = event.message && event.message.content;
if (!Array.isArray(content)) {
return;
}
for (const item of content) {
if (!item || item.type !== "tool_result") {
continue;
}
lines.push("");
lines.push("## Tool Result");
if (item.tool_use_id) {
lines.push(`- For: ${item.tool_use_id}`);
}
const result = event.tool_use_result;
if (result && typeof result === "object") {
const facts = [];
if (result.url) {
facts.push(`URL: ${result.url}`);
}
if (result.code || result.codeText) {
facts.push(`HTTP: ${[result.code, result.codeText].filter(Boolean).join(" ")}`);
}
if (result.durationMs !== undefined) {
facts.push(`Duration: ${result.durationMs}ms`);
}
if (facts.length > 0) {
lines.push("");
lines.push(indent(facts.join("\n")));
}
}
const text = getToolResultText(item.content);
if (text) {
lines.push("");
lines.push(indent(truncate(text)));
}
}
}
function renderResult(event, lines) {
lines.push("");
lines.push("## Run Result");
lines.push("");
lines.push(`- Status: ${event.is_error ? "error" : "success"}`);
if (event.subtype) {
lines.push(`- Subtype: ${event.subtype}`);
}
if (event.terminal_reason) {
lines.push(`- Terminal reason: ${event.terminal_reason}`);
}
if (event.stop_reason) {
lines.push(`- Stop reason: ${event.stop_reason}`);
}
if (event.num_turns !== undefined) {
lines.push(`- Turns: ${event.num_turns}`);
}
if (event.duration_ms !== undefined) {
lines.push(`- Duration: ${(event.duration_ms / 1000).toFixed(1)}s`);
}
if (event.total_cost_usd !== undefined) {
lines.push(`- Cost: $${Number(event.total_cost_usd).toFixed(6)}`);
}
if (Array.isArray(event.errors) && event.errors.length > 0) {
lines.push(`- Errors: ${event.errors.join("; ")}`);
}
}
function renderPlainText(raw, lines) {
const text = raw.trim();
if (!text) {
return;
}
lines.push("");
lines.push("## Output");
lines.push("");
lines.push(text);
}
function renderTranscript(stdoutPath, options) {
const runDir = path.dirname(stdoutPath);
const metadata = readJsonIfExists(path.join(runDir, "metadata.json"));
const prompt = fs.existsSync(path.join(runDir, "prompt.md"))
? fs.readFileSync(path.join(runDir, "prompt.md"), "utf8").trim()
: "";
const lines = [];
lines.push(`# Claude Code Transcript`);
lines.push("");
lines.push(`- Source: ${stdoutPath}`);
if (metadata) {
lines.push(`- Run ID: ${metadata.run_id || path.basename(runDir)}`);
lines.push(`- Exit code: ${metadata.exit_code}`);
lines.push(`- Output format: ${metadata.output_format}`);
lines.push(`- Max turns: ${metadata.max_turns}`);
}
if (prompt) {
lines.push("");
lines.push("## Prompt");
lines.push("");
lines.push(prompt);
}
const raw = fs.readFileSync(stdoutPath, "utf8");
const rawLines = raw.split(/\r?\n/).filter((line) => line.trim());
let parsedAny = false;
for (const line of rawLines) {
let event;
try {
event = JSON.parse(line);
parsedAny = true;
} catch {
if (options.showRaw) {
renderPlainText(line, lines);
} else if (!parsedAny && rawLines.length === 1) {
renderPlainText(line, lines);
}
continue;
}
if (event.type === "system") {
lines.push("");
renderSystem(event, lines);
} else if (event.type === "assistant") {
renderAssistant(event, lines, options);
} else if (event.type === "user") {
renderUserToolResult(event, lines);
} else if (event.type === "result") {
renderResult(event, lines);
} else if (options.showRaw) {
lines.push("");
lines.push(`## Raw Event: ${event.type || "unknown"}`);
lines.push("");
lines.push("```json");
lines.push(prettyJson(event));
lines.push("```");
}
}
return `${lines.join("\n")}\n`;
}
const options = parseArgs(process.argv.slice(2));
const stdoutPath = resolveInput(options.input);
const transcript = renderTranscript(stdoutPath, options);
if (options.output) {
fs.writeFileSync(path.resolve(options.output), transcript);
} else {
process.stdout.write(transcript);
}
skill-test/scripts/run-claude-session.sh
Executable
+303
View File
@@ -0,0 +1,303 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}"
ENV_FILE=""
if [[ -f "$REPO_ROOT/.env" ]]; then
ENV_FILE="$REPO_ROOT/.env"
fi
PROMPT_FILE=""
SKILLS_DIR=""
PLUGIN_DIR=""
WORKSPACE_DIR=""
WORKSPACE_MODE="ro"
PERMISSION_MODE="auto"
MODEL=""
CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}"
ALLOW_MISSING_AUTH="0"
PLUGIN_URLS=()
usage() {
cat <<'USAGE'
Usage: scripts/run-claude-session.sh [options] [PROMPT_FILE]
Starts an interactive Claude Code session inside a fresh disposable Docker
container. Use this when you want to ask follow-up questions in the same test
session. When you exit Claude, the container is removed.
Options:
--image NAME Docker image tag to run.
--env-file FILE Docker env-file with credentials. Defaults to .env if present.
--no-env-file Do not pass an env-file.
--skills-dir DIR Mount a local Claude skill or directory of skills.
--plugin-dir DIR Mount a local Claude Code plugin directory.
--plugin-url URL Load a plugin zip URL for this session. Repeatable.
--workspace DIR Mount a host workspace read-only at /workspace.
--workspace-rw Make --workspace read-write.
--permission-mode MODE default, acceptEdits, plan, auto, dontAsk, or bypassPermissions.
Default: auto.
--model MODEL Pass --model to Claude Code.
--extra-args "ARGS" Advanced Claude Code flags passed through by the container.
--allow-missing-auth Skip local auth preflight checks.
-h, --help Show this help.
Auth:
Put ANTHROPIC_API_KEY=... in .env, pass --env-file, or export it in your shell.
USAGE
}
abs_path() {
local path="$1"
if [[ "$path" == /* ]]; then
printf '%s\n' "$path"
else
printf '%s/%s\n' "$(pwd)" "$path"
fi
}
require_value() {
local option="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
echo "Missing value for $option" >&2
exit 64
fi
}
VALID_PERMISSION_MODES=(default manual acceptEdits plan auto dontAsk bypassPermissions)
validate_permission_mode() {
local mode="$1"
local valid
for valid in "${VALID_PERMISSION_MODES[@]}"; do
if [[ "$mode" == "$valid" ]]; then
return 0
fi
done
echo "Invalid --permission-mode: '$mode'" >&2
echo "Valid modes: ${VALID_PERMISSION_MODES[*]}" >&2
echo "See https://code.claude.com/docs/en/permission-modes" >&2
exit 64
}
env_file_has_value() {
local file="$1"
local name="$2"
[[ -f "$file" ]] || return 1
grep -Eq "^[[:space:]]*${name}[[:space:]]*=[[:space:]]*['\"]?[^'\"#[:space:]]" "$file"
}
auth_value_set() {
local name="$1"
if [[ -n "${!name:-}" ]]; then
return 0
fi
if [[ -n "$ENV_FILE" ]]; then
env_file_has_value "$(abs_path "$ENV_FILE")" "$name"
return $?
fi
return 1
}
auth_configured() {
if auth_value_set ANTHROPIC_API_KEY || auth_value_set ANTHROPIC_AUTH_TOKEN; then
return 0
fi
if auth_value_set CLAUDE_CODE_USE_BEDROCK && auth_value_set AWS_ACCESS_KEY_ID; then
return 0
fi
if auth_value_set CLAUDE_CODE_USE_VERTEX && auth_value_set ANTHROPIC_VERTEX_PROJECT_ID; then
return 0
fi
return 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--image)
require_value "$1" "${2:-}"
IMAGE="$2"
shift 2
;;
--env-file)
require_value "$1" "${2:-}"
ENV_FILE="$2"
shift 2
;;
--no-env-file)
ENV_FILE=""
shift
;;
--skills-dir)
require_value "$1" "${2:-}"
SKILLS_DIR="$2"
shift 2
;;
--plugin-dir)
require_value "$1" "${2:-}"
PLUGIN_DIR="$2"
shift 2
;;
--plugin-url)
require_value "$1" "${2:-}"
PLUGIN_URLS+=("$2")
shift 2
;;
--workspace)
require_value "$1" "${2:-}"
WORKSPACE_DIR="$2"
shift 2
;;
--workspace-rw)
WORKSPACE_MODE="rw"
shift
;;
--permission-mode)
require_value "$1" "${2:-}"
PERMISSION_MODE="$2"
shift 2
;;
--model)
require_value "$1" "${2:-}"
MODEL="$2"
shift 2
;;
--extra-args)
require_value "$1" "${2:-}"
CLAUDE_EXTRA_ARGS="$2"
shift 2
;;
--allow-missing-auth)
ALLOW_MISSING_AUTH="1"
shift
;;
-h|--help)
usage
exit 0
;;
--)
shift
break
;;
-*)
echo "Unknown option: $1" >&2
usage >&2
exit 64
;;
*)
if [[ -n "$PROMPT_FILE" ]]; then
echo "Unexpected extra argument: $1" >&2
usage >&2
exit 64
fi
PROMPT_FILE="$1"
shift
;;
esac
done
validate_permission_mode "$PERMISSION_MODE"
if [[ -n "$ENV_FILE" ]]; then
ENV_PATH="$(abs_path "$ENV_FILE")"
if [[ ! -f "$ENV_PATH" ]]; then
echo "Env file not found: $ENV_PATH" >&2
exit 66
fi
fi
if [[ "$ALLOW_MISSING_AUTH" != "1" ]] && ! auth_configured; then
cat >&2 <<'AUTH_ERROR'
No Claude Code auth was found.
Set ANTHROPIC_API_KEY in .env, pass --env-file with a non-empty key, or export
ANTHROPIC_API_KEY in your shell before running this script.
For Bedrock or Vertex, set the matching Claude Code mode variables as well.
Use --allow-missing-auth to skip this preflight if you intentionally want Claude
Code to fail or authenticate some other way inside the container.
AUTH_ERROR
exit 78
fi
docker_args=(
run
--rm
-it
-e "CLAUDE_WORKSPACE=/workspace"
-e "CLAUDE_PERMISSION_MODE=$PERMISSION_MODE"
-e "CLAUDE_MODEL=$MODEL"
-e "CLAUDE_EXTRA_ARGS=$CLAUDE_EXTRA_ARGS"
-e ANTHROPIC_API_KEY
-e ANTHROPIC_AUTH_TOKEN
-e ANTHROPIC_BASE_URL
-e ANTHROPIC_MODEL
-e ANTHROPIC_BETAS
-e ANTHROPIC_CUSTOM_HEADERS
-e CLAUDE_CODE_USE_BEDROCK
-e CLAUDE_CODE_USE_VERTEX
-e AWS_ACCESS_KEY_ID
-e AWS_SECRET_ACCESS_KEY
-e AWS_SESSION_TOKEN
-e AWS_REGION
-e ANTHROPIC_VERTEX_PROJECT_ID
-e GOOGLE_APPLICATION_CREDENTIALS
)
if [[ -n "$ENV_FILE" ]]; then
docker_args+=(--env-file "$ENV_PATH")
fi
if [[ -n "$PROMPT_FILE" ]]; then
PROMPT_PATH="$(abs_path "$PROMPT_FILE")"
if [[ ! -f "$PROMPT_PATH" ]]; then
echo "Prompt file not found: $PROMPT_PATH" >&2
exit 66
fi
docker_args+=(-e "PROMPT_FILE=/prompt.md" -v "$PROMPT_PATH:/prompt.md:ro")
fi
if [[ -n "$SKILLS_DIR" ]]; then
SKILLS_PATH="$(abs_path "$SKILLS_DIR")"
if [[ ! -d "$SKILLS_PATH" ]]; then
echo "Skills directory not found: $SKILLS_PATH" >&2
exit 66
fi
docker_args+=(-v "$SKILLS_PATH:/input-skills:ro")
fi
if [[ -n "$PLUGIN_DIR" ]]; then
PLUGIN_PATH="$(abs_path "$PLUGIN_DIR")"
if [[ ! -d "$PLUGIN_PATH" ]]; then
echo "Plugin directory not found: $PLUGIN_PATH" >&2
exit 66
fi
docker_args+=(-v "$PLUGIN_PATH:/input-plugin:ro")
fi
if [[ -n "$WORKSPACE_DIR" ]]; then
WORKSPACE_PATH="$(abs_path "$WORKSPACE_DIR")"
if [[ ! -d "$WORKSPACE_PATH" ]]; then
echo "Workspace directory not found: $WORKSPACE_PATH" >&2
exit 66
fi
docker_args+=(-v "$WORKSPACE_PATH:/workspace:$WORKSPACE_MODE")
fi
if [[ "${#PLUGIN_URLS[@]}" -gt 0 ]]; then
docker_args+=(-e "CLAUDE_PLUGIN_URLS=$(printf '%s\n' "${PLUGIN_URLS[@]}")")
fi
docker_args+=("$IMAGE" run-claude-session)
docker "${docker_args[@]}"
skill-test/scripts/run-claude-test-batch.sh
Executable
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
RUNNER="$SCRIPT_DIR/run-claude-test.sh"
usage() {
cat <<'USAGE'
Usage: scripts/run-claude-test-batch.sh [PASSTHROUGH_OPTS -- ] PATH [PATH...]
Runs run-claude-test.sh once per test-prompt. Each PATH is either:
- a file (.json test-prompt or a plain prompt file), or
- a directory (every *.json inside it is run, sorted by name).
Options placed before a literal `--` are forwarded verbatim to every
run-claude-test.sh invocation, e.g. --model, --skills-dir, --permission-mode.
With no `--`, all arguments are treated as PATHs.
The batch continues past a failing run and prints a pass/fail summary at the
end. It exits non-zero if any run failed.
Examples:
scripts/run-claude-test-batch.sh ../evals/test-prompts
scripts/run-claude-test-batch.sh --model sonnet -- a.json b.json
USAGE
}
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
usage
exit 0
fi
PASSTHROUGH=()
INPUTS=()
has_sep=0
for arg in "$@"; do
if [[ "$arg" == "--" ]]; then
has_sep=1
break
fi
done
if [[ "$has_sep" == "1" ]]; then
seen_sep=0
for arg in "$@"; do
if [[ "$seen_sep" == "0" && "$arg" == "--" ]]; then
seen_sep=1
continue
fi
if [[ "$seen_sep" == "0" ]]; then
PASSTHROUGH+=("$arg")
else
INPUTS+=("$arg")
fi
done
else
INPUTS=("$@")
fi
# Expand any directories into their *.json test-prompts (sorted, stable order).
FILES=()
for path in ${INPUTS[@]+"${INPUTS[@]}"}; do
if [[ -d "$path" ]]; then
shopt -s nullglob
matches=("$path"/*.json)
shopt -u nullglob
if [[ "${#matches[@]}" -eq 0 ]]; then
echo "No .json test-prompts in directory: $path" >&2
continue
fi
while IFS= read -r match; do
FILES+=("$match")
done < <(printf '%s\n' "${matches[@]}" | sort)
elif [[ -f "$path" ]]; then
FILES+=("$path")
else
echo "Path not found: $path" >&2
exit 66
fi
done
if [[ "${#FILES[@]}" -eq 0 ]]; then
echo "No test-prompts to run" >&2
usage >&2
exit 64
fi
total="${#FILES[@]}"
pass=0
fail=0
FAILED=()
i=0
for file in "${FILES[@]}"; do
i=$((i + 1))
echo "=== [$i/$total] $file ==="
if "$RUNNER" ${PASSTHROUGH[@]+"${PASSTHROUGH[@]}"} "$file"; then
pass=$((pass + 1))
else
status=$?
fail=$((fail + 1))
FAILED+=("$file")
echo "Run failed (exit $status): $file" >&2
fi
echo
done
echo "Batch complete: $pass passed, $fail failed (of $total)."
if [[ "$fail" -gt 0 ]]; then
printf 'Failed: %s\n' "${FAILED[@]}" >&2
exit 1
fi
skill-test/scripts/run-claude-test.sh
Executable
+458
View File
@@ -0,0 +1,458 @@
#!/usr/bin/env bash
set -Eeuo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}"
RUNS_DIR="${CLAUDE_TEST_RUNS_DIR:-runs}"
ENV_FILE=""
if [[ -f "$REPO_ROOT/.env" ]]; then
ENV_FILE="$REPO_ROOT/.env"
fi
PROMPT_FILE=""
SKILLS_DIR=""
PLUGIN_DIR=""
WORKSPACE_DIR=""
WORKSPACE_MODE="ro"
OUTPUT_FORMAT="stream-json"
PERMISSION_MODE="auto"
MAX_TURNS="20"
MODEL=""
MAX_BUDGET_USD=""
BUILD_IMAGE="0"
CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-latest}"
CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}"
ALLOW_MISSING_AUTH="0"
RENDER_TRANSCRIPT="1"
PLUGIN_URLS=()
usage() {
cat <<'USAGE'
Usage: scripts/run-claude-test.sh [options] PROMPT_FILE
Runs a fresh Docker container with Claude Code, sends PROMPT_FILE to `claude -p`,
and stores stdout, stderr, the prompt, and metadata under runs/<run-id>/.
PROMPT_FILE may be a plain prompt file (its whole contents are the prompt) or a
JSON test-prompt with a "prompt" field (that field is extracted and used, and the
JSON is copied to runs/<run-id>/test-prompt.json for scoring).
Options:
--build Build the Docker image before running.
--image NAME Docker image tag to run.
--claude-code-version VER Claude Code version to use when --build is set.
--runs-dir DIR Host directory for captured runs.
--env-file FILE Docker env-file with credentials. Defaults to .env if present.
--no-env-file Do not pass an env-file.
--skills-dir DIR Mount a local Claude skill or directory of skills.
--plugin-dir DIR Mount a local Claude Code plugin directory.
--plugin-url URL Load a plugin zip URL for this run. Repeatable.
--workspace DIR Mount a host workspace read-only at /workspace.
--workspace-rw Make --workspace read-write.
--output-format FORMAT text, json, or stream-json. Default: stream-json.
--permission-mode MODE default, acceptEdits, plan, auto, dontAsk, or bypassPermissions.
Default: auto.
--max-turns N Claude Code print-mode max turns. Default: 20.
--model MODEL Pass --model to Claude Code.
--choose-model Interactively choose a model from a menu.
--max-budget-usd USD Stop once this print-mode budget is reached.
--extra-args "ARGS" Advanced Claude Code flags passed through by the container.
--allow-missing-auth Skip local auth preflight checks.
--no-render Do not generate readable.md after the run.
-h, --help Show this help.
Auth:
Put ANTHROPIC_API_KEY=... in .env, pass --env-file, or export it in your shell.
The runner also forwards common Anthropic/Bedrock/Vertex env vars by name.
USAGE
}
abs_path() {
local path="$1"
if [[ "$path" == /* ]]; then
printf '%s\n' "$path"
else
printf '%s/%s\n' "$(pwd)" "$path"
fi
}
require_value() {
local option="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
echo "Missing value for $option" >&2
exit 64
fi
}
VALID_PERMISSION_MODES=(default manual acceptEdits plan auto dontAsk bypassPermissions)
validate_permission_mode() {
local mode="$1"
local valid
for valid in "${VALID_PERMISSION_MODES[@]}"; do
if [[ "$mode" == "$valid" ]]; then
return 0
fi
done
echo "Invalid --permission-mode: '$mode'" >&2
echo "Valid modes: ${VALID_PERMISSION_MODES[*]}" >&2
echo "See https://code.claude.com/docs/en/permission-modes" >&2
exit 64
}
env_file_has_value() {
local file="$1"
local name="$2"
[[ -f "$file" ]] || return 1
grep -Eq "^[[:space:]]*${name}[[:space:]]*=[[:space:]]*['\"]?[^'\"#[:space:]]" "$file"
}
auth_value_set() {
local name="$1"
if [[ -n "${!name:-}" ]]; then
return 0
fi
if [[ -n "$ENV_FILE" ]]; then
env_file_has_value "$(abs_path "$ENV_FILE")" "$name"
return $?
fi
return 1
}
auth_configured() {
if auth_value_set ANTHROPIC_API_KEY || auth_value_set ANTHROPIC_AUTH_TOKEN; then
return 0
fi
if auth_value_set CLAUDE_CODE_USE_BEDROCK && auth_value_set AWS_ACCESS_KEY_ID; then
return 0
fi
if auth_value_set CLAUDE_CODE_USE_VERTEX && auth_value_set ANTHROPIC_VERTEX_PROJECT_ID; then
return 0
fi
return 1
}
choose_model() {
local model
PS3="Select a Claude model: "
select model in "haiku" "sonnet" "opus"; do
if [[ -n "$model" ]]; then
echo "$model"
return 0
fi
done
return 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--build)
BUILD_IMAGE="1"
shift
;;
--image)
require_value "$1" "${2:-}"
IMAGE="$2"
shift 2
;;
--claude-code-version)
require_value "$1" "${2:-}"
CLAUDE_CODE_VERSION="$2"
shift 2
;;
--runs-dir)
require_value "$1" "${2:-}"
RUNS_DIR="$2"
shift 2
;;
--env-file)
require_value "$1" "${2:-}"
ENV_FILE="$2"
shift 2
;;
--no-env-file)
ENV_FILE=""
shift
;;
--skills-dir)
require_value "$1" "${2:-}"
SKILLS_DIR="$2"
shift 2
;;
--plugin-dir)
require_value "$1" "${2:-}"
PLUGIN_DIR="$2"
shift 2
;;
--plugin-url)
require_value "$1" "${2:-}"
PLUGIN_URLS+=("$2")
shift 2
;;
--workspace)
require_value "$1" "${2:-}"
WORKSPACE_DIR="$2"
shift 2
;;
--workspace-rw)
WORKSPACE_MODE="rw"
shift
;;
--output-format)
require_value "$1" "${2:-}"
OUTPUT_FORMAT="$2"
shift 2
;;
--permission-mode)
require_value "$1" "${2:-}"
PERMISSION_MODE="$2"
shift 2
;;
--max-turns)
require_value "$1" "${2:-}"
MAX_TURNS="$2"
shift 2
;;
--model)
require_value "$1" "${2:-}"
MODEL="$2"
shift 2
;;
--choose-model)
MODEL="$(choose_model)" || exit 1
shift
;;
--max-budget-usd)
require_value "$1" "${2:-}"
MAX_BUDGET_USD="$2"
shift 2
;;
--extra-args)
require_value "$1" "${2:-}"
CLAUDE_EXTRA_ARGS="$2"
shift 2
;;
--allow-missing-auth)
ALLOW_MISSING_AUTH="1"
shift
;;
--no-render)
RENDER_TRANSCRIPT="0"
shift
;;
-h|--help)
usage
exit 0
;;
--)
shift
break
;;
-*)
echo "Unknown option: $1" >&2
usage >&2
exit 64
;;
*)
if [[ -n "$PROMPT_FILE" ]]; then
echo "Unexpected extra argument: $1" >&2
usage >&2
exit 64
fi
PROMPT_FILE="$1"
shift
;;
esac
done
if [[ -z "$PROMPT_FILE" ]]; then
echo "Missing PROMPT_FILE" >&2
usage >&2
exit 64
fi
validate_permission_mode "$PERMISSION_MODE"
PROMPT_PATH="$(abs_path "$PROMPT_FILE")"
RUNS_PATH="$(abs_path "$RUNS_DIR")"
if [[ ! -f "$PROMPT_PATH" ]]; then
echo "Prompt file not found: $PROMPT_PATH" >&2
exit 66
fi
# Track the original input separately from the prompt actually mounted into the
# container. For a JSON test-prompt we extract its .prompt field into a temp
# Markdown file and mount that, while keeping the JSON for naming and metadata.
PROMPT_SOURCE_PATH="$PROMPT_PATH"
TEST_PROMPT_JSON=""
if [[ "$PROMPT_PATH" == *.json ]]; then
if ! command -v jq >/dev/null 2>&1; then
echo "jq is required to use a JSON test-prompt: $PROMPT_PATH" >&2
exit 69
fi
if ! jq -e . "$PROMPT_PATH" >/dev/null 2>&1; then
echo "Invalid JSON test-prompt: $PROMPT_PATH" >&2
exit 65
fi
if ! jq -e '(.prompt | type == "string") and (.prompt | length > 0)' \
"$PROMPT_PATH" >/dev/null 2>&1; then
echo "JSON test-prompt needs a non-empty string \"prompt\" field: $PROMPT_PATH" >&2
exit 65
fi
TEST_PROMPT_JSON="$PROMPT_PATH"
EXTRACTED_PROMPT_FILE="$(mktemp "${TMPDIR:-/tmp}/claude-test-prompt.XXXXXX")"
trap 'rm -f "$EXTRACTED_PROMPT_FILE"' EXIT
jq -r '.prompt' "$PROMPT_PATH" > "$EXTRACTED_PROMPT_FILE"
PROMPT_PATH="$EXTRACTED_PROMPT_FILE"
fi
if [[ -n "$ENV_FILE" ]]; then
ENV_PATH="$(abs_path "$ENV_FILE")"
if [[ ! -f "$ENV_PATH" ]]; then
echo "Env file not found: $ENV_PATH" >&2
exit 66
fi
fi
if [[ "$ALLOW_MISSING_AUTH" != "1" ]] && ! auth_configured; then
cat >&2 <<'AUTH_ERROR'
No Claude Code auth was found.
Set ANTHROPIC_API_KEY in .env, pass --env-file with a non-empty key, or export
ANTHROPIC_API_KEY in your shell before running this script.
For Bedrock or Vertex, set the matching Claude Code mode variables as well.
Use --allow-missing-auth to skip this preflight if you intentionally want Claude
Code to fail or authenticate some other way inside the container.
AUTH_ERROR
exit 78
fi
mkdir -p "$RUNS_PATH"
if [[ "$BUILD_IMAGE" == "1" ]]; then
"$SCRIPT_DIR/build-image.sh" \
--image "$IMAGE" \
--claude-code-version "$CLAUDE_CODE_VERSION"
fi
if [[ -n "$TEST_PROMPT_JSON" ]]; then
# Prefer the test-prompt's canonical .name over the file name so the run id
# tracks the test even if the file is renamed. Fall back to the file name if
# .name is missing or empty.
prompt_base="$(jq -r 'if (.name | type == "string") and (.name | length > 0) then .name else empty end' "$TEST_PROMPT_JSON")"
if [[ -z "$prompt_base" ]]; then
prompt_base="$(basename "$PROMPT_SOURCE_PATH")"
prompt_base="${prompt_base%.*}"
fi
else
prompt_base="$(basename "$PROMPT_SOURCE_PATH")"
prompt_base="${prompt_base%.*}"
fi
prompt_slug="$(printf '%s' "$prompt_base" | tr -c 'A-Za-z0-9._-' '_')"
run_id="$(date -u +%Y%m%dT%H%M%SZ)-$prompt_slug"
plugin_urls_joined=""
if [[ "${#PLUGIN_URLS[@]}" -gt 0 ]]; then
plugin_urls_joined="$(printf '%s\n' "${PLUGIN_URLS[@]}")"
fi
docker_args=(
run
--rm
--name "$run_id"
-e "PROMPT_FILE=/prompt.md"
-e "RUNS_DIR=/runs"
-e "CLAUDE_RUN_ID=$run_id"
-e "CLAUDE_WORKSPACE=/workspace"
-e "CLAUDE_OUTPUT_FORMAT=$OUTPUT_FORMAT"
-e "CLAUDE_PERMISSION_MODE=$PERMISSION_MODE"
-e "CLAUDE_MAX_TURNS=$MAX_TURNS"
-e "CLAUDE_MODEL=$MODEL"
-e "CLAUDE_MAX_BUDGET_USD=$MAX_BUDGET_USD"
-e "CLAUDE_PLUGIN_URLS=$plugin_urls_joined"
-e "CLAUDE_EXTRA_ARGS=$CLAUDE_EXTRA_ARGS"
-e ANTHROPIC_API_KEY
-e ANTHROPIC_AUTH_TOKEN
-e ANTHROPIC_BASE_URL
-e ANTHROPIC_MODEL
-e ANTHROPIC_BETAS
-e ANTHROPIC_CUSTOM_HEADERS
-e CLAUDE_CODE_USE_BEDROCK
-e CLAUDE_CODE_USE_VERTEX
-e AWS_ACCESS_KEY_ID
-e AWS_SECRET_ACCESS_KEY
-e AWS_SESSION_TOKEN
-e AWS_REGION
-e ANTHROPIC_VERTEX_PROJECT_ID
-e GOOGLE_APPLICATION_CREDENTIALS
-v "$PROMPT_PATH:/prompt.md:ro"
-v "$RUNS_PATH:/runs"
)
if [[ -n "$ENV_FILE" ]]; then
docker_args+=(--env-file "$ENV_PATH")
fi
if [[ -n "$SKILLS_DIR" ]]; then
SKILLS_PATH="$(abs_path "$SKILLS_DIR")"
if [[ ! -d "$SKILLS_PATH" ]]; then
echo "Skills directory not found: $SKILLS_PATH" >&2
exit 66
fi
docker_args+=(-v "$SKILLS_PATH:/input-skills:ro")
fi
if [[ -n "$PLUGIN_DIR" ]]; then
PLUGIN_PATH="$(abs_path "$PLUGIN_DIR")"
if [[ ! -d "$PLUGIN_PATH" ]]; then
echo "Plugin directory not found: $PLUGIN_PATH" >&2
exit 66
fi
docker_args+=(-v "$PLUGIN_PATH:/input-plugin:ro")
fi
if [[ -n "$WORKSPACE_DIR" ]]; then
WORKSPACE_PATH="$(abs_path "$WORKSPACE_DIR")"
if [[ ! -d "$WORKSPACE_PATH" ]]; then
echo "Workspace directory not found: $WORKSPACE_PATH" >&2
exit 66
fi
docker_args+=(-v "$WORKSPACE_PATH:/workspace:$WORKSPACE_MODE")
fi
docker_args+=("$IMAGE" run-claude-prompt)
echo "Starting Claude Code test: $run_id"
echo "Capturing output under: $RUNS_PATH/$run_id"
set +e
docker "${docker_args[@]}"
docker_status=$?
set -e
run_dir="$RUNS_PATH/$run_id"
if [[ -n "$TEST_PROMPT_JSON" && -d "$run_dir" ]]; then
cp "$TEST_PROMPT_JSON" "$run_dir/test-prompt.json"
fi
if [[ "$RENDER_TRANSCRIPT" == "1" && -f "$run_dir/stdout.txt" ]]; then
transcript_path="$run_dir/readable.md"
if "$SCRIPT_DIR/render-claude-stdout.js" "$run_dir" --output "$transcript_path"; then
echo "Readable transcript: $transcript_path"
else
echo "Warning: failed to render readable transcript for $run_dir" >&2
fi
fi
exit "$docker_status"