diff --git a/.lycheeignore b/.lycheeignore index 3bd135b..b7f2181 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1 +1,5 @@ https://docs.cursor.com/context/skills + +# skill-test/README.md placeholder URLs (illustrative, not real endpoints) +https://skills.qdrant.tech/path/to/plugin.zip +https://skills.qdrant.tech/\.\.\./SKILL.md diff --git a/skill-test/.dockerignore b/skill-test/.dockerignore new file mode 100644 index 0000000..23e8be0 --- /dev/null +++ b/skill-test/.dockerignore @@ -0,0 +1,4 @@ +.env +runs +.git +.DS_Store diff --git a/skill-test/.env.example b/skill-test/.env.example new file mode 100644 index 0000000..57fc7b2 --- /dev/null +++ b/skill-test/.env.example @@ -0,0 +1,17 @@ +# Copy to .env and fill in the values you want Docker to pass to Claude Code. +# Do not commit .env. + +ANTHROPIC_API_KEY= + +# Optional examples: +# ANTHROPIC_MODEL=sonnet +# ANTHROPIC_BASE_URL= +# ANTHROPIC_BETAS= + +# Optional Bedrock/Vertex examples: +# CLAUDE_CODE_USE_BEDROCK=1 +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_REGION= +# CLAUDE_CODE_USE_VERTEX=1 +# ANTHROPIC_VERTEX_PROJECT_ID= diff --git a/skill-test/.gitignore b/skill-test/.gitignore new file mode 100644 index 0000000..35acecd --- /dev/null +++ b/skill-test/.gitignore @@ -0,0 +1,8 @@ +# Captured run outputs (generated, not source) +runs/ + +# Local credentials +.env + +# OS +.DS_Store diff --git a/skill-test/Dockerfile b/skill-test/Dockerfile new file mode 100644 index 0000000..b80489c --- /dev/null +++ b/skill-test/Dockerfile @@ -0,0 +1,33 @@ +ARG NODE_IMAGE=node:22-bookworm-slim +FROM ${NODE_IMAGE} + +ARG CLAUDE_CODE_VERSION=latest + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + curl \ + git \ + jq \ + python3 \ + ripgrep \ + tini \ + && rm -rf /var/lib/apt/lists/* + +RUN npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" + +RUN useradd --create-home --shell /bin/bash claude \ + && mkdir -p /workspace /runs \ + && chown -R claude:claude /workspace /runs + +COPY container/run-claude-prompt.sh /usr/local/bin/run-claude-prompt +COPY container/run-claude-session.sh /usr/local/bin/run-claude-session +RUN chmod +x /usr/local/bin/run-claude-prompt /usr/local/bin/run-claude-session + +USER claude +ENV HOME=/home/claude +WORKDIR /workspace + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["claude", "--version"] diff --git a/skill-test/README.md b/skill-test/README.md new file mode 100644 index 0000000..497ab05 --- /dev/null +++ b/skill-test/README.md @@ -0,0 +1,312 @@ +# Claude Code Skill Test Harness + +This repo runs Claude Code inside a fresh Docker container for each prompt, captures +the answer, and avoids reusing local Claude state between tests. + +## Why This Is Fresh + +Each run starts a new container with a new container-local `HOME`. The host +`~/.claude` directory is not mounted. The container command also uses +`claude -p --no-session-persistence`, so Claude Code does not save a resumable +session for the prompt. + +## Build + +```bash +scripts/build-image.sh +``` + +The base image and package install come from the network, so transient Docker +Hub or npm timeouts can happen. The build script retries three times by default: + +```bash +scripts/build-image.sh --attempts 5 +``` + +If you already have a different Node image locally, or Docker Hub is struggling +with that exact tag, use another Debian-based Node image: + +```bash +scripts/build-image.sh --node-image node:22-bookworm +``` + +To pin Claude Code: + +```bash +scripts/build-image.sh --claude-code-version 2.1.89 +``` + +## Auth + +Use an API key for scripted runs: + +Generate a Claude Platform API key at https://platform.claude.com/. Next, add this key to your local `.env` file: + +```bash +cp .env.example .env +``` + +Then edit `.env` and set `ANTHROPIC_API_KEY`. + +You can also skip `.env` and export credentials in your shell before running the +script. + +If a run exits with `Not logged in · Please run /login`, the fresh container did +not receive usable credentials. Check that `.env` contains a non-empty +`ANTHROPIC_API_KEY`, or pass `--env-file /path/to/env`. + +## Run A Smoke Test + +```bash +scripts/run-claude-test.sh prompts/qdrant-smoke.md +``` + +Or build and run in one step: + +```bash +scripts/run-claude-test.sh --build prompts/qdrant-smoke.md +``` + +Each run writes: + +```text +runs// + metadata.json + prompt.md + readable.md + stderr.txt + stdout.txt +``` + +`readable.md` is generated automatically after each run. To regenerate it, or to +turn an older Claude Code `stream-json` output into a readable transcript: + +```bash +scripts/render-claude-stdout.js runs/ +``` + +With no argument, it renders the newest run under `runs/`: + +```bash +scripts/render-claude-stdout.js +``` + +To save the transcript: + +```bash +scripts/render-claude-stdout.js runs/ --output runs//readable.md +``` + +## Run A JSON Test-Prompt + +The prompt file may also be a JSON test-prompt (for example the files under +`evals/test-prompts/`) that carries the prompt plus scoring metadata: + +```json +{ + "name": "qdrant-hybrid-search", + "product_area": "hybrid search", + "skill_url": "https://skills.qdrant.tech/.../SKILL.md", + "prompt": "We run hybrid search (dense + sparse) inside one large collection ...", + "rubric": [ { "type": "must", "text": "..." } ] +} +``` + +Pass the `.json` file directly: + +```bash +scripts/run-claude-test.sh ../evals/test-prompts/qdrant-hybrid-search.json +``` + +The runner validates that the file parses and has a non-empty string `prompt` +field, extracts that field, and sends only it to Claude Code. The run id is +derived from the test-prompt's `name` field (falling back to the file name if +`name` is missing), and the original JSON is copied to +`runs//test-prompt.json` so its `rubric`, `skill_url`, and +`product_area` are available for scoring alongside the transcript. + +Reading a JSON test-prompt requires `jq` on the host (it extracts the `prompt` +field before the container starts). On macOS, install it with `brew install jq`. +The runner exits with a clear error if `jq` is missing. + +## Run A Batch Of Test-Prompts + +To run several test-prompts in one go, use the batch wrapper. Each argument is +either a file or a directory (every `*.json` inside it is run, sorted by name): + +```bash +scripts/run-claude-test-batch.sh ../evals/test-prompts +``` + +Options placed before a literal `--` are forwarded verbatim to every underlying +`run-claude-test.sh` invocation: + +```bash +scripts/run-claude-test-batch.sh --model sonnet --max-turns 20 -- \ + ../evals/test-prompts/qdrant-hybrid-search.json \ + ../evals/test-prompts/qdrant-tenant-scaling.json +``` + +Build the image once first (`scripts/build-image.sh`) rather than passing +`--build`, which would rebuild before every run. The batch continues past a +failing run, prints a pass/fail summary, and exits non-zero if any run failed. + +## Test Local Skills + +If you have a local skill directory containing `SKILL.md`: + +```bash +scripts/run-claude-test.sh \ + --skills-dir ../skills/qdrant-scaling \ + prompts/qdrant-smoke.md +``` + +If you have a directory containing multiple skills, each child directory with a +`SKILL.md` is installed into the fresh container for that run. + +## Test Plugin URLs + +If `skills.qdrant.tech` provides a Claude Code plugin zip URL, pass it directly: + +```bash +scripts/run-claude-test.sh \ + --plugin-url https://skills.qdrant.tech/path/to/plugin.zip \ + prompts/qdrant-smoke.md +``` + +Repeat `--plugin-url` for multiple plugin zips. + +## Test Remote Skill Discovery + +To test a prompt where the skill is not installed locally and Claude must reach +the remote URL itself: + +```bash +scripts/run-claude-test.sh \ + --permission-mode bypassPermissions \ + --max-turns 20 \ + prompts/qdrant-latency-remote-skill.md +``` + +This prompt contains: + +```text +My search latency jumped from 80ms to 400ms p99 over the weekend. How do I figure out what changed? Use skills.qdrant.tech +``` + +Use `bypassPermissions` only in the disposable Docker container. It lets Claude +Code run commands such as `curl` to inspect `skills.qdrant.tech`; without that, +a non-interactive run may be unable to fetch the remote skill source and may +answer from general knowledge instead. + +For an auditable transcript that can show whether Claude actually used a tool to +inspect the URL, add verbose output: + +```bash +scripts/run-claude-test.sh \ + --permission-mode bypassPermissions \ + --max-turns 20 \ + --extra-args "--verbose" \ + prompts/qdrant-latency-remote-skill.md +``` + +## Interrogate Further + +For an interactive same-instance investigation, start a disposable session: + +```bash +scripts/run-claude-session.sh \ + --skills-dir ../skills/qdrant-scaling \ + prompts/qdrant-smoke.md +``` + +You can ask follow-up questions inside Claude Code. When you exit, the container +is removed, so the session does not leak into the next test. + +For stricter auditability, create a second prompt and run it as a new test. To +preserve visible context, include the previous `stdout.txt` content in your +follow-up prompt file and run another fresh container. + +## Permission Modes + +Pass `--permission-mode MODE` to `run-claude-test.sh` or `run-claude-session.sh` +to set, for that single test run, which actions Claude Code may take without +stopping to ask you for approval. The runner validates the value and rejects +anything outside this list: + +- `default` — Claude asks before each file edit, shell command, or network request; only reads run without a prompt. Shown as "Manual" in the CLI, and `manual` is an accepted alias. +- `acceptEdits` — Auto-approves file edits and common filesystem commands (`mkdir`, `touch`, `mv`, `cp`, etc.) inside the working directory; everything else still prompts. +- `plan` — Claude researches and proposes changes without editing anything; edits stay blocked until you approve a plan. +- `auto` — Runs without routine prompts while a background classifier blocks risky actions; requires a supported plan and model. +- `dontAsk` — Auto-denies anything not pre-approved, running only allow-listed tools and read-only commands, and never waits for input; best for unattended runs. In a non-interactive container there is no one to answer a permission prompt, so instead of stalling, `dontAsk` denies the call, hands the denial back to Claude, and lets the run continue to completion. +- `bypassPermissions` — Skips all permission checks so every tool call runs immediately. Use only inside an isolated container or VM. + +Mode names are **case-sensitive**: pass them exactly as written above, for +example `dontAsk` (not `dontask` or `DontAsk`). The runner rejects any other +spelling. + +For the full reference, see the Claude Code docs: +. + +## Useful Options + +Flags may appear in any order relative to the prompt file, so +`run-claude-test.sh prompts/x.md --permission-mode plan` and +`run-claude-test.sh --permission-mode plan prompts/x.md` are equivalent. An +unexpected second positional argument is rejected rather than silently ignored. + +```bash +scripts/run-claude-test.sh \ + --model sonnet \ + --max-turns 20 \ + --max-budget-usd 1.00 \ + --permission-mode auto \ + prompts/qdrant-smoke.md +``` + +`auto` is the default permission mode here. Rather than hard-denying anything not +pre-approved the way `dontAsk` does, it lets Claude work through in-scope actions +without asking permission from the user while a background classifier still blocks +anything beyond the task's scope — a better fit for unattended runs that should get +real work done. For a stricter, fully locked-down run, pass +`--permission-mode dontAsk`, which only ever runs pre-approved tools. + +Note one edge case for headless (`-p`) runs like these: in `auto` mode the run +**aborts** if the classifier blocks the same action 3 times in a row or 20 times +total, since there is no user to approve a fallback prompt. A test that repeatedly +attempts out-of-scope actions can therefore end early, where `dontAsk` would +deny each call and let the run continue to completion. + +For tests that intentionally need Claude Code to execute commands or edit a +throwaway workspace, use a disposable workspace and pass a more permissive mode, +for example: + +```bash +scripts/run-claude-test.sh \ + --workspace ./fixtures/example-project \ + --workspace-rw \ + --permission-mode bypassPermissions \ + prompts/my-agentic-test.md +``` + +## Choose Model Interactively + +Instead of specifying a model name directly, use `--choose-model` to select from +a menu: + +```bash +scripts/run-claude-test.sh --choose-model prompts/qdrant-smoke.md +``` + +This will prompt you: + +```text +Select a Claude model: +1) haiku +2) sonnet +3) opus +#? +``` + +Type `1`, `2`, or `3` and press Enter. The test will run with your chosen model. +The selected model is recorded in the run's `metadata.json` for reference. diff --git a/skill-test/container/run-claude-prompt.sh b/skill-test/container/run-claude-prompt.sh new file mode 100755 index 0000000..37e3b21 --- /dev/null +++ b/skill-test/container/run-claude-prompt.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +PROMPT_FILE="${PROMPT_FILE:-/prompt.md}" +RUNS_DIR="${RUNS_DIR:-/runs}" +CLAUDE_RUN_ID="${CLAUDE_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}" +CLAUDE_WORKSPACE="${CLAUDE_WORKSPACE:-/workspace}" +CLAUDE_OUTPUT_FORMAT="${CLAUDE_OUTPUT_FORMAT:-text}" +CLAUDE_PERMISSION_MODE="${CLAUDE_PERMISSION_MODE:-auto}" +CLAUDE_MAX_TURNS="${CLAUDE_MAX_TURNS:-20}" +CLAUDE_MODEL="${CLAUDE_MODEL:-}" +CLAUDE_MAX_BUDGET_USD="${CLAUDE_MAX_BUDGET_USD:-}" +CLAUDE_PLUGIN_URLS="${CLAUDE_PLUGIN_URLS:-}" +CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}" + +if [[ ! -f "$PROMPT_FILE" ]]; then + echo "Prompt file not found: $PROMPT_FILE" >&2 + exit 64 +fi + +run_dir="$RUNS_DIR/$CLAUDE_RUN_ID" +mkdir -p "$run_dir" "$CLAUDE_WORKSPACE" "$HOME/.claude/skills" + +cp "$PROMPT_FILE" "$run_dir/prompt.md" + +install_skill_dir() { + local source_dir="$1" + local skill_name="$2" + + if [[ -f "$source_dir/SKILL.md" ]]; then + mkdir -p "$HOME/.claude/skills/$skill_name" + cp -R "$source_dir/." "$HOME/.claude/skills/$skill_name/" + fi +} + +if [[ -d /input-skills ]]; then + if [[ -f /input-skills/SKILL.md ]]; then + install_skill_dir /input-skills mounted-skill + else + shopt -s nullglob + for skill_dir in /input-skills/*; do + if [[ -d "$skill_dir" && -f "$skill_dir/SKILL.md" ]]; then + install_skill_dir "$skill_dir" "$(basename "$skill_dir")" + fi + done + fi +fi + +args=( + -p + --no-session-persistence + --output-format "$CLAUDE_OUTPUT_FORMAT" + --permission-mode "$CLAUDE_PERMISSION_MODE" +) + +if [[ "$CLAUDE_OUTPUT_FORMAT" == "stream-json" ]]; then + args+=(--verbose) +fi + +if [[ -n "$CLAUDE_MAX_TURNS" ]]; then + args+=(--max-turns "$CLAUDE_MAX_TURNS") +fi + +if [[ -n "$CLAUDE_MODEL" ]]; then + args+=(--model "$CLAUDE_MODEL") +fi + +if [[ -n "$CLAUDE_MAX_BUDGET_USD" ]]; then + args+=(--max-budget-usd "$CLAUDE_MAX_BUDGET_USD") +fi + +if [[ -d /input-plugin ]]; then + args+=(--plugin-dir /input-plugin) +fi + +if [[ -n "$CLAUDE_PLUGIN_URLS" ]]; then + while IFS= read -r plugin_url; do + if [[ -n "$plugin_url" ]]; then + args+=(--plugin-url "$plugin_url") + fi + done <<< "$CLAUDE_PLUGIN_URLS" +fi + +if [[ -n "$CLAUDE_EXTRA_ARGS" ]]; then + # Whitespace tokenization is intentional here so callers can pass advanced + # Claude Code flags without this wrapper needing to model every one. + read -r -a extra_args <<< "$CLAUDE_EXTRA_ARGS" + args+=("${extra_args[@]}") +fi + +prompt="$(< "$PROMPT_FILE")" +started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +set +e +( + cd "$CLAUDE_WORKSPACE" + claude "${args[@]}" "$prompt" +) > "$run_dir/stdout.txt" 2> "$run_dir/stderr.txt" +status=$? +set -e + +finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +jq -n \ + --arg run_id "$CLAUDE_RUN_ID" \ + --arg started_at "$started_at" \ + --arg finished_at "$finished_at" \ + --arg output_format "$CLAUDE_OUTPUT_FORMAT" \ + --arg permission_mode "$CLAUDE_PERMISSION_MODE" \ + --arg max_turns "$CLAUDE_MAX_TURNS" \ + --arg model "$CLAUDE_MODEL" \ + --arg max_budget_usd "$CLAUDE_MAX_BUDGET_USD" \ + --argjson exit_code "$status" \ + '{ + run_id: $run_id, + started_at: $started_at, + finished_at: $finished_at, + exit_code: $exit_code, + output_format: $output_format, + permission_mode: $permission_mode, + max_turns: $max_turns, + model: $model, + max_budget_usd: $max_budget_usd + }' > "$run_dir/metadata.json" + +echo "Run directory: $run_dir" +echo "Exit code: $status" +exit "$status" diff --git a/skill-test/container/run-claude-session.sh b/skill-test/container/run-claude-session.sh new file mode 100755 index 0000000..dd99276 --- /dev/null +++ b/skill-test/container/run-claude-session.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +PROMPT_FILE="${PROMPT_FILE:-}" +CLAUDE_WORKSPACE="${CLAUDE_WORKSPACE:-/workspace}" +CLAUDE_PERMISSION_MODE="${CLAUDE_PERMISSION_MODE:-auto}" +CLAUDE_MODEL="${CLAUDE_MODEL:-}" +CLAUDE_PLUGIN_URLS="${CLAUDE_PLUGIN_URLS:-}" +CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}" + +mkdir -p "$CLAUDE_WORKSPACE" "$HOME/.claude/skills" + +install_skill_dir() { + local source_dir="$1" + local skill_name="$2" + + if [[ -f "$source_dir/SKILL.md" ]]; then + mkdir -p "$HOME/.claude/skills/$skill_name" + cp -R "$source_dir/." "$HOME/.claude/skills/$skill_name/" + fi +} + +if [[ -d /input-skills ]]; then + if [[ -f /input-skills/SKILL.md ]]; then + install_skill_dir /input-skills mounted-skill + else + shopt -s nullglob + for skill_dir in /input-skills/*; do + if [[ -d "$skill_dir" && -f "$skill_dir/SKILL.md" ]]; then + install_skill_dir "$skill_dir" "$(basename "$skill_dir")" + fi + done + fi +fi + +args=(--permission-mode "$CLAUDE_PERMISSION_MODE") + +if [[ -n "$CLAUDE_MODEL" ]]; then + args+=(--model "$CLAUDE_MODEL") +fi + +if [[ -d /input-plugin ]]; then + args+=(--plugin-dir /input-plugin) +fi + +if [[ -n "$CLAUDE_PLUGIN_URLS" ]]; then + while IFS= read -r plugin_url; do + if [[ -n "$plugin_url" ]]; then + args+=(--plugin-url "$plugin_url") + fi + done <<< "$CLAUDE_PLUGIN_URLS" +fi + +if [[ -n "$CLAUDE_EXTRA_ARGS" ]]; then + read -r -a extra_args <<< "$CLAUDE_EXTRA_ARGS" + args+=("${extra_args[@]}") +fi + +if [[ -n "$PROMPT_FILE" && -f "$PROMPT_FILE" ]]; then + args+=("$(< "$PROMPT_FILE")") +fi + +cd "$CLAUDE_WORKSPACE" +exec claude "${args[@]}" diff --git a/skill-test/prompts/qdrant-latency-remote-skill.md b/skill-test/prompts/qdrant-latency-remote-skill.md new file mode 100644 index 0000000..41e3a32 --- /dev/null +++ b/skill-test/prompts/qdrant-latency-remote-skill.md @@ -0,0 +1,3 @@ +My search latency jumped from 80ms to 400ms p99 over the weekend. How do I figure out what changed? + +Use skills.qdrant.tech. diff --git a/skill-test/prompts/qdrant-migration-skill.md b/skill-test/prompts/qdrant-migration-skill.md new file mode 100644 index 0000000..3831598 --- /dev/null +++ b/skill-test/prompts/qdrant-migration-skill.md @@ -0,0 +1,3 @@ +I have two Qdrant instances: +cluster-A: API key: And cluster-B: API key: +I want you to migrate the collections from cluster-A to cluster-B. Use skills.qdrant.tech \ No newline at end of file diff --git a/skill-test/prompts/qdrant-migration.md b/skill-test/prompts/qdrant-migration.md new file mode 100644 index 0000000..0b9ce06 --- /dev/null +++ b/skill-test/prompts/qdrant-migration.md @@ -0,0 +1,3 @@ + I have two Qdrant instances: +cluster-A: API key: And cluster-B: API key: +I want you to migrate the collections from cluster-A to cluster-B. \ No newline at end of file diff --git a/skill-test/prompts/qdrant-smoke.md b/skill-test/prompts/qdrant-smoke.md new file mode 100644 index 0000000..28f79c5 --- /dev/null +++ b/skill-test/prompts/qdrant-smoke.md @@ -0,0 +1,7 @@ +You are testing whether Qdrant-related Claude skills are available in this fresh container. + +Please do three things: + +1. State which Qdrant-related skill, plugin, or local instruction source appears to be loaded. +2. Explain how you would create a minimal Qdrant collection for text embeddings. +3. If no Qdrant skill appears to be loaded, say that plainly and answer from general knowledge only. diff --git a/skill-test/scripts/build-image.sh b/skill-test/scripts/build-image.sh new file mode 100755 index 0000000..b992873 --- /dev/null +++ b/skill-test/scripts/build-image.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}" +CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-latest}" +NODE_IMAGE="${CLAUDE_TEST_NODE_IMAGE:-node:22-bookworm-slim}" +BUILD_ATTEMPTS="${CLAUDE_TEST_BUILD_ATTEMPTS:-3}" +PULL_BASE="0" + +usage() { + cat <<'USAGE' +Usage: scripts/build-image.sh [options] + +Options: + --image NAME Docker image tag to build. + --claude-code-version VER Claude Code npm package version or "latest". + --node-image IMAGE Base Node image. Default: node:22-bookworm-slim. + --attempts N Retry docker build up to N times. Default: 3. + --pull Always attempt to pull a newer base image. + -h, --help Show this help. + +Environment: + CLAUDE_TEST_IMAGE Default image tag. + CLAUDE_CODE_VERSION Default Claude Code package version. + CLAUDE_TEST_NODE_IMAGE Default base Node image. + CLAUDE_TEST_BUILD_ATTEMPTS Default retry count. +USAGE +} + +require_value() { + local option="$1" + local value="${2:-}" + if [[ -z "$value" ]]; then + echo "Missing value for $option" >&2 + exit 64 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --image) + require_value "$1" "${2:-}" + IMAGE="$2" + shift 2 + ;; + --claude-code-version) + require_value "$1" "${2:-}" + CLAUDE_CODE_VERSION="$2" + shift 2 + ;; + --node-image) + require_value "$1" "${2:-}" + NODE_IMAGE="$2" + shift 2 + ;; + --attempts) + require_value "$1" "${2:-}" + BUILD_ATTEMPTS="$2" + shift 2 + ;; + --pull) + PULL_BASE="1" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 64 + ;; + esac +done + +if ! [[ "$BUILD_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then + echo "--attempts must be a positive integer" >&2 + exit 64 +fi + +docker_args=( + build + --build-arg "NODE_IMAGE=$NODE_IMAGE" + --build-arg "CLAUDE_CODE_VERSION=$CLAUDE_CODE_VERSION" + -t "$IMAGE" +) + +if [[ "$PULL_BASE" == "1" ]]; then + docker_args+=(--pull) +fi + +docker_args+=("$REPO_ROOT") + +for attempt in $(seq 1 "$BUILD_ATTEMPTS"); do + echo "Docker build attempt $attempt/$BUILD_ATTEMPTS using base $NODE_IMAGE" + if docker "${docker_args[@]}"; then + exit 0 + fi + + if [[ "$attempt" -lt "$BUILD_ATTEMPTS" ]]; then + sleep_seconds=$((attempt * 5)) + echo "Build failed; retrying in ${sleep_seconds}s..." >&2 + sleep "$sleep_seconds" + fi +done + +echo "Docker build failed after $BUILD_ATTEMPTS attempts." >&2 +exit 1 diff --git a/skill-test/scripts/render-claude-stdout.js b/skill-test/scripts/render-claude-stdout.js new file mode 100755 index 0000000..f526490 --- /dev/null +++ b/skill-test/scripts/render-claude-stdout.js @@ -0,0 +1,440 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const path = require("path"); + +const repoRoot = path.resolve(__dirname, ".."); + +function usage() { + console.log(`Usage: scripts/render-claude-stdout.js [RUN_DIR|STDOUT_FILE] [options] + +Turns Claude Code stdout.txt into a readable transcript. + +Arguments: + RUN_DIR|STDOUT_FILE A runs/ directory or stdout.txt file. + Defaults to the newest directory under runs/. + +Options: + --output FILE Write transcript to FILE instead of stdout. + --show-tool-json Include full tool input JSON. + --show-raw Include unparsable raw lines. + -h, --help Show this help. + +Examples: + scripts/render-claude-stdout.js + scripts/render-claude-stdout.js runs/20260616T125005Z-qdrant-latency-remote-skill + scripts/render-claude-stdout.js runs/20260616T125005Z-qdrant-latency-remote-skill/stdout.txt --output readable.txt +`); +} + +function parseArgs(argv) { + const options = { + input: "", + output: "", + showToolJson: false, + showRaw: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "-h" || arg === "--help") { + usage(); + process.exit(0); + } + if (arg === "--output") { + options.output = requireValue(arg, argv[i + 1]); + i += 1; + continue; + } + if (arg === "--show-tool-json") { + options.showToolJson = true; + continue; + } + if (arg === "--show-raw") { + options.showRaw = true; + continue; + } + if (arg.startsWith("-")) { + fail(`Unknown option: ${arg}`); + } + if (options.input) { + fail(`Only one input path is supported: got ${options.input} and ${arg}`); + } + options.input = arg; + } + + return options; +} + +function requireValue(option, value) { + if (!value) { + fail(`Missing value for ${option}`); + } + return value; +} + +function fail(message) { + console.error(message); + process.exit(64); +} + +function resolveInput(input) { + if (input) { + const absolute = path.resolve(input); + return stdoutPathFor(absolute); + } + + const runsDir = path.join(repoRoot, "runs"); + if (!fs.existsSync(runsDir)) { + fail("No input provided and runs/ does not exist."); + } + + const newest = fs + .readdirSync(runsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const dir = path.join(runsDir, entry.name); + return { dir, mtimeMs: fs.statSync(dir).mtimeMs }; + }) + .sort((a, b) => b.mtimeMs - a.mtimeMs)[0]; + + if (!newest) { + fail("No input provided and runs/ has no run directories."); + } + + return stdoutPathFor(newest.dir); +} + +function stdoutPathFor(inputPath) { + if (!fs.existsSync(inputPath)) { + fail(`Input path not found: ${inputPath}`); + } + + const stats = fs.statSync(inputPath); + if (stats.isDirectory()) { + const stdoutPath = path.join(inputPath, "stdout.txt"); + if (!fs.existsSync(stdoutPath)) { + fail(`Run directory has no stdout.txt: ${inputPath}`); + } + return stdoutPath; + } + + return inputPath; +} + +function readJsonIfExists(filePath) { + if (!fs.existsSync(filePath)) { + return null; + } + + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return null; + } +} + +function oneLine(value) { + return String(value ?? "") + .replace(/\s+/g, " ") + .trim(); +} + +function indent(text, spaces = 2) { + const prefix = " ".repeat(spaces); + return String(text) + .split("\n") + .map((line) => (line ? `${prefix}${line}` : "")) + .join("\n"); +} + +function prettyJson(value) { + return JSON.stringify(value, null, 2); +} + +function summarizeToolInput(input) { + if (!input || typeof input !== "object") { + return ""; + } + + const parts = []; + if (input.url) { + parts.push(`url=${input.url}`); + } + if (input.query) { + parts.push(`query=${input.query}`); + } + if (input.command) { + parts.push(`command=${input.command}`); + } + if (input.cmd) { + parts.push(`cmd=${input.cmd}`); + } + if (input.file_path) { + parts.push(`file=${input.file_path}`); + } + if (input.path) { + parts.push(`path=${input.path}`); + } + if (input.prompt) { + parts.push(`prompt=${oneLine(input.prompt).slice(0, 220)}`); + } + return parts.join("\n"); +} + +function getToolResultText(content) { + if (typeof content === "string") { + return content; + } + if (!Array.isArray(content)) { + return ""; + } + + return content + .map((item) => { + if (typeof item === "string") { + return item; + } + if (item && typeof item === "object") { + if (typeof item.content === "string") { + return item.content; + } + if (typeof item.text === "string") { + return item.text; + } + if (item.type === "tool_reference" && item.tool_name) { + return `tool reference: ${item.tool_name}`; + } + } + return ""; + }) + .filter(Boolean) + .join("\n"); +} + +function truncate(text, max = 1800) { + const value = String(text ?? ""); + if (value.length <= max) { + return value; + } + return `${value.slice(0, max)}\n... [truncated ${value.length - max} chars]`; +} + +function renderSystem(event, lines) { + if (event.subtype !== "init") { + lines.push(`## System: ${event.subtype || event.type}`); + return; + } + + lines.push("## Claude Code Session"); + lines.push(""); + lines.push(`- Session: ${event.session_id || "unknown"}`); + lines.push(`- CWD: ${event.cwd || "unknown"}`); + lines.push(`- Model: ${event.model || "unknown"}`); + lines.push(`- Claude Code: ${event.claude_code_version || "unknown"}`); + lines.push(`- Permission mode: ${event.permissionMode || "unknown"}`); + lines.push(`- Auth source: ${event.apiKeySource || "unknown"}`); + if (Array.isArray(event.skills) && event.skills.length > 0) { + lines.push(`- Loaded skills: ${event.skills.join(", ")}`); + } + if (Array.isArray(event.plugins) && event.plugins.length > 0) { + lines.push(`- Plugins: ${event.plugins.join(", ")}`); + } +} + +function renderAssistant(event, lines, options) { + const content = event.message && event.message.content; + if (!Array.isArray(content)) { + return; + } + + for (const item of content) { + if (!item || typeof item !== "object") { + continue; + } + + if (item.type === "text" && item.text) { + lines.push(""); + lines.push("## Assistant"); + lines.push(""); + lines.push(item.text.trim()); + continue; + } + + if (item.type === "tool_use") { + lines.push(""); + lines.push(`## Tool Use: ${item.name || "unknown"}`); + if (item.id) { + lines.push(`- ID: ${item.id}`); + } + const summary = summarizeToolInput(item.input); + if (summary) { + lines.push(""); + lines.push(indent(summary)); + } + if (options.showToolJson && item.input !== undefined) { + lines.push(""); + lines.push("```json"); + lines.push(prettyJson(item.input)); + lines.push("```"); + } + continue; + } + } +} + +function renderUserToolResult(event, lines) { + const content = event.message && event.message.content; + if (!Array.isArray(content)) { + return; + } + + for (const item of content) { + if (!item || item.type !== "tool_result") { + continue; + } + + lines.push(""); + lines.push("## Tool Result"); + if (item.tool_use_id) { + lines.push(`- For: ${item.tool_use_id}`); + } + + const result = event.tool_use_result; + if (result && typeof result === "object") { + const facts = []; + if (result.url) { + facts.push(`URL: ${result.url}`); + } + if (result.code || result.codeText) { + facts.push(`HTTP: ${[result.code, result.codeText].filter(Boolean).join(" ")}`); + } + if (result.durationMs !== undefined) { + facts.push(`Duration: ${result.durationMs}ms`); + } + if (facts.length > 0) { + lines.push(""); + lines.push(indent(facts.join("\n"))); + } + } + + const text = getToolResultText(item.content); + if (text) { + lines.push(""); + lines.push(indent(truncate(text))); + } + } +} + +function renderResult(event, lines) { + lines.push(""); + lines.push("## Run Result"); + lines.push(""); + lines.push(`- Status: ${event.is_error ? "error" : "success"}`); + if (event.subtype) { + lines.push(`- Subtype: ${event.subtype}`); + } + if (event.terminal_reason) { + lines.push(`- Terminal reason: ${event.terminal_reason}`); + } + if (event.stop_reason) { + lines.push(`- Stop reason: ${event.stop_reason}`); + } + if (event.num_turns !== undefined) { + lines.push(`- Turns: ${event.num_turns}`); + } + if (event.duration_ms !== undefined) { + lines.push(`- Duration: ${(event.duration_ms / 1000).toFixed(1)}s`); + } + if (event.total_cost_usd !== undefined) { + lines.push(`- Cost: $${Number(event.total_cost_usd).toFixed(6)}`); + } + if (Array.isArray(event.errors) && event.errors.length > 0) { + lines.push(`- Errors: ${event.errors.join("; ")}`); + } +} + +function renderPlainText(raw, lines) { + const text = raw.trim(); + if (!text) { + return; + } + lines.push(""); + lines.push("## Output"); + lines.push(""); + lines.push(text); +} + +function renderTranscript(stdoutPath, options) { + const runDir = path.dirname(stdoutPath); + const metadata = readJsonIfExists(path.join(runDir, "metadata.json")); + const prompt = fs.existsSync(path.join(runDir, "prompt.md")) + ? fs.readFileSync(path.join(runDir, "prompt.md"), "utf8").trim() + : ""; + + const lines = []; + lines.push(`# Claude Code Transcript`); + lines.push(""); + lines.push(`- Source: ${stdoutPath}`); + if (metadata) { + lines.push(`- Run ID: ${metadata.run_id || path.basename(runDir)}`); + lines.push(`- Exit code: ${metadata.exit_code}`); + lines.push(`- Output format: ${metadata.output_format}`); + lines.push(`- Max turns: ${metadata.max_turns}`); + } + if (prompt) { + lines.push(""); + lines.push("## Prompt"); + lines.push(""); + lines.push(prompt); + } + + const raw = fs.readFileSync(stdoutPath, "utf8"); + const rawLines = raw.split(/\r?\n/).filter((line) => line.trim()); + let parsedAny = false; + + for (const line of rawLines) { + let event; + try { + event = JSON.parse(line); + parsedAny = true; + } catch { + if (options.showRaw) { + renderPlainText(line, lines); + } else if (!parsedAny && rawLines.length === 1) { + renderPlainText(line, lines); + } + continue; + } + + if (event.type === "system") { + lines.push(""); + renderSystem(event, lines); + } else if (event.type === "assistant") { + renderAssistant(event, lines, options); + } else if (event.type === "user") { + renderUserToolResult(event, lines); + } else if (event.type === "result") { + renderResult(event, lines); + } else if (options.showRaw) { + lines.push(""); + lines.push(`## Raw Event: ${event.type || "unknown"}`); + lines.push(""); + lines.push("```json"); + lines.push(prettyJson(event)); + lines.push("```"); + } + } + + return `${lines.join("\n")}\n`; +} + +const options = parseArgs(process.argv.slice(2)); +const stdoutPath = resolveInput(options.input); +const transcript = renderTranscript(stdoutPath, options); + +if (options.output) { + fs.writeFileSync(path.resolve(options.output), transcript); +} else { + process.stdout.write(transcript); +} diff --git a/skill-test/scripts/run-claude-session.sh b/skill-test/scripts/run-claude-session.sh new file mode 100755 index 0000000..8a930c2 --- /dev/null +++ b/skill-test/scripts/run-claude-session.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}" +ENV_FILE="" +if [[ -f "$REPO_ROOT/.env" ]]; then + ENV_FILE="$REPO_ROOT/.env" +fi + +PROMPT_FILE="" +SKILLS_DIR="" +PLUGIN_DIR="" +WORKSPACE_DIR="" +WORKSPACE_MODE="ro" +PERMISSION_MODE="auto" +MODEL="" +CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}" +ALLOW_MISSING_AUTH="0" +PLUGIN_URLS=() + +usage() { + cat <<'USAGE' +Usage: scripts/run-claude-session.sh [options] [PROMPT_FILE] + +Starts an interactive Claude Code session inside a fresh disposable Docker +container. Use this when you want to ask follow-up questions in the same test +session. When you exit Claude, the container is removed. + +Options: + --image NAME Docker image tag to run. + --env-file FILE Docker env-file with credentials. Defaults to .env if present. + --no-env-file Do not pass an env-file. + --skills-dir DIR Mount a local Claude skill or directory of skills. + --plugin-dir DIR Mount a local Claude Code plugin directory. + --plugin-url URL Load a plugin zip URL for this session. Repeatable. + --workspace DIR Mount a host workspace read-only at /workspace. + --workspace-rw Make --workspace read-write. + --permission-mode MODE default, acceptEdits, plan, auto, dontAsk, or bypassPermissions. + Default: auto. + --model MODEL Pass --model to Claude Code. + --extra-args "ARGS" Advanced Claude Code flags passed through by the container. + --allow-missing-auth Skip local auth preflight checks. + -h, --help Show this help. + +Auth: + Put ANTHROPIC_API_KEY=... in .env, pass --env-file, or export it in your shell. +USAGE +} + +abs_path() { + local path="$1" + if [[ "$path" == /* ]]; then + printf '%s\n' "$path" + else + printf '%s/%s\n' "$(pwd)" "$path" + fi +} + +require_value() { + local option="$1" + local value="${2:-}" + if [[ -z "$value" ]]; then + echo "Missing value for $option" >&2 + exit 64 + fi +} + +VALID_PERMISSION_MODES=(default manual acceptEdits plan auto dontAsk bypassPermissions) + +validate_permission_mode() { + local mode="$1" + local valid + for valid in "${VALID_PERMISSION_MODES[@]}"; do + if [[ "$mode" == "$valid" ]]; then + return 0 + fi + done + echo "Invalid --permission-mode: '$mode'" >&2 + echo "Valid modes: ${VALID_PERMISSION_MODES[*]}" >&2 + echo "See https://code.claude.com/docs/en/permission-modes" >&2 + exit 64 +} + +env_file_has_value() { + local file="$1" + local name="$2" + + [[ -f "$file" ]] || return 1 + grep -Eq "^[[:space:]]*${name}[[:space:]]*=[[:space:]]*['\"]?[^'\"#[:space:]]" "$file" +} + +auth_value_set() { + local name="$1" + + if [[ -n "${!name:-}" ]]; then + return 0 + fi + + if [[ -n "$ENV_FILE" ]]; then + env_file_has_value "$(abs_path "$ENV_FILE")" "$name" + return $? + fi + + return 1 +} + +auth_configured() { + if auth_value_set ANTHROPIC_API_KEY || auth_value_set ANTHROPIC_AUTH_TOKEN; then + return 0 + fi + + if auth_value_set CLAUDE_CODE_USE_BEDROCK && auth_value_set AWS_ACCESS_KEY_ID; then + return 0 + fi + + if auth_value_set CLAUDE_CODE_USE_VERTEX && auth_value_set ANTHROPIC_VERTEX_PROJECT_ID; then + return 0 + fi + + return 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --image) + require_value "$1" "${2:-}" + IMAGE="$2" + shift 2 + ;; + --env-file) + require_value "$1" "${2:-}" + ENV_FILE="$2" + shift 2 + ;; + --no-env-file) + ENV_FILE="" + shift + ;; + --skills-dir) + require_value "$1" "${2:-}" + SKILLS_DIR="$2" + shift 2 + ;; + --plugin-dir) + require_value "$1" "${2:-}" + PLUGIN_DIR="$2" + shift 2 + ;; + --plugin-url) + require_value "$1" "${2:-}" + PLUGIN_URLS+=("$2") + shift 2 + ;; + --workspace) + require_value "$1" "${2:-}" + WORKSPACE_DIR="$2" + shift 2 + ;; + --workspace-rw) + WORKSPACE_MODE="rw" + shift + ;; + --permission-mode) + require_value "$1" "${2:-}" + PERMISSION_MODE="$2" + shift 2 + ;; + --model) + require_value "$1" "${2:-}" + MODEL="$2" + shift 2 + ;; + --extra-args) + require_value "$1" "${2:-}" + CLAUDE_EXTRA_ARGS="$2" + shift 2 + ;; + --allow-missing-auth) + ALLOW_MISSING_AUTH="1" + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 64 + ;; + *) + if [[ -n "$PROMPT_FILE" ]]; then + echo "Unexpected extra argument: $1" >&2 + usage >&2 + exit 64 + fi + PROMPT_FILE="$1" + shift + ;; + esac +done + +validate_permission_mode "$PERMISSION_MODE" + +if [[ -n "$ENV_FILE" ]]; then + ENV_PATH="$(abs_path "$ENV_FILE")" + if [[ ! -f "$ENV_PATH" ]]; then + echo "Env file not found: $ENV_PATH" >&2 + exit 66 + fi +fi + +if [[ "$ALLOW_MISSING_AUTH" != "1" ]] && ! auth_configured; then + cat >&2 <<'AUTH_ERROR' +No Claude Code auth was found. + +Set ANTHROPIC_API_KEY in .env, pass --env-file with a non-empty key, or export +ANTHROPIC_API_KEY in your shell before running this script. + +For Bedrock or Vertex, set the matching Claude Code mode variables as well. +Use --allow-missing-auth to skip this preflight if you intentionally want Claude +Code to fail or authenticate some other way inside the container. +AUTH_ERROR + exit 78 +fi + +docker_args=( + run + --rm + -it + -e "CLAUDE_WORKSPACE=/workspace" + -e "CLAUDE_PERMISSION_MODE=$PERMISSION_MODE" + -e "CLAUDE_MODEL=$MODEL" + -e "CLAUDE_EXTRA_ARGS=$CLAUDE_EXTRA_ARGS" + -e ANTHROPIC_API_KEY + -e ANTHROPIC_AUTH_TOKEN + -e ANTHROPIC_BASE_URL + -e ANTHROPIC_MODEL + -e ANTHROPIC_BETAS + -e ANTHROPIC_CUSTOM_HEADERS + -e CLAUDE_CODE_USE_BEDROCK + -e CLAUDE_CODE_USE_VERTEX + -e AWS_ACCESS_KEY_ID + -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN + -e AWS_REGION + -e ANTHROPIC_VERTEX_PROJECT_ID + -e GOOGLE_APPLICATION_CREDENTIALS +) + +if [[ -n "$ENV_FILE" ]]; then + docker_args+=(--env-file "$ENV_PATH") +fi + +if [[ -n "$PROMPT_FILE" ]]; then + PROMPT_PATH="$(abs_path "$PROMPT_FILE")" + if [[ ! -f "$PROMPT_PATH" ]]; then + echo "Prompt file not found: $PROMPT_PATH" >&2 + exit 66 + fi + docker_args+=(-e "PROMPT_FILE=/prompt.md" -v "$PROMPT_PATH:/prompt.md:ro") +fi + +if [[ -n "$SKILLS_DIR" ]]; then + SKILLS_PATH="$(abs_path "$SKILLS_DIR")" + if [[ ! -d "$SKILLS_PATH" ]]; then + echo "Skills directory not found: $SKILLS_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$SKILLS_PATH:/input-skills:ro") +fi + +if [[ -n "$PLUGIN_DIR" ]]; then + PLUGIN_PATH="$(abs_path "$PLUGIN_DIR")" + if [[ ! -d "$PLUGIN_PATH" ]]; then + echo "Plugin directory not found: $PLUGIN_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$PLUGIN_PATH:/input-plugin:ro") +fi + +if [[ -n "$WORKSPACE_DIR" ]]; then + WORKSPACE_PATH="$(abs_path "$WORKSPACE_DIR")" + if [[ ! -d "$WORKSPACE_PATH" ]]; then + echo "Workspace directory not found: $WORKSPACE_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$WORKSPACE_PATH:/workspace:$WORKSPACE_MODE") +fi + +if [[ "${#PLUGIN_URLS[@]}" -gt 0 ]]; then + docker_args+=(-e "CLAUDE_PLUGIN_URLS=$(printf '%s\n' "${PLUGIN_URLS[@]}")") +fi + +docker_args+=("$IMAGE" run-claude-session) + +docker "${docker_args[@]}" diff --git a/skill-test/scripts/run-claude-test-batch.sh b/skill-test/scripts/run-claude-test-batch.sh new file mode 100755 index 0000000..be398e8 --- /dev/null +++ b/skill-test/scripts/run-claude-test-batch.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run-claude-test.sh" + +usage() { + cat <<'USAGE' +Usage: scripts/run-claude-test-batch.sh [PASSTHROUGH_OPTS -- ] PATH [PATH...] + +Runs run-claude-test.sh once per test-prompt. Each PATH is either: + - a file (.json test-prompt or a plain prompt file), or + - a directory (every *.json inside it is run, sorted by name). + +Options placed before a literal `--` are forwarded verbatim to every +run-claude-test.sh invocation, e.g. --model, --skills-dir, --permission-mode. +With no `--`, all arguments are treated as PATHs. + +The batch continues past a failing run and prints a pass/fail summary at the +end. It exits non-zero if any run failed. + +Examples: + scripts/run-claude-test-batch.sh ../evals/test-prompts + scripts/run-claude-test-batch.sh --model sonnet -- a.json b.json +USAGE +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +PASSTHROUGH=() +INPUTS=() + +has_sep=0 +for arg in "$@"; do + if [[ "$arg" == "--" ]]; then + has_sep=1 + break + fi +done + +if [[ "$has_sep" == "1" ]]; then + seen_sep=0 + for arg in "$@"; do + if [[ "$seen_sep" == "0" && "$arg" == "--" ]]; then + seen_sep=1 + continue + fi + if [[ "$seen_sep" == "0" ]]; then + PASSTHROUGH+=("$arg") + else + INPUTS+=("$arg") + fi + done +else + INPUTS=("$@") +fi + +# Expand any directories into their *.json test-prompts (sorted, stable order). +FILES=() +for path in ${INPUTS[@]+"${INPUTS[@]}"}; do + if [[ -d "$path" ]]; then + shopt -s nullglob + matches=("$path"/*.json) + shopt -u nullglob + if [[ "${#matches[@]}" -eq 0 ]]; then + echo "No .json test-prompts in directory: $path" >&2 + continue + fi + while IFS= read -r match; do + FILES+=("$match") + done < <(printf '%s\n' "${matches[@]}" | sort) + elif [[ -f "$path" ]]; then + FILES+=("$path") + else + echo "Path not found: $path" >&2 + exit 66 + fi +done + +if [[ "${#FILES[@]}" -eq 0 ]]; then + echo "No test-prompts to run" >&2 + usage >&2 + exit 64 +fi + +total="${#FILES[@]}" +pass=0 +fail=0 +FAILED=() + +i=0 +for file in "${FILES[@]}"; do + i=$((i + 1)) + echo "=== [$i/$total] $file ===" + if "$RUNNER" ${PASSTHROUGH[@]+"${PASSTHROUGH[@]}"} "$file"; then + pass=$((pass + 1)) + else + status=$? + fail=$((fail + 1)) + FAILED+=("$file") + echo "Run failed (exit $status): $file" >&2 + fi + echo +done + +echo "Batch complete: $pass passed, $fail failed (of $total)." +if [[ "$fail" -gt 0 ]]; then + printf 'Failed: %s\n' "${FAILED[@]}" >&2 + exit 1 +fi diff --git a/skill-test/scripts/run-claude-test.sh b/skill-test/scripts/run-claude-test.sh new file mode 100755 index 0000000..1467994 --- /dev/null +++ b/skill-test/scripts/run-claude-test.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +IMAGE="${CLAUDE_TEST_IMAGE:-claude-code-skill-test:latest}" +RUNS_DIR="${CLAUDE_TEST_RUNS_DIR:-runs}" +ENV_FILE="" +if [[ -f "$REPO_ROOT/.env" ]]; then + ENV_FILE="$REPO_ROOT/.env" +fi + +PROMPT_FILE="" +SKILLS_DIR="" +PLUGIN_DIR="" +WORKSPACE_DIR="" +WORKSPACE_MODE="ro" +OUTPUT_FORMAT="stream-json" +PERMISSION_MODE="auto" +MAX_TURNS="20" +MODEL="" +MAX_BUDGET_USD="" +BUILD_IMAGE="0" +CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-latest}" +CLAUDE_EXTRA_ARGS="${CLAUDE_EXTRA_ARGS:-}" +ALLOW_MISSING_AUTH="0" +RENDER_TRANSCRIPT="1" +PLUGIN_URLS=() + +usage() { + cat <<'USAGE' +Usage: scripts/run-claude-test.sh [options] PROMPT_FILE + +Runs a fresh Docker container with Claude Code, sends PROMPT_FILE to `claude -p`, +and stores stdout, stderr, the prompt, and metadata under runs//. + +PROMPT_FILE may be a plain prompt file (its whole contents are the prompt) or a +JSON test-prompt with a "prompt" field (that field is extracted and used, and the +JSON is copied to runs//test-prompt.json for scoring). + +Options: + --build Build the Docker image before running. + --image NAME Docker image tag to run. + --claude-code-version VER Claude Code version to use when --build is set. + --runs-dir DIR Host directory for captured runs. + --env-file FILE Docker env-file with credentials. Defaults to .env if present. + --no-env-file Do not pass an env-file. + --skills-dir DIR Mount a local Claude skill or directory of skills. + --plugin-dir DIR Mount a local Claude Code plugin directory. + --plugin-url URL Load a plugin zip URL for this run. Repeatable. + --workspace DIR Mount a host workspace read-only at /workspace. + --workspace-rw Make --workspace read-write. + --output-format FORMAT text, json, or stream-json. Default: stream-json. + --permission-mode MODE default, acceptEdits, plan, auto, dontAsk, or bypassPermissions. + Default: auto. + --max-turns N Claude Code print-mode max turns. Default: 20. + --model MODEL Pass --model to Claude Code. + --choose-model Interactively choose a model from a menu. + --max-budget-usd USD Stop once this print-mode budget is reached. + --extra-args "ARGS" Advanced Claude Code flags passed through by the container. + --allow-missing-auth Skip local auth preflight checks. + --no-render Do not generate readable.md after the run. + -h, --help Show this help. + +Auth: + Put ANTHROPIC_API_KEY=... in .env, pass --env-file, or export it in your shell. + The runner also forwards common Anthropic/Bedrock/Vertex env vars by name. +USAGE +} + +abs_path() { + local path="$1" + if [[ "$path" == /* ]]; then + printf '%s\n' "$path" + else + printf '%s/%s\n' "$(pwd)" "$path" + fi +} + +require_value() { + local option="$1" + local value="${2:-}" + if [[ -z "$value" ]]; then + echo "Missing value for $option" >&2 + exit 64 + fi +} + +VALID_PERMISSION_MODES=(default manual acceptEdits plan auto dontAsk bypassPermissions) + +validate_permission_mode() { + local mode="$1" + local valid + for valid in "${VALID_PERMISSION_MODES[@]}"; do + if [[ "$mode" == "$valid" ]]; then + return 0 + fi + done + echo "Invalid --permission-mode: '$mode'" >&2 + echo "Valid modes: ${VALID_PERMISSION_MODES[*]}" >&2 + echo "See https://code.claude.com/docs/en/permission-modes" >&2 + exit 64 +} + +env_file_has_value() { + local file="$1" + local name="$2" + + [[ -f "$file" ]] || return 1 + grep -Eq "^[[:space:]]*${name}[[:space:]]*=[[:space:]]*['\"]?[^'\"#[:space:]]" "$file" +} + +auth_value_set() { + local name="$1" + + if [[ -n "${!name:-}" ]]; then + return 0 + fi + + if [[ -n "$ENV_FILE" ]]; then + env_file_has_value "$(abs_path "$ENV_FILE")" "$name" + return $? + fi + + return 1 +} + +auth_configured() { + if auth_value_set ANTHROPIC_API_KEY || auth_value_set ANTHROPIC_AUTH_TOKEN; then + return 0 + fi + + if auth_value_set CLAUDE_CODE_USE_BEDROCK && auth_value_set AWS_ACCESS_KEY_ID; then + return 0 + fi + + if auth_value_set CLAUDE_CODE_USE_VERTEX && auth_value_set ANTHROPIC_VERTEX_PROJECT_ID; then + return 0 + fi + + return 1 +} + +choose_model() { + local model + PS3="Select a Claude model: " + select model in "haiku" "sonnet" "opus"; do + if [[ -n "$model" ]]; then + echo "$model" + return 0 + fi + done + return 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --build) + BUILD_IMAGE="1" + shift + ;; + --image) + require_value "$1" "${2:-}" + IMAGE="$2" + shift 2 + ;; + --claude-code-version) + require_value "$1" "${2:-}" + CLAUDE_CODE_VERSION="$2" + shift 2 + ;; + --runs-dir) + require_value "$1" "${2:-}" + RUNS_DIR="$2" + shift 2 + ;; + --env-file) + require_value "$1" "${2:-}" + ENV_FILE="$2" + shift 2 + ;; + --no-env-file) + ENV_FILE="" + shift + ;; + --skills-dir) + require_value "$1" "${2:-}" + SKILLS_DIR="$2" + shift 2 + ;; + --plugin-dir) + require_value "$1" "${2:-}" + PLUGIN_DIR="$2" + shift 2 + ;; + --plugin-url) + require_value "$1" "${2:-}" + PLUGIN_URLS+=("$2") + shift 2 + ;; + --workspace) + require_value "$1" "${2:-}" + WORKSPACE_DIR="$2" + shift 2 + ;; + --workspace-rw) + WORKSPACE_MODE="rw" + shift + ;; + --output-format) + require_value "$1" "${2:-}" + OUTPUT_FORMAT="$2" + shift 2 + ;; + --permission-mode) + require_value "$1" "${2:-}" + PERMISSION_MODE="$2" + shift 2 + ;; + --max-turns) + require_value "$1" "${2:-}" + MAX_TURNS="$2" + shift 2 + ;; + --model) + require_value "$1" "${2:-}" + MODEL="$2" + shift 2 + ;; + --choose-model) + MODEL="$(choose_model)" || exit 1 + shift + ;; + --max-budget-usd) + require_value "$1" "${2:-}" + MAX_BUDGET_USD="$2" + shift 2 + ;; + --extra-args) + require_value "$1" "${2:-}" + CLAUDE_EXTRA_ARGS="$2" + shift 2 + ;; + --allow-missing-auth) + ALLOW_MISSING_AUTH="1" + shift + ;; + --no-render) + RENDER_TRANSCRIPT="0" + shift + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 64 + ;; + *) + if [[ -n "$PROMPT_FILE" ]]; then + echo "Unexpected extra argument: $1" >&2 + usage >&2 + exit 64 + fi + PROMPT_FILE="$1" + shift + ;; + esac +done + +if [[ -z "$PROMPT_FILE" ]]; then + echo "Missing PROMPT_FILE" >&2 + usage >&2 + exit 64 +fi + +validate_permission_mode "$PERMISSION_MODE" + +PROMPT_PATH="$(abs_path "$PROMPT_FILE")" +RUNS_PATH="$(abs_path "$RUNS_DIR")" + +if [[ ! -f "$PROMPT_PATH" ]]; then + echo "Prompt file not found: $PROMPT_PATH" >&2 + exit 66 +fi + +# Track the original input separately from the prompt actually mounted into the +# container. For a JSON test-prompt we extract its .prompt field into a temp +# Markdown file and mount that, while keeping the JSON for naming and metadata. +PROMPT_SOURCE_PATH="$PROMPT_PATH" +TEST_PROMPT_JSON="" +if [[ "$PROMPT_PATH" == *.json ]]; then + if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to use a JSON test-prompt: $PROMPT_PATH" >&2 + exit 69 + fi + if ! jq -e . "$PROMPT_PATH" >/dev/null 2>&1; then + echo "Invalid JSON test-prompt: $PROMPT_PATH" >&2 + exit 65 + fi + if ! jq -e '(.prompt | type == "string") and (.prompt | length > 0)' \ + "$PROMPT_PATH" >/dev/null 2>&1; then + echo "JSON test-prompt needs a non-empty string \"prompt\" field: $PROMPT_PATH" >&2 + exit 65 + fi + TEST_PROMPT_JSON="$PROMPT_PATH" + EXTRACTED_PROMPT_FILE="$(mktemp "${TMPDIR:-/tmp}/claude-test-prompt.XXXXXX")" + trap 'rm -f "$EXTRACTED_PROMPT_FILE"' EXIT + jq -r '.prompt' "$PROMPT_PATH" > "$EXTRACTED_PROMPT_FILE" + PROMPT_PATH="$EXTRACTED_PROMPT_FILE" +fi + +if [[ -n "$ENV_FILE" ]]; then + ENV_PATH="$(abs_path "$ENV_FILE")" + if [[ ! -f "$ENV_PATH" ]]; then + echo "Env file not found: $ENV_PATH" >&2 + exit 66 + fi +fi + +if [[ "$ALLOW_MISSING_AUTH" != "1" ]] && ! auth_configured; then + cat >&2 <<'AUTH_ERROR' +No Claude Code auth was found. + +Set ANTHROPIC_API_KEY in .env, pass --env-file with a non-empty key, or export +ANTHROPIC_API_KEY in your shell before running this script. + +For Bedrock or Vertex, set the matching Claude Code mode variables as well. +Use --allow-missing-auth to skip this preflight if you intentionally want Claude +Code to fail or authenticate some other way inside the container. +AUTH_ERROR + exit 78 +fi + +mkdir -p "$RUNS_PATH" + +if [[ "$BUILD_IMAGE" == "1" ]]; then + "$SCRIPT_DIR/build-image.sh" \ + --image "$IMAGE" \ + --claude-code-version "$CLAUDE_CODE_VERSION" +fi + +if [[ -n "$TEST_PROMPT_JSON" ]]; then + # Prefer the test-prompt's canonical .name over the file name so the run id + # tracks the test even if the file is renamed. Fall back to the file name if + # .name is missing or empty. + prompt_base="$(jq -r 'if (.name | type == "string") and (.name | length > 0) then .name else empty end' "$TEST_PROMPT_JSON")" + if [[ -z "$prompt_base" ]]; then + prompt_base="$(basename "$PROMPT_SOURCE_PATH")" + prompt_base="${prompt_base%.*}" + fi +else + prompt_base="$(basename "$PROMPT_SOURCE_PATH")" + prompt_base="${prompt_base%.*}" +fi +prompt_slug="$(printf '%s' "$prompt_base" | tr -c 'A-Za-z0-9._-' '_')" +run_id="$(date -u +%Y%m%dT%H%M%SZ)-$prompt_slug" + +plugin_urls_joined="" +if [[ "${#PLUGIN_URLS[@]}" -gt 0 ]]; then + plugin_urls_joined="$(printf '%s\n' "${PLUGIN_URLS[@]}")" +fi + +docker_args=( + run + --rm + --name "$run_id" + -e "PROMPT_FILE=/prompt.md" + -e "RUNS_DIR=/runs" + -e "CLAUDE_RUN_ID=$run_id" + -e "CLAUDE_WORKSPACE=/workspace" + -e "CLAUDE_OUTPUT_FORMAT=$OUTPUT_FORMAT" + -e "CLAUDE_PERMISSION_MODE=$PERMISSION_MODE" + -e "CLAUDE_MAX_TURNS=$MAX_TURNS" + -e "CLAUDE_MODEL=$MODEL" + -e "CLAUDE_MAX_BUDGET_USD=$MAX_BUDGET_USD" + -e "CLAUDE_PLUGIN_URLS=$plugin_urls_joined" + -e "CLAUDE_EXTRA_ARGS=$CLAUDE_EXTRA_ARGS" + -e ANTHROPIC_API_KEY + -e ANTHROPIC_AUTH_TOKEN + -e ANTHROPIC_BASE_URL + -e ANTHROPIC_MODEL + -e ANTHROPIC_BETAS + -e ANTHROPIC_CUSTOM_HEADERS + -e CLAUDE_CODE_USE_BEDROCK + -e CLAUDE_CODE_USE_VERTEX + -e AWS_ACCESS_KEY_ID + -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN + -e AWS_REGION + -e ANTHROPIC_VERTEX_PROJECT_ID + -e GOOGLE_APPLICATION_CREDENTIALS + -v "$PROMPT_PATH:/prompt.md:ro" + -v "$RUNS_PATH:/runs" +) + +if [[ -n "$ENV_FILE" ]]; then + docker_args+=(--env-file "$ENV_PATH") +fi + +if [[ -n "$SKILLS_DIR" ]]; then + SKILLS_PATH="$(abs_path "$SKILLS_DIR")" + if [[ ! -d "$SKILLS_PATH" ]]; then + echo "Skills directory not found: $SKILLS_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$SKILLS_PATH:/input-skills:ro") +fi + +if [[ -n "$PLUGIN_DIR" ]]; then + PLUGIN_PATH="$(abs_path "$PLUGIN_DIR")" + if [[ ! -d "$PLUGIN_PATH" ]]; then + echo "Plugin directory not found: $PLUGIN_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$PLUGIN_PATH:/input-plugin:ro") +fi + +if [[ -n "$WORKSPACE_DIR" ]]; then + WORKSPACE_PATH="$(abs_path "$WORKSPACE_DIR")" + if [[ ! -d "$WORKSPACE_PATH" ]]; then + echo "Workspace directory not found: $WORKSPACE_PATH" >&2 + exit 66 + fi + docker_args+=(-v "$WORKSPACE_PATH:/workspace:$WORKSPACE_MODE") +fi + +docker_args+=("$IMAGE" run-claude-prompt) + +echo "Starting Claude Code test: $run_id" +echo "Capturing output under: $RUNS_PATH/$run_id" + +set +e +docker "${docker_args[@]}" +docker_status=$? +set -e + +run_dir="$RUNS_PATH/$run_id" +if [[ -n "$TEST_PROMPT_JSON" && -d "$run_dir" ]]; then + cp "$TEST_PROMPT_JSON" "$run_dir/test-prompt.json" +fi + +if [[ "$RENDER_TRANSCRIPT" == "1" && -f "$run_dir/stdout.txt" ]]; then + transcript_path="$run_dir/readable.md" + if "$SCRIPT_DIR/render-claude-stdout.js" "$run_dir" --output "$transcript_path"; then + echo "Readable transcript: $transcript_path" + else + echo "Warning: failed to render readable transcript for $run_dir" >&2 + fi +fi + +exit "$docker_status"