diff --git a/.claude/skills/qe-browser/evals/qe-browser.yaml b/.claude/skills/qe-browser/evals/qe-browser.yaml index 4270335f..7bcbde69 100644 --- a/.claude/skills/qe-browser/evals/qe-browser.yaml +++ b/.claude/skills/qe-browser/evals/qe-browser.yaml @@ -52,26 +52,18 @@ setup: optional_tools: - pixelmatch - pngjs - # NOTE: this yaml deliberately uses ONLY pinned public fixtures - # (httpbin.org/*) so it can be run end-to-end by CommandEvalRunner without - # any prerequisite services. Tests that need a local poisoned-HTML fixture - # (the check-injection severity path) live in scripts/smoke-test.sh, which - # starts fixtures/serve-skills.js out of band. + # The CI workflow starts fixtures/serve-skills.js on loopback before this + # suite runs. Keeping browser targets local removes rate limits and makes + # every page stable across repeated visual comparisons. fixtures: - public_pinned: - # Pinned public endpoints — chosen because they're stable, well-known, and - # serve predictable forms / HTML. Per feedback_no_unverified_failure_modes, - # these are the canonical "does the tool actually work" fixtures. - httpbin_form: - url: "https://httpbin.org/forms/post" - description: "Classic simple form — custname, custtel, custemail, size, toppings" - httpbin_html: - url: "https://httpbin.org/html" - description: "Static HTML page with known headings" - httpbin_status_404: - url: "https://httpbin.org/status/404" - description: "Known 404 for testing no_failed_requests" + local: + form: + url: "http://127.0.0.1:8088/fixtures/form.html" + description: "Deterministic form with an email input and submit button" + content: + url: "http://127.0.0.1:8088/fixtures/content.html" + description: "Deterministic static page with a known heading" test_cases: # -------- assert.js -------- @@ -81,10 +73,10 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/forms/post" + - "vibium --headless go http://127.0.0.1:8088/fixtures/form.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ - '[{"kind": "url_contains", "text": "httpbin.org/forms"}]' + '[{"kind": "url_contains", "text": "/fixtures/form.html"}]' expected: exit_code: 0 json_fields: @@ -98,7 +90,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ '[{"kind": "selector_visible", "selector": "h1"}]' @@ -113,7 +105,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ '[{"kind": "url_contains", "text": "this-does-not-exist"}]' @@ -132,10 +124,10 @@ test_cases: command: | node .claude/skills/qe-browser/scripts/batch.js --steps \ '[ - {"action": "go", "url": "https://httpbin.org/html"}, + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"}, {"action": "wait_load"}, {"action": "assert", "checks": [ - {"kind": "url_contains", "text": "/html"}, + {"kind": "url_contains", "text": "/fixtures/content.html"}, {"kind": "selector_visible", "selector": "h1"} ]} ]' --summary-only @@ -154,9 +146,11 @@ test_cases: command: | node .claude/skills/qe-browser/scripts/batch.js --steps \ '[ - {"action": "go", "url": "https://httpbin.org/html"}, - {"action": "click", "selector": "#does-not-exist"}, - {"action": "go", "url": "https://httpbin.org/forms/post"} + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"}, + {"action": "assert", "checks": [ + {"kind": "url_contains", "text": "/does-not-exist"} + ]}, + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/form.html"} ]' expected: exit_code: 1 @@ -177,7 +171,7 @@ test_cases: # different dimensions (768×654 vs 765×672 observed), making the # pixel-diff in tc007 spuriously fail. Mirrors scripts/smoke-test.sh. - "vibium --headless viewport 1280 720" - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" - "rm -rf .aqe/visual-baselines/eval_httpbin_html*" command: | node .claude/skills/qe-browser/scripts/visual-diff.js \ @@ -195,7 +189,7 @@ test_cases: input: setup: - "vibium --headless viewport 1280 720" - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/visual-diff.js \ --name eval_httpbin_html --threshold 0.02 @@ -212,7 +206,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/check-injection.js --include-hidden expected: @@ -221,15 +215,11 @@ test_cases: ".status": "success" ".output.checkInjection.severity": "none" - # GAP: the "poisoned-page detected with severity>=high" contract needs a - # local fixture (fixtures/injection-poisoned.html) served by - # fixtures/serve-skills.js. That's out of scope for this yaml — we keep - # CommandEvalRunner dependency-free so it can run anywhere httpbin.org - # is reachable. Coverage of the high-severity path is currently + # The "poisoned-page detected with severity>=high" contract is currently + # asserted outside this command suite. Coverage of the high-severity path is # only asserted by unit tests on check-injection.js (see # tests/unit/scripts/qe-browser-check-injection.test.ts). Follow-up: - # either teach CommandEvalRunner to spawn the fixture server, or add a - # tc009 to scripts/smoke-test.sh that starts/stops it out of band. + # add tc009 against fixtures/injection-poisoned.html. # -------- intent-score.js -------- - id: tc010_intent_submit_form_on_httpbin @@ -238,7 +228,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/forms/post" + - "vibium --headless go http://127.0.0.1:8088/fixtures/form.html" command: | node .claude/skills/qe-browser/scripts/intent-score.js \ --intent submit_form @@ -255,7 +245,7 @@ test_cases: priority: medium input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/intent-score.js --intent fill_email expected: @@ -270,5 +260,4 @@ validation: notes: | Evaluation assumes: - `vibium` v26.3.x+ is on PATH (from `aqe init` or `npm install -g vibium`) - - Network access to httpbin.org (public, stable) - - Local fixtures server running on :8088 (started in setup.local_docs_server) + - Local fixtures server running on 127.0.0.1:8088 diff --git a/.claude/skills/qe-browser/fixtures/serve-skills.js b/.claude/skills/qe-browser/fixtures/serve-skills.js index 8f277e4d..a9d68fa4 100644 --- a/.claude/skills/qe-browser/fixtures/serve-skills.js +++ b/.claude/skills/qe-browser/fixtures/serve-skills.js @@ -74,6 +74,31 @@ const POISONED_HTML = ` `; +const CONTENT_HTML = ` + +QE Browser Content Fixture + +
+

QE Browser Content Fixture

+

This deterministic page exercises browser assertions and visual comparisons.

+
+ +`; + +const FORM_HTML = ` + +QE Browser Form Fixture + +
+

Contact form

+
+ + +
+
+ +`; + function notFound(res, message) { res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' }); res.end(`404 Not Found: ${message}\n`); @@ -95,6 +120,18 @@ function serve(req, res) { return; } + if (pathname === '/fixtures/content.html') { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(CONTENT_HTML); + return; + } + + if (pathname === '/fixtures/form.html') { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(FORM_HTML); + return; + } + // Rewrite /foo/SKILL.md.html → /foo/SKILL.md (or similar) and serve wrapped. if (pathname.endsWith('.html')) { const mdPath = pathname.replace(/\.html$/, ''); diff --git a/.claude/skills/qe-browser/scripts/lib/vibium.js b/.claude/skills/qe-browser/scripts/lib/vibium.js index c930bfee..eeea32c4 100644 --- a/.claude/skills/qe-browser/scripts/lib/vibium.js +++ b/.claude/skills/qe-browser/scripts/lib/vibium.js @@ -292,7 +292,7 @@ function readInlineOrFile(value) { // "we couldn't run the test because the browser engine isn't installed." function emit(env) { process.stdout.write(`${JSON.stringify(env, null, 2)}\n`); - if (env.status === 'success') return 0; + if (env.status === 'success' || env.status === 'partial') return 0; if (env.status === 'skipped') return 2; return 1; } diff --git a/.claude/skills/qe-browser/scripts/smoke-test.sh b/.claude/skills/qe-browser/scripts/smoke-test.sh index 47d849fd..8893c384 100755 --- a/.claude/skills/qe-browser/scripts/smoke-test.sh +++ b/.claude/skills/qe-browser/scripts/smoke-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # qe-browser smoke test (bash mirror of evals/qe-browser.yaml) # -# Runs each helper script against pinned public fixtures (httpbin.org) and +# Runs each helper script against deterministic local fixtures and # verifies the output structure. Gates PR-reopen per ADR-091 Phase 3. # # RELATIONSHIP TO evals/qe-browser.yaml @@ -24,16 +24,25 @@ # 2 — vibium binary not on PATH (precondition unmet) # # Per feedback_no_unverified_failure_modes.md, this is the script we -# actually run, not just write. Per feedback_synthetic_fixtures_dont_count, -# all fixtures are pinned public endpoints (httpbin.org) — no synthetic -# stubs, no inline HTML. +# actually run, not just write. The fixture pages are maintained with the +# skill and served only on loopback, removing public-service rate limits. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" WORK_DIR="$(mktemp -d)" -trap 'rm -rf "$WORK_DIR"' EXIT +FIXTURE_PORT="${QE_BROWSER_FIXTURE_PORT:-8088}" +FIXTURE_URL="${QE_BROWSER_FIXTURE_URL:-http://127.0.0.1:${FIXTURE_PORT}}" +FIXTURE_PID="" + +cleanup() { + if [ -n "$FIXTURE_PID" ]; then + kill "$FIXTURE_PID" >/dev/null 2>&1 || true + fi + rm -rf "$WORK_DIR" +} +trap cleanup EXIT GREEN='\033[0;32m' RED='\033[0;31m' @@ -57,6 +66,28 @@ if ! command -v vibium >/dev/null 2>&1; then exit 2 fi +if ! curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then + QE_BROWSER_FIXTURE_PORT="$FIXTURE_PORT" \ + node "$SKILL_DIR/fixtures/serve-skills.js" >"$WORK_DIR/fixture-server.log" 2>&1 & + FIXTURE_PID=$! + fixture_ready=0 + for _attempt in 1 2 3 4 5; do + if curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then + fixture_ready=1 + break + fi + sleep 1 + done + if [ "$fixture_ready" != "1" ]; then + cat "$WORK_DIR/fixture-server.log" >&2 + echo -e "${RED}qe-browser fixture server did not become ready${NC}" >&2 + exit 1 + fi +fi + +CONTENT_URL="$FIXTURE_URL/fixtures/content.html" +FORM_URL="$FIXTURE_URL/fixtures/form.html" + VIBIUM_VERSION=$(vibium --version 2>&1 | head -1) echo "Smoke testing against $VIBIUM_VERSION" echo "Skill dir: $SKILL_DIR" @@ -64,29 +95,29 @@ echo "Work dir: $WORK_DIR" echo "" # --------------------------------------------------------------------------- -# tc001 — assert.js url_contains against pinned httpbin form +# tc001 — assert.js url_contains against the local form fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true +vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \ - '[{"kind": "url_contains", "text": "httpbin.org/forms"}]' 2>&1) + '[{"kind": "url_contains", "text": "/fixtures/form.html"}]' 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"status": "success"'; then - ok "tc001 url_contains on httpbin form" + ok "tc001 url_contains on local form" else - bad "tc001 url_contains on httpbin form" "exit=$EXIT, result=$RESULT" + bad "tc001 url_contains on local form" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- -# tc002 — assert.js selector_visible against pinned httpbin /html +# tc002 — assert.js selector_visible against the local content fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \ '[{"kind": "selector_visible", "selector": "h1"}]' 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passed": true'; then - ok "tc002 selector_visible h1 on httpbin /html" + ok "tc002 selector_visible h1 on local content" else - bad "tc002 selector_visible h1 on httpbin /html" "exit=$EXIT" + bad "tc002 selector_visible h1 on local content" "exit=$EXIT" fi # --------------------------------------------------------------------------- @@ -105,7 +136,7 @@ fi # tc004 — batch.js navigate + wait + assert in one call # --------------------------------------------------------------------------- RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \ - '[{"action":"go","url":"https://httpbin.org/html"},{"action":"wait_load"},{"action":"assert","checks":[{"kind":"url_contains","text":"/html"}]}]' \ + "[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"wait_load\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/fixtures/content.html\"}]}]" \ --summary-only 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passedSteps": 3'; then @@ -118,7 +149,7 @@ fi # tc005 — batch.js stops on failure # --------------------------------------------------------------------------- RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \ - '[{"action":"go","url":"https://httpbin.org/html"},{"action":"click","selector":"#does-not-exist-selector"},{"action":"go","url":"https://httpbin.org/forms/post"}]' 2>&1) + "[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/does-not-exist\"}]},{\"action\":\"go\",\"url\":\"$FORM_URL\"}]" 2>&1) EXIT=$? if [ "$EXIT" = "1" ] && echo "$RESULT" | grep -q '"failedStep"'; then ok "tc005 batch stops on first failure" @@ -131,19 +162,19 @@ fi # # Set explicit viewport BEFORE screenshot so the two visual-diff runs have # the same dimensions. Without this the chromium headless window picks -# whatever size it likes per run, and httpbin.org/html renders at different +# whatever size it likes per run, and pages may render at different # sizes between runs (768×654 vs 765×672 observed), making pixel-diff # spuriously fail. This is documented in references/assertion-kinds.md. # --------------------------------------------------------------------------- rm -rf "$PWD/.aqe/visual-baselines/smoke_test_baseline"* vibium --headless viewport 1280 720 >/dev/null 2>&1 || true -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/visual-diff.js" --name smoke_test_baseline 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"baseline_created"'; then ok "tc006 visual-diff baseline created" else - bad "tc006 visual-diff baseline created" "exit=$EXIT" + bad "tc006 visual-diff baseline created" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- @@ -157,13 +188,13 @@ EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -qE '"(match|baseline_created)"'; then ok "tc007 visual-diff second run matches" else - bad "tc007 visual-diff second run matches" "exit=$EXIT" + bad "tc007 visual-diff second run matches" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- # tc008 — check-injection.js clean page # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/check-injection.js" --include-hidden 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"severity": "none"'; then @@ -173,15 +204,15 @@ else fi # --------------------------------------------------------------------------- -# tc010 — intent-score.js submit_form on pinned httpbin form +# tc010 — intent-score.js submit_form on the local form fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true +vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/intent-score.js" --intent submit_form 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"intent": "submit_form"'; then - ok "tc010 intent-score submit_form on httpbin form" + ok "tc010 intent-score submit_form on local form" else - bad "tc010 intent-score submit_form on httpbin form" "exit=$EXIT" + bad "tc010 intent-score submit_form on local form" "exit=$EXIT" fi # --------------------------------------------------------------------------- diff --git a/.claude/skills/qe-browser/scripts/visual-diff.js b/.claude/skills/qe-browser/scripts/visual-diff.js index 2f23502c..d66618f4 100644 --- a/.claude/skills/qe-browser/scripts/visual-diff.js +++ b/.claude/skills/qe-browser/scripts/visual-diff.js @@ -45,11 +45,10 @@ function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); } -// Vibium screenshot quirks (verified against v26.3.18 on 2026-04-09): -// 1. `vibium screenshot -o ` IGNORES the directory in . -// Only the basename is used, and the file is saved to -// `~/Pictures/Vibium/`. We work around this by reading from -// Vibium's actual output dir and copying to the requested location. +// Vibium screenshot output changed across the supported 26.x releases. +// Older releases ignore the directory passed to `-o` and save under +// `~/Pictures/Vibium`; newer releases may honor the requested path. Accept +// both behaviors so AQE init's compatible-version range stays usable. // 2. `--selector` flag does NOT exist on `vibium screenshot`. Selector- // scoped baselines are not supported in v26.3.x. We surface a clear // error if a caller passes one. Future Vibium versions may add it. @@ -69,21 +68,31 @@ function captureScreenshot(selector, outputPath) { ); } const basename = path.basename(outputPath); - const args = ['screenshot', '-o', basename, '--full-page']; + const candidates = [ + outputPath, + path.resolve(process.cwd(), basename), + path.join(vibiumPicturesDir(), basename), + ]; + for (const candidate of candidates) { + fs.rmSync(candidate, { force: true }); + } + const args = ['screenshot', '-o', outputPath, '--full-page']; const res = vibium(args); if (res.status !== 0) { throw new Error(`vibium screenshot failed: ${res.stderr.trim() || res.stdout.trim()}`); } - // Vibium wrote the file to ~/Pictures/Vibium/, not outputPath. - // Copy it to where the caller asked. Use copy-then-unlink so we leave - // Vibium's own dir clean for the next run. - const vibiumPath = path.join(vibiumPicturesDir(), basename); - if (!fs.existsSync(vibiumPath)) { - throw new Error(`screenshot output not created at ${vibiumPath} (vibium said: ${res.stdout.trim()})`); + const screenshotPath = candidates.find((candidate) => fs.existsSync(candidate)); + if (!screenshotPath) { + throw new Error( + `screenshot output not created at any supported path (${candidates.join(', ')}; ` + + `vibium said: ${res.stdout.trim()})` + ); + } + if (screenshotPath !== outputPath) { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.copyFileSync(screenshotPath, outputPath); + fs.unlinkSync(screenshotPath); } - fs.mkdirSync(path.dirname(outputPath), { recursive: true }); - fs.copyFileSync(vibiumPath, outputPath); - fs.unlinkSync(vibiumPath); return outputPath; } diff --git a/.github/workflows/test-qe-browser.yml b/.github/workflows/test-qe-browser.yml index c872d310..e9120a9f 100644 --- a/.github/workflows/test-qe-browser.yml +++ b/.github/workflows/test-qe-browser.yml @@ -7,8 +7,8 @@ # # Jobs: # unit : vitest suite covering the 5 primitives + CommandEvalRunner -# smoke : real Vibium install + 5-primitive smoke-test.sh against live -# public fixtures (httpbin.org) +# smoke : real Vibium install + 5-primitive smoke-test.sh against local +# deterministic fixtures # eval : build the AQE CLI and run `aqe eval run --skill qe-browser` # to exercise the CommandEvalRunner dispatch path end-to-end # against real Vibium (closes devil's-advocate gap G2) @@ -78,7 +78,7 @@ jobs: tests/unit/scripts/qe-browser-vibium-lib.test.ts smoke: - name: Smoke test (real Vibium + httpbin) + name: Smoke test (real Vibium + local fixtures) runs-on: ubuntu-latest timeout-minutes: 15 needs: unit @@ -126,8 +126,6 @@ jobs: - name: Run qe-browser smoke test run: bash .claude/skills/qe-browser/scripts/smoke-test.sh env: - # Smoke test uses public httpbin endpoints; set CI so the script - # can short-circuit interactive retries if Vibium misbehaves. CI: 'true' eval: @@ -179,6 +177,13 @@ jobs: chmod +x "$CHROME" head -3 "$CHROME" + - name: Start deterministic browser fixtures + run: | + node .claude/skills/qe-browser/fixtures/serve-skills.js > /tmp/qe-browser-fixtures.log 2>&1 & + echo $! > /tmp/qe-browser-fixtures.pid + curl --retry 5 --retry-connrefused --retry-delay 1 --fail \ + http://127.0.0.1:8088/fixtures/content.html > /dev/null + - name: Run aqe eval against qe-browser # The --model flag is required by the CLI but ignored by # CommandEvalRunner for command-mode suites. Exit 0 means all diff --git a/assets/skills/qe-browser/evals/qe-browser.yaml b/assets/skills/qe-browser/evals/qe-browser.yaml index 4270335f..7bcbde69 100644 --- a/assets/skills/qe-browser/evals/qe-browser.yaml +++ b/assets/skills/qe-browser/evals/qe-browser.yaml @@ -52,26 +52,18 @@ setup: optional_tools: - pixelmatch - pngjs - # NOTE: this yaml deliberately uses ONLY pinned public fixtures - # (httpbin.org/*) so it can be run end-to-end by CommandEvalRunner without - # any prerequisite services. Tests that need a local poisoned-HTML fixture - # (the check-injection severity path) live in scripts/smoke-test.sh, which - # starts fixtures/serve-skills.js out of band. + # The CI workflow starts fixtures/serve-skills.js on loopback before this + # suite runs. Keeping browser targets local removes rate limits and makes + # every page stable across repeated visual comparisons. fixtures: - public_pinned: - # Pinned public endpoints — chosen because they're stable, well-known, and - # serve predictable forms / HTML. Per feedback_no_unverified_failure_modes, - # these are the canonical "does the tool actually work" fixtures. - httpbin_form: - url: "https://httpbin.org/forms/post" - description: "Classic simple form — custname, custtel, custemail, size, toppings" - httpbin_html: - url: "https://httpbin.org/html" - description: "Static HTML page with known headings" - httpbin_status_404: - url: "https://httpbin.org/status/404" - description: "Known 404 for testing no_failed_requests" + local: + form: + url: "http://127.0.0.1:8088/fixtures/form.html" + description: "Deterministic form with an email input and submit button" + content: + url: "http://127.0.0.1:8088/fixtures/content.html" + description: "Deterministic static page with a known heading" test_cases: # -------- assert.js -------- @@ -81,10 +73,10 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/forms/post" + - "vibium --headless go http://127.0.0.1:8088/fixtures/form.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ - '[{"kind": "url_contains", "text": "httpbin.org/forms"}]' + '[{"kind": "url_contains", "text": "/fixtures/form.html"}]' expected: exit_code: 0 json_fields: @@ -98,7 +90,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ '[{"kind": "selector_visible", "selector": "h1"}]' @@ -113,7 +105,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/assert.js --checks \ '[{"kind": "url_contains", "text": "this-does-not-exist"}]' @@ -132,10 +124,10 @@ test_cases: command: | node .claude/skills/qe-browser/scripts/batch.js --steps \ '[ - {"action": "go", "url": "https://httpbin.org/html"}, + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"}, {"action": "wait_load"}, {"action": "assert", "checks": [ - {"kind": "url_contains", "text": "/html"}, + {"kind": "url_contains", "text": "/fixtures/content.html"}, {"kind": "selector_visible", "selector": "h1"} ]} ]' --summary-only @@ -154,9 +146,11 @@ test_cases: command: | node .claude/skills/qe-browser/scripts/batch.js --steps \ '[ - {"action": "go", "url": "https://httpbin.org/html"}, - {"action": "click", "selector": "#does-not-exist"}, - {"action": "go", "url": "https://httpbin.org/forms/post"} + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"}, + {"action": "assert", "checks": [ + {"kind": "url_contains", "text": "/does-not-exist"} + ]}, + {"action": "go", "url": "http://127.0.0.1:8088/fixtures/form.html"} ]' expected: exit_code: 1 @@ -177,7 +171,7 @@ test_cases: # different dimensions (768×654 vs 765×672 observed), making the # pixel-diff in tc007 spuriously fail. Mirrors scripts/smoke-test.sh. - "vibium --headless viewport 1280 720" - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" - "rm -rf .aqe/visual-baselines/eval_httpbin_html*" command: | node .claude/skills/qe-browser/scripts/visual-diff.js \ @@ -195,7 +189,7 @@ test_cases: input: setup: - "vibium --headless viewport 1280 720" - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/visual-diff.js \ --name eval_httpbin_html --threshold 0.02 @@ -212,7 +206,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/check-injection.js --include-hidden expected: @@ -221,15 +215,11 @@ test_cases: ".status": "success" ".output.checkInjection.severity": "none" - # GAP: the "poisoned-page detected with severity>=high" contract needs a - # local fixture (fixtures/injection-poisoned.html) served by - # fixtures/serve-skills.js. That's out of scope for this yaml — we keep - # CommandEvalRunner dependency-free so it can run anywhere httpbin.org - # is reachable. Coverage of the high-severity path is currently + # The "poisoned-page detected with severity>=high" contract is currently + # asserted outside this command suite. Coverage of the high-severity path is # only asserted by unit tests on check-injection.js (see # tests/unit/scripts/qe-browser-check-injection.test.ts). Follow-up: - # either teach CommandEvalRunner to spawn the fixture server, or add a - # tc009 to scripts/smoke-test.sh that starts/stops it out of band. + # add tc009 against fixtures/injection-poisoned.html. # -------- intent-score.js -------- - id: tc010_intent_submit_form_on_httpbin @@ -238,7 +228,7 @@ test_cases: priority: critical input: setup: - - "vibium --headless go https://httpbin.org/forms/post" + - "vibium --headless go http://127.0.0.1:8088/fixtures/form.html" command: | node .claude/skills/qe-browser/scripts/intent-score.js \ --intent submit_form @@ -255,7 +245,7 @@ test_cases: priority: medium input: setup: - - "vibium --headless go https://httpbin.org/html" + - "vibium --headless go http://127.0.0.1:8088/fixtures/content.html" command: | node .claude/skills/qe-browser/scripts/intent-score.js --intent fill_email expected: @@ -270,5 +260,4 @@ validation: notes: | Evaluation assumes: - `vibium` v26.3.x+ is on PATH (from `aqe init` or `npm install -g vibium`) - - Network access to httpbin.org (public, stable) - - Local fixtures server running on :8088 (started in setup.local_docs_server) + - Local fixtures server running on 127.0.0.1:8088 diff --git a/assets/skills/qe-browser/fixtures/serve-skills.js b/assets/skills/qe-browser/fixtures/serve-skills.js index 8f277e4d..a9d68fa4 100644 --- a/assets/skills/qe-browser/fixtures/serve-skills.js +++ b/assets/skills/qe-browser/fixtures/serve-skills.js @@ -74,6 +74,31 @@ const POISONED_HTML = ` `; +const CONTENT_HTML = ` + +QE Browser Content Fixture + +
+

QE Browser Content Fixture

+

This deterministic page exercises browser assertions and visual comparisons.

+
+ +`; + +const FORM_HTML = ` + +QE Browser Form Fixture + +
+

Contact form

+
+ + +
+
+ +`; + function notFound(res, message) { res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' }); res.end(`404 Not Found: ${message}\n`); @@ -95,6 +120,18 @@ function serve(req, res) { return; } + if (pathname === '/fixtures/content.html') { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(CONTENT_HTML); + return; + } + + if (pathname === '/fixtures/form.html') { + res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + res.end(FORM_HTML); + return; + } + // Rewrite /foo/SKILL.md.html → /foo/SKILL.md (or similar) and serve wrapped. if (pathname.endsWith('.html')) { const mdPath = pathname.replace(/\.html$/, ''); diff --git a/assets/skills/qe-browser/scripts/lib/vibium.js b/assets/skills/qe-browser/scripts/lib/vibium.js index c930bfee..eeea32c4 100644 --- a/assets/skills/qe-browser/scripts/lib/vibium.js +++ b/assets/skills/qe-browser/scripts/lib/vibium.js @@ -292,7 +292,7 @@ function readInlineOrFile(value) { // "we couldn't run the test because the browser engine isn't installed." function emit(env) { process.stdout.write(`${JSON.stringify(env, null, 2)}\n`); - if (env.status === 'success') return 0; + if (env.status === 'success' || env.status === 'partial') return 0; if (env.status === 'skipped') return 2; return 1; } diff --git a/assets/skills/qe-browser/scripts/smoke-test.sh b/assets/skills/qe-browser/scripts/smoke-test.sh index cf5ab5d9..cc4465dd 100755 --- a/assets/skills/qe-browser/scripts/smoke-test.sh +++ b/assets/skills/qe-browser/scripts/smoke-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # qe-browser smoke test # -# Runs each helper script against pinned public fixtures (httpbin.org) and +# Runs each helper script against deterministic local fixtures and # verifies the output structure. This is the script that gates PR-reopen # per ADR-091 Phase 3 — it MUST be run on a machine with vibium installed # before the qe-browser PR is considered safe to reopen. @@ -12,16 +12,25 @@ # 2 — vibium binary not on PATH (precondition unmet) # # Per feedback_no_unverified_failure_modes.md, this is the script we -# actually run, not just write. Per feedback_synthetic_fixtures_dont_count, -# all fixtures are pinned public endpoints (httpbin.org) — no synthetic -# stubs, no inline HTML. +# actually run, not just write. The fixture pages are maintained with the +# skill and served only on loopback, removing public-service rate limits. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" WORK_DIR="$(mktemp -d)" -trap 'rm -rf "$WORK_DIR"' EXIT +FIXTURE_PORT="${QE_BROWSER_FIXTURE_PORT:-8088}" +FIXTURE_URL="${QE_BROWSER_FIXTURE_URL:-http://127.0.0.1:${FIXTURE_PORT}}" +FIXTURE_PID="" + +cleanup() { + if [ -n "$FIXTURE_PID" ]; then + kill "$FIXTURE_PID" >/dev/null 2>&1 || true + fi + rm -rf "$WORK_DIR" +} +trap cleanup EXIT GREEN='\033[0;32m' RED='\033[0;31m' @@ -45,6 +54,28 @@ if ! command -v vibium >/dev/null 2>&1; then exit 2 fi +if ! curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then + QE_BROWSER_FIXTURE_PORT="$FIXTURE_PORT" \ + node "$SKILL_DIR/fixtures/serve-skills.js" >"$WORK_DIR/fixture-server.log" 2>&1 & + FIXTURE_PID=$! + fixture_ready=0 + for _attempt in 1 2 3 4 5; do + if curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then + fixture_ready=1 + break + fi + sleep 1 + done + if [ "$fixture_ready" != "1" ]; then + cat "$WORK_DIR/fixture-server.log" >&2 + echo -e "${RED}qe-browser fixture server did not become ready${NC}" >&2 + exit 1 + fi +fi + +CONTENT_URL="$FIXTURE_URL/fixtures/content.html" +FORM_URL="$FIXTURE_URL/fixtures/form.html" + VIBIUM_VERSION=$(vibium --version 2>&1 | head -1) echo "Smoke testing against $VIBIUM_VERSION" echo "Skill dir: $SKILL_DIR" @@ -52,29 +83,29 @@ echo "Work dir: $WORK_DIR" echo "" # --------------------------------------------------------------------------- -# tc001 — assert.js url_contains against pinned httpbin form +# tc001 — assert.js url_contains against the local form fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true +vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \ - '[{"kind": "url_contains", "text": "httpbin.org/forms"}]' 2>&1) + '[{"kind": "url_contains", "text": "/fixtures/form.html"}]' 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"status": "success"'; then - ok "tc001 url_contains on httpbin form" + ok "tc001 url_contains on local form" else - bad "tc001 url_contains on httpbin form" "exit=$EXIT, result=$RESULT" + bad "tc001 url_contains on local form" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- -# tc002 — assert.js selector_visible against pinned httpbin /html +# tc002 — assert.js selector_visible against the local content fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \ '[{"kind": "selector_visible", "selector": "h1"}]' 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passed": true'; then - ok "tc002 selector_visible h1 on httpbin /html" + ok "tc002 selector_visible h1 on local content" else - bad "tc002 selector_visible h1 on httpbin /html" "exit=$EXIT" + bad "tc002 selector_visible h1 on local content" "exit=$EXIT" fi # --------------------------------------------------------------------------- @@ -93,7 +124,7 @@ fi # tc004 — batch.js navigate + wait + assert in one call # --------------------------------------------------------------------------- RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \ - '[{"action":"go","url":"https://httpbin.org/html"},{"action":"wait_load"},{"action":"assert","checks":[{"kind":"url_contains","text":"/html"}]}]' \ + "[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"wait_load\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/fixtures/content.html\"}]}]" \ --summary-only 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passedSteps": 3'; then @@ -106,7 +137,7 @@ fi # tc005 — batch.js stops on failure # --------------------------------------------------------------------------- RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \ - '[{"action":"go","url":"https://httpbin.org/html"},{"action":"click","selector":"#does-not-exist-selector"},{"action":"go","url":"https://httpbin.org/forms/post"}]' 2>&1) + "[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/does-not-exist\"}]},{\"action\":\"go\",\"url\":\"$FORM_URL\"}]" 2>&1) EXIT=$? if [ "$EXIT" = "1" ] && echo "$RESULT" | grep -q '"failedStep"'; then ok "tc005 batch stops on first failure" @@ -119,19 +150,19 @@ fi # # Set explicit viewport BEFORE screenshot so the two visual-diff runs have # the same dimensions. Without this the chromium headless window picks -# whatever size it likes per run, and httpbin.org/html renders at different +# whatever size it likes per run, and pages may render at different # sizes between runs (768×654 vs 765×672 observed), making pixel-diff # spuriously fail. This is documented in references/assertion-kinds.md. # --------------------------------------------------------------------------- rm -rf "$PWD/.aqe/visual-baselines/smoke_test_baseline"* vibium --headless viewport 1280 720 >/dev/null 2>&1 || true -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/visual-diff.js" --name smoke_test_baseline 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"baseline_created"'; then ok "tc006 visual-diff baseline created" else - bad "tc006 visual-diff baseline created" "exit=$EXIT" + bad "tc006 visual-diff baseline created" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- @@ -145,13 +176,13 @@ EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -qE '"(match|baseline_created)"'; then ok "tc007 visual-diff second run matches" else - bad "tc007 visual-diff second run matches" "exit=$EXIT" + bad "tc007 visual-diff second run matches" "exit=$EXIT, result=$RESULT" fi # --------------------------------------------------------------------------- # tc008 — check-injection.js clean page # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true +vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/check-injection.js" --include-hidden 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"severity": "none"'; then @@ -161,15 +192,15 @@ else fi # --------------------------------------------------------------------------- -# tc010 — intent-score.js submit_form on pinned httpbin form +# tc010 — intent-score.js submit_form on the local form fixture # --------------------------------------------------------------------------- -vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true +vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true RESULT=$(node "$SKILL_DIR/scripts/intent-score.js" --intent submit_form 2>&1) EXIT=$? if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"intent": "submit_form"'; then - ok "tc010 intent-score submit_form on httpbin form" + ok "tc010 intent-score submit_form on local form" else - bad "tc010 intent-score submit_form on httpbin form" "exit=$EXIT" + bad "tc010 intent-score submit_form on local form" "exit=$EXIT" fi # --------------------------------------------------------------------------- diff --git a/assets/skills/qe-browser/scripts/visual-diff.js b/assets/skills/qe-browser/scripts/visual-diff.js index 2f23502c..d66618f4 100644 --- a/assets/skills/qe-browser/scripts/visual-diff.js +++ b/assets/skills/qe-browser/scripts/visual-diff.js @@ -45,11 +45,10 @@ function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); } -// Vibium screenshot quirks (verified against v26.3.18 on 2026-04-09): -// 1. `vibium screenshot -o ` IGNORES the directory in . -// Only the basename is used, and the file is saved to -// `~/Pictures/Vibium/`. We work around this by reading from -// Vibium's actual output dir and copying to the requested location. +// Vibium screenshot output changed across the supported 26.x releases. +// Older releases ignore the directory passed to `-o` and save under +// `~/Pictures/Vibium`; newer releases may honor the requested path. Accept +// both behaviors so AQE init's compatible-version range stays usable. // 2. `--selector` flag does NOT exist on `vibium screenshot`. Selector- // scoped baselines are not supported in v26.3.x. We surface a clear // error if a caller passes one. Future Vibium versions may add it. @@ -69,21 +68,31 @@ function captureScreenshot(selector, outputPath) { ); } const basename = path.basename(outputPath); - const args = ['screenshot', '-o', basename, '--full-page']; + const candidates = [ + outputPath, + path.resolve(process.cwd(), basename), + path.join(vibiumPicturesDir(), basename), + ]; + for (const candidate of candidates) { + fs.rmSync(candidate, { force: true }); + } + const args = ['screenshot', '-o', outputPath, '--full-page']; const res = vibium(args); if (res.status !== 0) { throw new Error(`vibium screenshot failed: ${res.stderr.trim() || res.stdout.trim()}`); } - // Vibium wrote the file to ~/Pictures/Vibium/, not outputPath. - // Copy it to where the caller asked. Use copy-then-unlink so we leave - // Vibium's own dir clean for the next run. - const vibiumPath = path.join(vibiumPicturesDir(), basename); - if (!fs.existsSync(vibiumPath)) { - throw new Error(`screenshot output not created at ${vibiumPath} (vibium said: ${res.stdout.trim()})`); + const screenshotPath = candidates.find((candidate) => fs.existsSync(candidate)); + if (!screenshotPath) { + throw new Error( + `screenshot output not created at any supported path (${candidates.join(', ')}; ` + + `vibium said: ${res.stdout.trim()})` + ); + } + if (screenshotPath !== outputPath) { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.copyFileSync(screenshotPath, outputPath); + fs.unlinkSync(screenshotPath); } - fs.mkdirSync(path.dirname(outputPath), { recursive: true }); - fs.copyFileSync(vibiumPath, outputPath); - fs.unlinkSync(vibiumPath); return outputPath; } diff --git a/package-lock.json b/package-lock.json index bd0015c7..ca21f7fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11442,9 +11442,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.21.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", - "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/tests/unit/scripts/qe-browser-fixtures-server.test.ts b/tests/unit/scripts/qe-browser-fixtures-server.test.ts index a7f120bb..a78fb78f 100644 --- a/tests/unit/scripts/qe-browser-fixtures-server.test.ts +++ b/tests/unit/scripts/qe-browser-fixtures-server.test.ts @@ -106,6 +106,20 @@ describe('qe-browser fixtures/serve-skills', () => { }, 10000); }); + describe('browser eval pages', () => { + it('serves deterministic content and form pages without an external service', async () => { + const { port } = await startServer({ QE_BROWSER_FIXTURE_PORT: '18806' }); + const content = await fetch('127.0.0.1', port, '/fixtures/content.html'); + const form = await fetch('127.0.0.1', port, '/fixtures/form.html'); + + expect(content.status).toBe(200); + expect(content.body).toContain('

QE Browser Content Fixture

'); + expect(form.status).toBe(200); + expect(form.body).toContain('input id="email"'); + expect(form.body).toContain('button type="submit"'); + }, 10000); + }); + // M3 regression: the previous startsWith() check could false-pass on // sibling directories with shared prefix. path.relative() is the // canonical traversal guard. diff --git a/tests/unit/scripts/qe-browser-vibium-lib.test.ts b/tests/unit/scripts/qe-browser-vibium-lib.test.ts index 5d229957..b3246efe 100644 --- a/tests/unit/scripts/qe-browser-vibium-lib.test.ts +++ b/tests/unit/scripts/qe-browser-vibium-lib.test.ts @@ -228,6 +228,10 @@ describe('qe-browser lib/vibium', () => { expect(captured(lib.envelope({ operation: 'x', summary: 'ok', status: 'success' }))).toBe(0); }); + it('returns 0 for a completed operation with status: partial', () => { + expect(captured(lib.envelope({ operation: 'x', summary: 'empty', status: 'partial' }))).toBe(0); + }); + it('returns 1 for status: failed', () => { expect(captured(lib.envelope({ operation: 'x', summary: 'no', status: 'failed' }))).toBe(1); }); diff --git a/tests/unit/validation/command-eval-runner-qe-browser.test.ts b/tests/unit/validation/command-eval-runner-qe-browser.test.ts index a7a6c739..8b05c001 100644 --- a/tests/unit/validation/command-eval-runner-qe-browser.test.ts +++ b/tests/unit/validation/command-eval-runner-qe-browser.test.ts @@ -84,7 +84,7 @@ describe('CommandEvalRunner — qe-browser integration', () => { } if (cmd.includes('scripts/batch.js')) { - const stopOnFail = cmd.includes('#does-not-exist'); + const stopOnFail = cmd.includes('/does-not-exist'); if (stopOnFail) { return { status: 1,