mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
Merge remote-tracking branch 'origin/fix/qe-browser-local-fixtures' into chore/node-22-floor
This commit is contained in:
Dragan Spiridonov
@@ -52,26 +52,18 @@ setup:
|
||||
optional_tools:
|
||||
- pixelmatch
|
||||
- pngjs
|
||||
# NOTE: this yaml deliberately uses ONLY pinned public fixtures
|
||||
# (httpbin.org/*) so it can be run end-to-end by CommandEvalRunner without
|
||||
# any prerequisite services. Tests that need a local poisoned-HTML fixture
|
||||
# (the check-injection severity path) live in scripts/smoke-test.sh, which
|
||||
# starts fixtures/serve-skills.js out of band.
|
||||
# The CI workflow starts fixtures/serve-skills.js on loopback before this
|
||||
# suite runs. Keeping browser targets local removes rate limits and makes
|
||||
# every page stable across repeated visual comparisons.
|
||||
|
||||
fixtures:
|
||||
public_pinned:
|
||||
# Pinned public endpoints — chosen because they're stable, well-known, and
|
||||
# serve predictable forms / HTML. Per feedback_no_unverified_failure_modes,
|
||||
# these are the canonical "does the tool actually work" fixtures.
|
||||
httpbin_form:
|
||||
url: "https://httpbin.org/forms/post"
|
||||
description: "Classic simple form — custname, custtel, custemail, size, toppings"
|
||||
httpbin_html:
|
||||
url: "https://httpbin.org/html"
|
||||
description: "Static HTML page with known headings"
|
||||
httpbin_status_404:
|
||||
url: "https://httpbin.org/status/404"
|
||||
description: "Known 404 for testing no_failed_requests"
|
||||
local:
|
||||
form:
|
||||
url: "http://127.0.0.1:8088/fixtures/form.html"
|
||||
description: "Deterministic form with an email input and submit button"
|
||||
content:
|
||||
url: "http://127.0.0.1:8088/fixtures/content.html"
|
||||
description: "Deterministic static page with a known heading"
|
||||
|
||||
test_cases:
|
||||
# -------- assert.js --------
|
||||
@@ -81,10 +73,10 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/forms/post"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/form.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "url_contains", "text": "httpbin.org/forms"}]'
|
||||
'[{"kind": "url_contains", "text": "/fixtures/form.html"}]'
|
||||
expected:
|
||||
exit_code: 0
|
||||
json_fields:
|
||||
@@ -98,7 +90,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "selector_visible", "selector": "h1"}]'
|
||||
@@ -113,7 +105,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "url_contains", "text": "this-does-not-exist"}]'
|
||||
@@ -132,10 +124,10 @@ test_cases:
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/batch.js --steps \
|
||||
'[
|
||||
{"action": "go", "url": "https://httpbin.org/html"},
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"},
|
||||
{"action": "wait_load"},
|
||||
{"action": "assert", "checks": [
|
||||
{"kind": "url_contains", "text": "/html"},
|
||||
{"kind": "url_contains", "text": "/fixtures/content.html"},
|
||||
{"kind": "selector_visible", "selector": "h1"}
|
||||
]}
|
||||
]' --summary-only
|
||||
@@ -154,9 +146,11 @@ test_cases:
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/batch.js --steps \
|
||||
'[
|
||||
{"action": "go", "url": "https://httpbin.org/html"},
|
||||
{"action": "click", "selector": "#does-not-exist"},
|
||||
{"action": "go", "url": "https://httpbin.org/forms/post"}
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"},
|
||||
{"action": "assert", "checks": [
|
||||
{"kind": "url_contains", "text": "/does-not-exist"}
|
||||
]},
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/form.html"}
|
||||
]'
|
||||
expected:
|
||||
exit_code: 1
|
||||
@@ -177,7 +171,7 @@ test_cases:
|
||||
# different dimensions (768×654 vs 765×672 observed), making the
|
||||
# pixel-diff in tc007 spuriously fail. Mirrors scripts/smoke-test.sh.
|
||||
- "vibium --headless viewport 1280 720"
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
- "rm -rf .aqe/visual-baselines/eval_httpbin_html*"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/visual-diff.js \
|
||||
@@ -195,7 +189,7 @@ test_cases:
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless viewport 1280 720"
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/visual-diff.js \
|
||||
--name eval_httpbin_html --threshold 0.02
|
||||
@@ -212,7 +206,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/check-injection.js --include-hidden
|
||||
expected:
|
||||
@@ -221,15 +215,11 @@ test_cases:
|
||||
".status": "success"
|
||||
".output.checkInjection.severity": "none"
|
||||
|
||||
# GAP: the "poisoned-page detected with severity>=high" contract needs a
|
||||
# local fixture (fixtures/injection-poisoned.html) served by
|
||||
# fixtures/serve-skills.js. That's out of scope for this yaml — we keep
|
||||
# CommandEvalRunner dependency-free so it can run anywhere httpbin.org
|
||||
# is reachable. Coverage of the high-severity path is currently
|
||||
# The "poisoned-page detected with severity>=high" contract is currently
|
||||
# asserted outside this command suite. Coverage of the high-severity path is
|
||||
# only asserted by unit tests on check-injection.js (see
|
||||
# tests/unit/scripts/qe-browser-check-injection.test.ts). Follow-up:
|
||||
# either teach CommandEvalRunner to spawn the fixture server, or add a
|
||||
# tc009 to scripts/smoke-test.sh that starts/stops it out of band.
|
||||
# add tc009 against fixtures/injection-poisoned.html.
|
||||
|
||||
# -------- intent-score.js --------
|
||||
- id: tc010_intent_submit_form_on_httpbin
|
||||
@@ -238,7 +228,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/forms/post"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/form.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/intent-score.js \
|
||||
--intent submit_form
|
||||
@@ -255,7 +245,7 @@ test_cases:
|
||||
priority: medium
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/intent-score.js --intent fill_email
|
||||
expected:
|
||||
@@ -270,5 +260,4 @@ validation:
|
||||
notes: |
|
||||
Evaluation assumes:
|
||||
- `vibium` v26.3.x+ is on PATH (from `aqe init` or `npm install -g vibium`)
|
||||
- Network access to httpbin.org (public, stable)
|
||||
- Local fixtures server running on :8088 (started in setup.local_docs_server)
|
||||
- Local fixtures server running on 127.0.0.1:8088
|
||||
|
||||
@@ -74,6 +74,31 @@ const POISONED_HTML = `<!doctype html>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const CONTENT_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>QE Browser Content Fixture</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>QE Browser Content Fixture</h1>
|
||||
<p>This deterministic page exercises browser assertions and visual comparisons.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const FORM_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>QE Browser Form Fixture</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Contact form</h1>
|
||||
<form method="post" action="/fixtures/form.html">
|
||||
<label>Email <input id="email" name="email" type="email" autocomplete="email"></label>
|
||||
<button type="submit">Submit</button>
|
||||
</form>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
function notFound(res, message) {
|
||||
res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end(`404 Not Found: ${message}\n`);
|
||||
@@ -95,6 +120,18 @@ function serve(req, res) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pathname === '/fixtures/content.html') {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
||||
res.end(CONTENT_HTML);
|
||||
return;
|
||||
}
|
||||
|
||||
if (pathname === '/fixtures/form.html') {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
||||
res.end(FORM_HTML);
|
||||
return;
|
||||
}
|
||||
|
||||
// Rewrite /foo/SKILL.md.html → /foo/SKILL.md (or similar) and serve wrapped.
|
||||
if (pathname.endsWith('.html')) {
|
||||
const mdPath = pathname.replace(/\.html$/, '');
|
||||
|
||||
@@ -292,7 +292,7 @@ function readInlineOrFile(value) {
|
||||
// "we couldn't run the test because the browser engine isn't installed."
|
||||
function emit(env) {
|
||||
process.stdout.write(`${JSON.stringify(env, null, 2)}\n`);
|
||||
if (env.status === 'success') return 0;
|
||||
if (env.status === 'success' || env.status === 'partial') return 0;
|
||||
if (env.status === 'skipped') return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# qe-browser smoke test (bash mirror of evals/qe-browser.yaml)
|
||||
#
|
||||
# Runs each helper script against pinned public fixtures (httpbin.org) and
|
||||
# Runs each helper script against deterministic local fixtures and
|
||||
# verifies the output structure. Gates PR-reopen per ADR-091 Phase 3.
|
||||
#
|
||||
# RELATIONSHIP TO evals/qe-browser.yaml
|
||||
@@ -24,16 +24,25 @@
|
||||
# 2 — vibium binary not on PATH (precondition unmet)
|
||||
#
|
||||
# Per feedback_no_unverified_failure_modes.md, this is the script we
|
||||
# actually run, not just write. Per feedback_synthetic_fixtures_dont_count,
|
||||
# all fixtures are pinned public endpoints (httpbin.org) — no synthetic
|
||||
# stubs, no inline HTML.
|
||||
# actually run, not just write. The fixture pages are maintained with the
|
||||
# skill and served only on loopback, removing public-service rate limits.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
FIXTURE_PORT="${QE_BROWSER_FIXTURE_PORT:-8088}"
|
||||
FIXTURE_URL="${QE_BROWSER_FIXTURE_URL:-http://127.0.0.1:${FIXTURE_PORT}}"
|
||||
FIXTURE_PID=""
|
||||
|
||||
cleanup() {
|
||||
if [ -n "$FIXTURE_PID" ]; then
|
||||
kill "$FIXTURE_PID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf "$WORK_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
@@ -57,6 +66,28 @@ if ! command -v vibium >/dev/null 2>&1; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then
|
||||
QE_BROWSER_FIXTURE_PORT="$FIXTURE_PORT" \
|
||||
node "$SKILL_DIR/fixtures/serve-skills.js" >"$WORK_DIR/fixture-server.log" 2>&1 &
|
||||
FIXTURE_PID=$!
|
||||
fixture_ready=0
|
||||
for _attempt in 1 2 3 4 5; do
|
||||
if curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then
|
||||
fixture_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$fixture_ready" != "1" ]; then
|
||||
cat "$WORK_DIR/fixture-server.log" >&2
|
||||
echo -e "${RED}qe-browser fixture server did not become ready${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
CONTENT_URL="$FIXTURE_URL/fixtures/content.html"
|
||||
FORM_URL="$FIXTURE_URL/fixtures/form.html"
|
||||
|
||||
VIBIUM_VERSION=$(vibium --version 2>&1 | head -1)
|
||||
echo "Smoke testing against $VIBIUM_VERSION"
|
||||
echo "Skill dir: $SKILL_DIR"
|
||||
@@ -64,29 +95,29 @@ echo "Work dir: $WORK_DIR"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc001 — assert.js url_contains against pinned httpbin form
|
||||
# tc001 — assert.js url_contains against the local form fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true
|
||||
vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \
|
||||
'[{"kind": "url_contains", "text": "httpbin.org/forms"}]' 2>&1)
|
||||
'[{"kind": "url_contains", "text": "/fixtures/form.html"}]' 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"status": "success"'; then
|
||||
ok "tc001 url_contains on httpbin form"
|
||||
ok "tc001 url_contains on local form"
|
||||
else
|
||||
bad "tc001 url_contains on httpbin form" "exit=$EXIT, result=$RESULT"
|
||||
bad "tc001 url_contains on local form" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc002 — assert.js selector_visible against pinned httpbin /html
|
||||
# tc002 — assert.js selector_visible against the local content fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \
|
||||
'[{"kind": "selector_visible", "selector": "h1"}]' 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passed": true'; then
|
||||
ok "tc002 selector_visible h1 on httpbin /html"
|
||||
ok "tc002 selector_visible h1 on local content"
|
||||
else
|
||||
bad "tc002 selector_visible h1 on httpbin /html" "exit=$EXIT"
|
||||
bad "tc002 selector_visible h1 on local content" "exit=$EXIT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -105,7 +136,7 @@ fi
|
||||
# tc004 — batch.js navigate + wait + assert in one call
|
||||
# ---------------------------------------------------------------------------
|
||||
RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \
|
||||
'[{"action":"go","url":"https://httpbin.org/html"},{"action":"wait_load"},{"action":"assert","checks":[{"kind":"url_contains","text":"/html"}]}]' \
|
||||
"[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"wait_load\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/fixtures/content.html\"}]}]" \
|
||||
--summary-only 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passedSteps": 3'; then
|
||||
@@ -118,7 +149,7 @@ fi
|
||||
# tc005 — batch.js stops on failure
|
||||
# ---------------------------------------------------------------------------
|
||||
RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \
|
||||
'[{"action":"go","url":"https://httpbin.org/html"},{"action":"click","selector":"#does-not-exist-selector"},{"action":"go","url":"https://httpbin.org/forms/post"}]' 2>&1)
|
||||
"[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/does-not-exist\"}]},{\"action\":\"go\",\"url\":\"$FORM_URL\"}]" 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "1" ] && echo "$RESULT" | grep -q '"failedStep"'; then
|
||||
ok "tc005 batch stops on first failure"
|
||||
@@ -131,19 +162,19 @@ fi
|
||||
#
|
||||
# Set explicit viewport BEFORE screenshot so the two visual-diff runs have
|
||||
# the same dimensions. Without this the chromium headless window picks
|
||||
# whatever size it likes per run, and httpbin.org/html renders at different
|
||||
# whatever size it likes per run, and pages may render at different
|
||||
# sizes between runs (768×654 vs 765×672 observed), making pixel-diff
|
||||
# spuriously fail. This is documented in references/assertion-kinds.md.
|
||||
# ---------------------------------------------------------------------------
|
||||
rm -rf "$PWD/.aqe/visual-baselines/smoke_test_baseline"*
|
||||
vibium --headless viewport 1280 720 >/dev/null 2>&1 || true
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/visual-diff.js" --name smoke_test_baseline 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"baseline_created"'; then
|
||||
ok "tc006 visual-diff baseline created"
|
||||
else
|
||||
bad "tc006 visual-diff baseline created" "exit=$EXIT"
|
||||
bad "tc006 visual-diff baseline created" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -157,13 +188,13 @@ EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -qE '"(match|baseline_created)"'; then
|
||||
ok "tc007 visual-diff second run matches"
|
||||
else
|
||||
bad "tc007 visual-diff second run matches" "exit=$EXIT"
|
||||
bad "tc007 visual-diff second run matches" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc008 — check-injection.js clean page
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/check-injection.js" --include-hidden 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"severity": "none"'; then
|
||||
@@ -173,15 +204,15 @@ else
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc010 — intent-score.js submit_form on pinned httpbin form
|
||||
# tc010 — intent-score.js submit_form on the local form fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true
|
||||
vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/intent-score.js" --intent submit_form 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"intent": "submit_form"'; then
|
||||
ok "tc010 intent-score submit_form on httpbin form"
|
||||
ok "tc010 intent-score submit_form on local form"
|
||||
else
|
||||
bad "tc010 intent-score submit_form on httpbin form" "exit=$EXIT"
|
||||
bad "tc010 intent-score submit_form on local form" "exit=$EXIT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -45,11 +45,10 @@ function ensureDir(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
// Vibium screenshot quirks (verified against v26.3.18 on 2026-04-09):
|
||||
// 1. `vibium screenshot -o <path>` IGNORES the directory in <path>.
|
||||
// Only the basename is used, and the file is saved to
|
||||
// `~/Pictures/Vibium/<basename>`. We work around this by reading from
|
||||
// Vibium's actual output dir and copying to the requested location.
|
||||
// Vibium screenshot output changed across the supported 26.x releases.
|
||||
// Older releases ignore the directory passed to `-o` and save under
|
||||
// `~/Pictures/Vibium`; newer releases may honor the requested path. Accept
|
||||
// both behaviors so AQE init's compatible-version range stays usable.
|
||||
// 2. `--selector` flag does NOT exist on `vibium screenshot`. Selector-
|
||||
// scoped baselines are not supported in v26.3.x. We surface a clear
|
||||
// error if a caller passes one. Future Vibium versions may add it.
|
||||
@@ -69,21 +68,31 @@ function captureScreenshot(selector, outputPath) {
|
||||
);
|
||||
}
|
||||
const basename = path.basename(outputPath);
|
||||
const args = ['screenshot', '-o', basename, '--full-page'];
|
||||
const candidates = [
|
||||
outputPath,
|
||||
path.resolve(process.cwd(), basename),
|
||||
path.join(vibiumPicturesDir(), basename),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
fs.rmSync(candidate, { force: true });
|
||||
}
|
||||
const args = ['screenshot', '-o', outputPath, '--full-page'];
|
||||
const res = vibium(args);
|
||||
if (res.status !== 0) {
|
||||
throw new Error(`vibium screenshot failed: ${res.stderr.trim() || res.stdout.trim()}`);
|
||||
}
|
||||
// Vibium wrote the file to ~/Pictures/Vibium/<basename>, not outputPath.
|
||||
// Copy it to where the caller asked. Use copy-then-unlink so we leave
|
||||
// Vibium's own dir clean for the next run.
|
||||
const vibiumPath = path.join(vibiumPicturesDir(), basename);
|
||||
if (!fs.existsSync(vibiumPath)) {
|
||||
throw new Error(`screenshot output not created at ${vibiumPath} (vibium said: ${res.stdout.trim()})`);
|
||||
const screenshotPath = candidates.find((candidate) => fs.existsSync(candidate));
|
||||
if (!screenshotPath) {
|
||||
throw new Error(
|
||||
`screenshot output not created at any supported path (${candidates.join(', ')}; ` +
|
||||
`vibium said: ${res.stdout.trim()})`
|
||||
);
|
||||
}
|
||||
if (screenshotPath !== outputPath) {
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.copyFileSync(screenshotPath, outputPath);
|
||||
fs.unlinkSync(screenshotPath);
|
||||
}
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.copyFileSync(vibiumPath, outputPath);
|
||||
fs.unlinkSync(vibiumPath);
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
#
|
||||
# Jobs:
|
||||
# unit : vitest suite covering the 5 primitives + CommandEvalRunner
|
||||
# smoke : real Vibium install + 5-primitive smoke-test.sh against live
|
||||
# public fixtures (httpbin.org)
|
||||
# smoke : real Vibium install + 5-primitive smoke-test.sh against local
|
||||
# deterministic fixtures
|
||||
# eval : build the AQE CLI and run `aqe eval run --skill qe-browser`
|
||||
# to exercise the CommandEvalRunner dispatch path end-to-end
|
||||
# against real Vibium (closes devil's-advocate gap G2)
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
tests/unit/scripts/qe-browser-vibium-lib.test.ts
|
||||
|
||||
smoke:
|
||||
name: Smoke test (real Vibium + httpbin)
|
||||
name: Smoke test (real Vibium + local fixtures)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
needs: unit
|
||||
@@ -126,8 +126,6 @@ jobs:
|
||||
- name: Run qe-browser smoke test
|
||||
run: bash .claude/skills/qe-browser/scripts/smoke-test.sh
|
||||
env:
|
||||
# Smoke test uses public httpbin endpoints; set CI so the script
|
||||
# can short-circuit interactive retries if Vibium misbehaves.
|
||||
CI: 'true'
|
||||
|
||||
eval:
|
||||
@@ -179,6 +177,13 @@ jobs:
|
||||
chmod +x "$CHROME"
|
||||
head -3 "$CHROME"
|
||||
|
||||
- name: Start deterministic browser fixtures
|
||||
run: |
|
||||
node .claude/skills/qe-browser/fixtures/serve-skills.js > /tmp/qe-browser-fixtures.log 2>&1 &
|
||||
echo $! > /tmp/qe-browser-fixtures.pid
|
||||
curl --retry 5 --retry-connrefused --retry-delay 1 --fail \
|
||||
http://127.0.0.1:8088/fixtures/content.html > /dev/null
|
||||
|
||||
- name: Run aqe eval against qe-browser
|
||||
# The --model flag is required by the CLI but ignored by
|
||||
# CommandEvalRunner for command-mode suites. Exit 0 means all
|
||||
|
||||
@@ -52,26 +52,18 @@ setup:
|
||||
optional_tools:
|
||||
- pixelmatch
|
||||
- pngjs
|
||||
# NOTE: this yaml deliberately uses ONLY pinned public fixtures
|
||||
# (httpbin.org/*) so it can be run end-to-end by CommandEvalRunner without
|
||||
# any prerequisite services. Tests that need a local poisoned-HTML fixture
|
||||
# (the check-injection severity path) live in scripts/smoke-test.sh, which
|
||||
# starts fixtures/serve-skills.js out of band.
|
||||
# The CI workflow starts fixtures/serve-skills.js on loopback before this
|
||||
# suite runs. Keeping browser targets local removes rate limits and makes
|
||||
# every page stable across repeated visual comparisons.
|
||||
|
||||
fixtures:
|
||||
public_pinned:
|
||||
# Pinned public endpoints — chosen because they're stable, well-known, and
|
||||
# serve predictable forms / HTML. Per feedback_no_unverified_failure_modes,
|
||||
# these are the canonical "does the tool actually work" fixtures.
|
||||
httpbin_form:
|
||||
url: "https://httpbin.org/forms/post"
|
||||
description: "Classic simple form — custname, custtel, custemail, size, toppings"
|
||||
httpbin_html:
|
||||
url: "https://httpbin.org/html"
|
||||
description: "Static HTML page with known headings"
|
||||
httpbin_status_404:
|
||||
url: "https://httpbin.org/status/404"
|
||||
description: "Known 404 for testing no_failed_requests"
|
||||
local:
|
||||
form:
|
||||
url: "http://127.0.0.1:8088/fixtures/form.html"
|
||||
description: "Deterministic form with an email input and submit button"
|
||||
content:
|
||||
url: "http://127.0.0.1:8088/fixtures/content.html"
|
||||
description: "Deterministic static page with a known heading"
|
||||
|
||||
test_cases:
|
||||
# -------- assert.js --------
|
||||
@@ -81,10 +73,10 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/forms/post"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/form.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "url_contains", "text": "httpbin.org/forms"}]'
|
||||
'[{"kind": "url_contains", "text": "/fixtures/form.html"}]'
|
||||
expected:
|
||||
exit_code: 0
|
||||
json_fields:
|
||||
@@ -98,7 +90,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "selector_visible", "selector": "h1"}]'
|
||||
@@ -113,7 +105,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/assert.js --checks \
|
||||
'[{"kind": "url_contains", "text": "this-does-not-exist"}]'
|
||||
@@ -132,10 +124,10 @@ test_cases:
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/batch.js --steps \
|
||||
'[
|
||||
{"action": "go", "url": "https://httpbin.org/html"},
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"},
|
||||
{"action": "wait_load"},
|
||||
{"action": "assert", "checks": [
|
||||
{"kind": "url_contains", "text": "/html"},
|
||||
{"kind": "url_contains", "text": "/fixtures/content.html"},
|
||||
{"kind": "selector_visible", "selector": "h1"}
|
||||
]}
|
||||
]' --summary-only
|
||||
@@ -154,9 +146,11 @@ test_cases:
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/batch.js --steps \
|
||||
'[
|
||||
{"action": "go", "url": "https://httpbin.org/html"},
|
||||
{"action": "click", "selector": "#does-not-exist"},
|
||||
{"action": "go", "url": "https://httpbin.org/forms/post"}
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/content.html"},
|
||||
{"action": "assert", "checks": [
|
||||
{"kind": "url_contains", "text": "/does-not-exist"}
|
||||
]},
|
||||
{"action": "go", "url": "http://127.0.0.1:8088/fixtures/form.html"}
|
||||
]'
|
||||
expected:
|
||||
exit_code: 1
|
||||
@@ -177,7 +171,7 @@ test_cases:
|
||||
# different dimensions (768×654 vs 765×672 observed), making the
|
||||
# pixel-diff in tc007 spuriously fail. Mirrors scripts/smoke-test.sh.
|
||||
- "vibium --headless viewport 1280 720"
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
- "rm -rf .aqe/visual-baselines/eval_httpbin_html*"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/visual-diff.js \
|
||||
@@ -195,7 +189,7 @@ test_cases:
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless viewport 1280 720"
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/visual-diff.js \
|
||||
--name eval_httpbin_html --threshold 0.02
|
||||
@@ -212,7 +206,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/check-injection.js --include-hidden
|
||||
expected:
|
||||
@@ -221,15 +215,11 @@ test_cases:
|
||||
".status": "success"
|
||||
".output.checkInjection.severity": "none"
|
||||
|
||||
# GAP: the "poisoned-page detected with severity>=high" contract needs a
|
||||
# local fixture (fixtures/injection-poisoned.html) served by
|
||||
# fixtures/serve-skills.js. That's out of scope for this yaml — we keep
|
||||
# CommandEvalRunner dependency-free so it can run anywhere httpbin.org
|
||||
# is reachable. Coverage of the high-severity path is currently
|
||||
# The "poisoned-page detected with severity>=high" contract is currently
|
||||
# asserted outside this command suite. Coverage of the high-severity path is
|
||||
# only asserted by unit tests on check-injection.js (see
|
||||
# tests/unit/scripts/qe-browser-check-injection.test.ts). Follow-up:
|
||||
# either teach CommandEvalRunner to spawn the fixture server, or add a
|
||||
# tc009 to scripts/smoke-test.sh that starts/stops it out of band.
|
||||
# add tc009 against fixtures/injection-poisoned.html.
|
||||
|
||||
# -------- intent-score.js --------
|
||||
- id: tc010_intent_submit_form_on_httpbin
|
||||
@@ -238,7 +228,7 @@ test_cases:
|
||||
priority: critical
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/forms/post"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/form.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/intent-score.js \
|
||||
--intent submit_form
|
||||
@@ -255,7 +245,7 @@ test_cases:
|
||||
priority: medium
|
||||
input:
|
||||
setup:
|
||||
- "vibium --headless go https://httpbin.org/html"
|
||||
- "vibium --headless go http://127.0.0.1:8088/fixtures/content.html"
|
||||
command: |
|
||||
node .claude/skills/qe-browser/scripts/intent-score.js --intent fill_email
|
||||
expected:
|
||||
@@ -270,5 +260,4 @@ validation:
|
||||
notes: |
|
||||
Evaluation assumes:
|
||||
- `vibium` v26.3.x+ is on PATH (from `aqe init` or `npm install -g vibium`)
|
||||
- Network access to httpbin.org (public, stable)
|
||||
- Local fixtures server running on :8088 (started in setup.local_docs_server)
|
||||
- Local fixtures server running on 127.0.0.1:8088
|
||||
|
||||
@@ -74,6 +74,31 @@ const POISONED_HTML = `<!doctype html>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const CONTENT_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>QE Browser Content Fixture</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>QE Browser Content Fixture</h1>
|
||||
<p>This deterministic page exercises browser assertions and visual comparisons.</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
const FORM_HTML = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head><meta charset="utf-8"><title>QE Browser Form Fixture</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Contact form</h1>
|
||||
<form method="post" action="/fixtures/form.html">
|
||||
<label>Email <input id="email" name="email" type="email" autocomplete="email"></label>
|
||||
<button type="submit">Submit</button>
|
||||
</form>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
function notFound(res, message) {
|
||||
res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end(`404 Not Found: ${message}\n`);
|
||||
@@ -95,6 +120,18 @@ function serve(req, res) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (pathname === '/fixtures/content.html') {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
||||
res.end(CONTENT_HTML);
|
||||
return;
|
||||
}
|
||||
|
||||
if (pathname === '/fixtures/form.html') {
|
||||
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
||||
res.end(FORM_HTML);
|
||||
return;
|
||||
}
|
||||
|
||||
// Rewrite /foo/SKILL.md.html → /foo/SKILL.md (or similar) and serve wrapped.
|
||||
if (pathname.endsWith('.html')) {
|
||||
const mdPath = pathname.replace(/\.html$/, '');
|
||||
|
||||
@@ -292,7 +292,7 @@ function readInlineOrFile(value) {
|
||||
// "we couldn't run the test because the browser engine isn't installed."
|
||||
function emit(env) {
|
||||
process.stdout.write(`${JSON.stringify(env, null, 2)}\n`);
|
||||
if (env.status === 'success') return 0;
|
||||
if (env.status === 'success' || env.status === 'partial') return 0;
|
||||
if (env.status === 'skipped') return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# qe-browser smoke test
|
||||
#
|
||||
# Runs each helper script against pinned public fixtures (httpbin.org) and
|
||||
# Runs each helper script against deterministic local fixtures and
|
||||
# verifies the output structure. This is the script that gates PR-reopen
|
||||
# per ADR-091 Phase 3 — it MUST be run on a machine with vibium installed
|
||||
# before the qe-browser PR is considered safe to reopen.
|
||||
@@ -12,16 +12,25 @@
|
||||
# 2 — vibium binary not on PATH (precondition unmet)
|
||||
#
|
||||
# Per feedback_no_unverified_failure_modes.md, this is the script we
|
||||
# actually run, not just write. Per feedback_synthetic_fixtures_dont_count,
|
||||
# all fixtures are pinned public endpoints (httpbin.org) — no synthetic
|
||||
# stubs, no inline HTML.
|
||||
# actually run, not just write. The fixture pages are maintained with the
|
||||
# skill and served only on loopback, removing public-service rate limits.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
FIXTURE_PORT="${QE_BROWSER_FIXTURE_PORT:-8088}"
|
||||
FIXTURE_URL="${QE_BROWSER_FIXTURE_URL:-http://127.0.0.1:${FIXTURE_PORT}}"
|
||||
FIXTURE_PID=""
|
||||
|
||||
cleanup() {
|
||||
if [ -n "$FIXTURE_PID" ]; then
|
||||
kill "$FIXTURE_PID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf "$WORK_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
@@ -45,6 +54,28 @@ if ! command -v vibium >/dev/null 2>&1; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then
|
||||
QE_BROWSER_FIXTURE_PORT="$FIXTURE_PORT" \
|
||||
node "$SKILL_DIR/fixtures/serve-skills.js" >"$WORK_DIR/fixture-server.log" 2>&1 &
|
||||
FIXTURE_PID=$!
|
||||
fixture_ready=0
|
||||
for _attempt in 1 2 3 4 5; do
|
||||
if curl --silent --fail "$FIXTURE_URL/fixtures/content.html" >/dev/null 2>&1; then
|
||||
fixture_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "$fixture_ready" != "1" ]; then
|
||||
cat "$WORK_DIR/fixture-server.log" >&2
|
||||
echo -e "${RED}qe-browser fixture server did not become ready${NC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
CONTENT_URL="$FIXTURE_URL/fixtures/content.html"
|
||||
FORM_URL="$FIXTURE_URL/fixtures/form.html"
|
||||
|
||||
VIBIUM_VERSION=$(vibium --version 2>&1 | head -1)
|
||||
echo "Smoke testing against $VIBIUM_VERSION"
|
||||
echo "Skill dir: $SKILL_DIR"
|
||||
@@ -52,29 +83,29 @@ echo "Work dir: $WORK_DIR"
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc001 — assert.js url_contains against pinned httpbin form
|
||||
# tc001 — assert.js url_contains against the local form fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true
|
||||
vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \
|
||||
'[{"kind": "url_contains", "text": "httpbin.org/forms"}]' 2>&1)
|
||||
'[{"kind": "url_contains", "text": "/fixtures/form.html"}]' 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"status": "success"'; then
|
||||
ok "tc001 url_contains on httpbin form"
|
||||
ok "tc001 url_contains on local form"
|
||||
else
|
||||
bad "tc001 url_contains on httpbin form" "exit=$EXIT, result=$RESULT"
|
||||
bad "tc001 url_contains on local form" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc002 — assert.js selector_visible against pinned httpbin /html
|
||||
# tc002 — assert.js selector_visible against the local content fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/assert.js" --checks \
|
||||
'[{"kind": "selector_visible", "selector": "h1"}]' 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passed": true'; then
|
||||
ok "tc002 selector_visible h1 on httpbin /html"
|
||||
ok "tc002 selector_visible h1 on local content"
|
||||
else
|
||||
bad "tc002 selector_visible h1 on httpbin /html" "exit=$EXIT"
|
||||
bad "tc002 selector_visible h1 on local content" "exit=$EXIT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -93,7 +124,7 @@ fi
|
||||
# tc004 — batch.js navigate + wait + assert in one call
|
||||
# ---------------------------------------------------------------------------
|
||||
RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \
|
||||
'[{"action":"go","url":"https://httpbin.org/html"},{"action":"wait_load"},{"action":"assert","checks":[{"kind":"url_contains","text":"/html"}]}]' \
|
||||
"[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"wait_load\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/fixtures/content.html\"}]}]" \
|
||||
--summary-only 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"passedSteps": 3'; then
|
||||
@@ -106,7 +137,7 @@ fi
|
||||
# tc005 — batch.js stops on failure
|
||||
# ---------------------------------------------------------------------------
|
||||
RESULT=$(node "$SKILL_DIR/scripts/batch.js" --steps \
|
||||
'[{"action":"go","url":"https://httpbin.org/html"},{"action":"click","selector":"#does-not-exist-selector"},{"action":"go","url":"https://httpbin.org/forms/post"}]' 2>&1)
|
||||
"[{\"action\":\"go\",\"url\":\"$CONTENT_URL\"},{\"action\":\"assert\",\"checks\":[{\"kind\":\"url_contains\",\"text\":\"/does-not-exist\"}]},{\"action\":\"go\",\"url\":\"$FORM_URL\"}]" 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "1" ] && echo "$RESULT" | grep -q '"failedStep"'; then
|
||||
ok "tc005 batch stops on first failure"
|
||||
@@ -119,19 +150,19 @@ fi
|
||||
#
|
||||
# Set explicit viewport BEFORE screenshot so the two visual-diff runs have
|
||||
# the same dimensions. Without this the chromium headless window picks
|
||||
# whatever size it likes per run, and httpbin.org/html renders at different
|
||||
# whatever size it likes per run, and pages may render at different
|
||||
# sizes between runs (768×654 vs 765×672 observed), making pixel-diff
|
||||
# spuriously fail. This is documented in references/assertion-kinds.md.
|
||||
# ---------------------------------------------------------------------------
|
||||
rm -rf "$PWD/.aqe/visual-baselines/smoke_test_baseline"*
|
||||
vibium --headless viewport 1280 720 >/dev/null 2>&1 || true
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/visual-diff.js" --name smoke_test_baseline 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"baseline_created"'; then
|
||||
ok "tc006 visual-diff baseline created"
|
||||
else
|
||||
bad "tc006 visual-diff baseline created" "exit=$EXIT"
|
||||
bad "tc006 visual-diff baseline created" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -145,13 +176,13 @@ EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -qE '"(match|baseline_created)"'; then
|
||||
ok "tc007 visual-diff second run matches"
|
||||
else
|
||||
bad "tc007 visual-diff second run matches" "exit=$EXIT"
|
||||
bad "tc007 visual-diff second run matches" "exit=$EXIT, result=$RESULT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc008 — check-injection.js clean page
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/html >/dev/null 2>&1 || true
|
||||
vibium --headless go "$CONTENT_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/check-injection.js" --include-hidden 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"severity": "none"'; then
|
||||
@@ -161,15 +192,15 @@ else
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tc010 — intent-score.js submit_form on pinned httpbin form
|
||||
# tc010 — intent-score.js submit_form on the local form fixture
|
||||
# ---------------------------------------------------------------------------
|
||||
vibium --headless go https://httpbin.org/forms/post >/dev/null 2>&1 || true
|
||||
vibium --headless go "$FORM_URL" >/dev/null 2>&1 || true
|
||||
RESULT=$(node "$SKILL_DIR/scripts/intent-score.js" --intent submit_form 2>&1)
|
||||
EXIT=$?
|
||||
if [ "$EXIT" = "0" ] && echo "$RESULT" | grep -q '"intent": "submit_form"'; then
|
||||
ok "tc010 intent-score submit_form on httpbin form"
|
||||
ok "tc010 intent-score submit_form on local form"
|
||||
else
|
||||
bad "tc010 intent-score submit_form on httpbin form" "exit=$EXIT"
|
||||
bad "tc010 intent-score submit_form on local form" "exit=$EXIT"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -45,11 +45,10 @@ function ensureDir(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
// Vibium screenshot quirks (verified against v26.3.18 on 2026-04-09):
|
||||
// 1. `vibium screenshot -o <path>` IGNORES the directory in <path>.
|
||||
// Only the basename is used, and the file is saved to
|
||||
// `~/Pictures/Vibium/<basename>`. We work around this by reading from
|
||||
// Vibium's actual output dir and copying to the requested location.
|
||||
// Vibium screenshot output changed across the supported 26.x releases.
|
||||
// Older releases ignore the directory passed to `-o` and save under
|
||||
// `~/Pictures/Vibium`; newer releases may honor the requested path. Accept
|
||||
// both behaviors so AQE init's compatible-version range stays usable.
|
||||
// 2. `--selector` flag does NOT exist on `vibium screenshot`. Selector-
|
||||
// scoped baselines are not supported in v26.3.x. We surface a clear
|
||||
// error if a caller passes one. Future Vibium versions may add it.
|
||||
@@ -69,21 +68,31 @@ function captureScreenshot(selector, outputPath) {
|
||||
);
|
||||
}
|
||||
const basename = path.basename(outputPath);
|
||||
const args = ['screenshot', '-o', basename, '--full-page'];
|
||||
const candidates = [
|
||||
outputPath,
|
||||
path.resolve(process.cwd(), basename),
|
||||
path.join(vibiumPicturesDir(), basename),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
fs.rmSync(candidate, { force: true });
|
||||
}
|
||||
const args = ['screenshot', '-o', outputPath, '--full-page'];
|
||||
const res = vibium(args);
|
||||
if (res.status !== 0) {
|
||||
throw new Error(`vibium screenshot failed: ${res.stderr.trim() || res.stdout.trim()}`);
|
||||
}
|
||||
// Vibium wrote the file to ~/Pictures/Vibium/<basename>, not outputPath.
|
||||
// Copy it to where the caller asked. Use copy-then-unlink so we leave
|
||||
// Vibium's own dir clean for the next run.
|
||||
const vibiumPath = path.join(vibiumPicturesDir(), basename);
|
||||
if (!fs.existsSync(vibiumPath)) {
|
||||
throw new Error(`screenshot output not created at ${vibiumPath} (vibium said: ${res.stdout.trim()})`);
|
||||
const screenshotPath = candidates.find((candidate) => fs.existsSync(candidate));
|
||||
if (!screenshotPath) {
|
||||
throw new Error(
|
||||
`screenshot output not created at any supported path (${candidates.join(', ')}; ` +
|
||||
`vibium said: ${res.stdout.trim()})`
|
||||
);
|
||||
}
|
||||
if (screenshotPath !== outputPath) {
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.copyFileSync(screenshotPath, outputPath);
|
||||
fs.unlinkSync(screenshotPath);
|
||||
}
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.copyFileSync(vibiumPath, outputPath);
|
||||
fs.unlinkSync(vibiumPath);
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
|
||||
Generated
+3
-3
@@ -11442,9 +11442,9 @@
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
|
||||
"integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
|
||||
"version": "8.21.3",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz",
|
||||
"integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
|
||||
@@ -106,6 +106,20 @@ describe('qe-browser fixtures/serve-skills', () => {
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
describe('browser eval pages', () => {
|
||||
it('serves deterministic content and form pages without an external service', async () => {
|
||||
const { port } = await startServer({ QE_BROWSER_FIXTURE_PORT: '18806' });
|
||||
const content = await fetch('127.0.0.1', port, '/fixtures/content.html');
|
||||
const form = await fetch('127.0.0.1', port, '/fixtures/form.html');
|
||||
|
||||
expect(content.status).toBe(200);
|
||||
expect(content.body).toContain('<h1>QE Browser Content Fixture</h1>');
|
||||
expect(form.status).toBe(200);
|
||||
expect(form.body).toContain('input id="email"');
|
||||
expect(form.body).toContain('button type="submit"');
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
// M3 regression: the previous startsWith() check could false-pass on
|
||||
// sibling directories with shared prefix. path.relative() is the
|
||||
// canonical traversal guard.
|
||||
|
||||
@@ -228,6 +228,10 @@ describe('qe-browser lib/vibium', () => {
|
||||
expect(captured(lib.envelope({ operation: 'x', summary: 'ok', status: 'success' }))).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 0 for a completed operation with status: partial', () => {
|
||||
expect(captured(lib.envelope({ operation: 'x', summary: 'empty', status: 'partial' }))).toBe(0);
|
||||
});
|
||||
|
||||
it('returns 1 for status: failed', () => {
|
||||
expect(captured(lib.envelope({ operation: 'x', summary: 'no', status: 'failed' }))).toBe(1);
|
||||
});
|
||||
|
||||
@@ -84,7 +84,7 @@ describe('CommandEvalRunner — qe-browser integration', () => {
|
||||
}
|
||||
|
||||
if (cmd.includes('scripts/batch.js')) {
|
||||
const stopOnFail = cmd.includes('#does-not-exist');
|
||||
const stopOnFail = cmd.includes('/does-not-exist');
|
||||
if (stopOnFail) {
|
||||
return {
|
||||
status: 1,
|
||||
|
||||
Reference in New Issue
Block a user