mirror of
https://github.com/CopilotKit/CopilotKit.git
synced 2026-09-14 16:26:20 +08:00
showcase-ops: replace GH Actions cron/alert workflows with in-cluster Railway service (#4113)
## Summary New in-cluster Railway service `showcase-ops` replaces four GitHub Actions cron/alert workflows for the CopilotKit showcase. PocketBase-backed state, declarative YAML alert DSL, a reusable probe-driver architecture, and server-side-filtered SSE into the dashboard. ## Why The legacy showcase alert stack was four ad-hoc cron workflows (`showcase_smoke-monitor.yml`, `showcase_drift-detection.yml`, `showcase_drift-report.yml`, `showcase_redirect-report.yml`) each persisting state via `actions/cache` + loose JSON + scattered `jq -n --rawfile` message assembly. This PR replaces that with: - One alert-state store (PocketBase) instead of four ad-hoc persistence mechanisms. - One structured-payload renderer (two-phase filter extraction, triple-brace safe-field allow-list) instead of ~20 scattered `jq` / `toJSON(format())` / `printf '%b'` sites. - One typed PB client with unified 401 re-auth, 429 `Retry-After`, network/5xx backoff. - One transition detector (pure function) — state-machine logic exists in exactly one place. - One probe-driver contract (`ProbeDriver<Input, Signal>`) + discovery-source contract (`DiscoverySource<Output>`) so every future probe shares one loader + invoker + typed-error taxonomy, configured declaratively via YAML. Adding a new dimension is now: add a driver + YAML + registry line. No core changes required. ## Dimensions covered `health`, `smoke`, `image_drift`, `version_drift`, `pin_drift`, `e2e_smoke` (scaffolded, runner deferred), `redirect_decommission`, `aimock_wiring`, `deploy`. ## Deleted in this PR - `.github/workflows/showcase_smoke-monitor.yml` - `.github/workflows/showcase_drift-detection.yml` - `.github/workflows/showcase_drift-report.yml` - `.github/workflows/showcase_redirect-report.yml` - `showcase/scripts/generate-status.ts` - `showcase/shell/src/data/status.json` - `showcase/shell-dashboard/src/lib/status.ts` - The `notify:` job of `showcase_deploy.yml` (replaced by a signed POST to `/webhooks/deploy`) ## Commit layout (12) Each commit is a coherent area; commit-by-commit review works cleanly. 1. \`chore(repo)\` — root workspace config + top-level showcase docs 2. \`feat(examples/v2)\` — rename interrupts-langraph → interrupts-langgraph + integration cleanup 3. \`chore(showcase/packages)\` — QA markdown parity across all packages + integration tooling 4. \`feat(showcase/shell-docs)\` — error-boundary consolidation + docs components 5. \`feat(showcase/shell-dashboard)\` — live PocketBase status wiring + tests 6. \`refactor(showcase/scripts)\` — extract pure-TS cores for pin-drift + redirect-decommission 7. \`feat(showcase/pocketbase)\` — service (migrations + hooks + Docker) 8. \`feat(showcase/ops)\` — core service (HTTP + HMAC + metrics + orchestrator + rules + render + storage + scheduler + targets) 9. \`feat(showcase/ops)\` — alert rule DSL + engine + YAML configs 10. \`feat(showcase/ops)\` — probe driver architecture + 6 drivers + 2 discovery sources + YAML configs 11. \`feat(showcase/ops)\` — Dockerfile + README + rotation drill 12. \`ci\` — retire legacy cron workflows + \`test_*\` rename + showcase_deploy webhook wiring ## Test plan - [x] \`pnpm -F @copilotkit/showcase-ops typecheck\` — clean - [x] \`pnpm -F @copilotkit/showcase-ops test\` — 692 tests passing across 37 files - [x] \`pnpm -F @copilotkit/showcase-ops build\` — clean - [x] \`showcase/shell-dashboard\` typecheck + test + build — clean - [x] \`docker build -f showcase/ops/Dockerfile .\` — builds clean (Playwright chromium included) - [x] \`docker build -f showcase/pocketbase/Dockerfile showcase/pocketbase/\` — builds clean - [x] \`docker build -f showcase/shell-dashboard/Dockerfile .\` — builds clean - [ ] Deploy \`showcase-ops\` + \`showcase-pocketbase\` to Railway and observe one red→red and one red→green transition - [ ] Fire deploy webhook with valid HMAC — alert lands in \`#oss-alerts\` - [ ] Fire deploy webhook with stale timestamp (>300s) or wrong signature — request rejected, \`webhook_rejections\` counter increments - [ ] Playwright visual regression at iPhone SE (375×667), iPhone 14 Pro Max (430×932), Desktop (1440×900) under all-green, mixed, and all-unknown states ## Design spec [Notion: showcase-ops design spec](https://www.notion.so/3493aa381852811aa21bf09b2efa4a2d) · [Probe drivers design](https://www.notion.so/34a3aa38185281bf8066f107401b2979)
This commit is contained in:
Jordan Ritter
File diff suppressed because it is too large
Load Diff
@@ -1,214 +0,0 @@
|
||||
name: "Showcase: Drift Detection"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 */6 * * *" # L1-L3 every 6 hours (0/6/12/18 UTC)
|
||||
- cron: "0 0 * * *" # L4 daily at midnight UTC
|
||||
- cron: "0 9 * * 1" # Weekly version drift (Monday 9am UTC) — unchanged
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
run_version_check:
|
||||
description: "Run version drift check"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
drift-detection:
|
||||
name: E2E Smoke Suite
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
# Only run on schedule (excluding weekly version-drift slot) or manual dispatch
|
||||
if: (github.event_name == 'schedule' && github.event.schedule != '0 9 * * 1') || github.event_name == 'workflow_dispatch'
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
lfs: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: showcase/tests
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright
|
||||
working-directory: showcase/tests
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run L1-L3 tests
|
||||
id: l1_l3
|
||||
working-directory: showcase/tests
|
||||
env:
|
||||
CI: true
|
||||
run: |
|
||||
npx playwright test integration-smoke --grep "@health|@agent|@chat" --reporter=github 2>&1 | tee /tmp/e2e-output.log
|
||||
exit ${PIPESTATUS[0]}
|
||||
|
||||
- name: Run L4 tests (daily only)
|
||||
id: l4
|
||||
if: github.event.schedule == '0 0 * * *' || github.event_name == 'workflow_dispatch'
|
||||
working-directory: showcase/tests
|
||||
env:
|
||||
CI: true
|
||||
run: |
|
||||
npx playwright test integration-smoke --grep "@tools" --reporter=github 2>&1 | tee -a /tmp/e2e-output.log
|
||||
exit ${PIPESTATUS[0]}
|
||||
|
||||
- name: Extract failure summary
|
||||
id: failures
|
||||
if: failure()
|
||||
run: |
|
||||
# Extract failed test names and error messages from playwright output
|
||||
DETAILS=$(grep -E "^\s+\d+\)|Error:" /tmp/e2e-output.log 2>/dev/null | head -15 | sed 's/"/\\"/g' || echo "See CI logs for details")
|
||||
DETAILS="${DETAILS:0:1200}"
|
||||
|
||||
{
|
||||
echo "details<<EOFEOF"
|
||||
echo "$DETAILS"
|
||||
echo "EOFEOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build Slack payload
|
||||
if: failure()
|
||||
id: slack-payload
|
||||
env:
|
||||
DETAILS_RAW: ${{ steps.failures.outputs.details }}
|
||||
run: |
|
||||
# Strip ANSI sequences (SGR, OSC, and G0/G1 charset designators), then truncate
|
||||
# to 200 bytes and drop any trailing partial UTF-8 bytes so we don't emit mojibake.
|
||||
SUMMARY=$(printf '%s' "$DETAILS_RAW" | head -3 \
|
||||
| sed -E 's/\x1b\[[0-9;?]*[A-Za-z]//g; s/\x1b\][^\x07]*\x07//g; s/\x1b[()][A-Za-z0-9]//g' \
|
||||
| head -c 200 | iconv -f UTF-8 -t UTF-8//IGNORE)
|
||||
if [ -z "$SUMMARY" ]; then
|
||||
SUMMARY="(no failure detail captured — see job log)"
|
||||
fi
|
||||
URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
JOB_URL="${URL}/job/${{ github.job }}"
|
||||
|
||||
SLACK_MSG=$(mktemp)
|
||||
SLACK_PAYLOAD=$(mktemp)
|
||||
|
||||
# Build the message with REAL newlines, then hand it to jq via --rawfile so escaping is handled correctly.
|
||||
{
|
||||
printf ':x: *Showcase E2E suite failed*\n'
|
||||
printf '<%s|View run> · <%s|View job>\n' "$URL" "$JOB_URL"
|
||||
printf '```\n%s\n```\n' "$SUMMARY"
|
||||
} > "$SLACK_MSG"
|
||||
jq -n --rawfile text "$SLACK_MSG" '{text: $text}' > "$SLACK_PAYLOAD"
|
||||
rm -f "$SLACK_MSG"
|
||||
echo "payload_path=${SLACK_PAYLOAD}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Post failure to Slack
|
||||
if: failure()
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload-file-path: ${{ steps.slack-payload.outputs.payload_path }}
|
||||
|
||||
- name: Clean up Slack payload tmpfile
|
||||
if: always() && steps.slack-payload.outputs.payload_path
|
||||
run: rm -f "${{ steps.slack-payload.outputs.payload_path }}"
|
||||
|
||||
version-drift:
|
||||
name: Version Drift Report
|
||||
runs-on: ubuntu-latest
|
||||
# Only on weekly Monday schedule or manual trigger
|
||||
if: github.event.schedule == '0 9 * * 1' || github.event.inputs.run_version_check == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Check Python package drift
|
||||
id: python_drift
|
||||
run: |
|
||||
report=""
|
||||
for pkg_dir in showcase/packages/*/; do
|
||||
req_file="$pkg_dir/requirements.txt"
|
||||
[ -f "$req_file" ] || continue
|
||||
slug=$(basename "$pkg_dir")
|
||||
echo "=== $slug ==="
|
||||
while IFS= read -r line; do
|
||||
# Skip empty lines and comments
|
||||
[[ -z "$line" || "$line" =~ ^# ]] && continue
|
||||
# Extract package name and pinned version
|
||||
if [[ "$line" =~ ^([a-zA-Z0-9_-]+)\[?[a-z]*\]?==([0-9.]+) ]]; then
|
||||
pkg="${BASH_REMATCH[1]}"
|
||||
pinned="${BASH_REMATCH[2]}"
|
||||
latest=$(pip index versions "$pkg" 2>/dev/null | head -1 | grep -oP '\([\d.]+\)' | tr -d '()' || echo "unknown")
|
||||
if [ "$latest" != "unknown" ] && [ "$latest" != "$pinned" ]; then
|
||||
echo " $pkg: pinned=$pinned latest=$latest"
|
||||
report+="| $slug | $pkg | $pinned | $latest |\n"
|
||||
fi
|
||||
fi
|
||||
done < "$req_file"
|
||||
done
|
||||
echo "report<<EOF" >> $GITHUB_OUTPUT
|
||||
echo -e "$report" >> $GITHUB_OUTPUT
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Check npm package drift
|
||||
id: npm_drift
|
||||
run: |
|
||||
tmpfile=$(mktemp)
|
||||
for pkg_dir in showcase/packages/*/; do
|
||||
pkg_json="$pkg_dir/package.json"
|
||||
[ -f "$pkg_json" ] || continue
|
||||
slug=$(basename "$pkg_dir")
|
||||
echo "=== $slug ==="
|
||||
# Check key deps: @copilotkit/*, @mastra/*, @ag-ui/*
|
||||
node -e "
|
||||
const pkg = require('./$pkg_json');
|
||||
const deps = {...(pkg.dependencies || {})};
|
||||
const interesting = Object.entries(deps).filter(([k]) =>
|
||||
k.startsWith('@copilotkit/') || k.startsWith('@copilotkitnext/') ||
|
||||
k.startsWith('@mastra/') || k.startsWith('@ag-ui/') ||
|
||||
k === 'langchain' || k === 'langgraph'
|
||||
);
|
||||
interesting.forEach(([name, version]) => {
|
||||
console.log(name + '=' + version);
|
||||
});
|
||||
" 2>/dev/null | while IFS='=' read -r name version; do
|
||||
if [[ "$version" =~ ^[0-9] ]]; then
|
||||
latest=$(npm view "$name" version 2>/dev/null || echo "unknown")
|
||||
if [ "$latest" != "unknown" ] && [ "$latest" != "$version" ]; then
|
||||
echo " $name: pinned=$version latest=$latest"
|
||||
echo "| $slug | $name | $version | $latest |" >> "$tmpfile"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
report=$(cat "$tmpfile" 2>/dev/null || echo "")
|
||||
rm -f "$tmpfile"
|
||||
echo "report<<EOF" >> $GITHUB_OUTPUT
|
||||
echo -e "$report" >> $GITHUB_OUTPUT
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Notify Slack (version drift)
|
||||
if: steps.python_drift.outputs.report != '' || steps.npm_drift.outputs.report != ''
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
{ "text": ":warning: *Version drift*: dependency updates available | <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" }
|
||||
|
||||
- name: Notify Slack (version drift failure)
|
||||
if: failure()
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
{ "text": ":x: *Version drift check*: failed | <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" }
|
||||
@@ -1,234 +0,0 @@
|
||||
name: "Showcase: Drift Report"
|
||||
|
||||
# Weekly report on pin-drift baseline (validate-pins FAIL count).
|
||||
# Complements showcase_validate.yml's ratchet gating by surfacing the
|
||||
# current backlog even when no PR is open. Goal: keep the drift count
|
||||
# visible so the drift-to-zero commitment stays in front of the team.
|
||||
# Drift-to-zero work is tracked in GitHub issue #4047.
|
||||
#
|
||||
# Reference: Notion: "Showcase Drift Baseline Runbook" (accessible to
|
||||
# copilotkit.ai team members only; not linked here to avoid embedding
|
||||
# private URLs in a public workflow).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Monday 10:00 UTC — intentionally offset from showcase_drift-detection.yml's
|
||||
# Monday 09:00 UTC version-drift slot so the two reports do not pile up
|
||||
# in Slack simultaneously.
|
||||
- cron: "0 10 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
# id-token: write is required for Depot OIDC auth (runs-on: depot-ubuntu-*).
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
report:
|
||||
name: Weekly Pin-Drift Report
|
||||
# Hoist the Slack webhook into an env var so step-level `if:`
|
||||
# expressions can reference it — `secrets.*` is not a valid
|
||||
# named-value inside `if:` and causes a workflow startup failure.
|
||||
env:
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
# Depot (Startup plan, unlimited minutes) for pnpm cache across
|
||||
# scheduled runs — keeps the weekly ratchet cheap.
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
timeout-minutes: 15
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# pnpm must be set up BEFORE setup-node so that `cache: 'pnpm'` can
|
||||
# detect the pnpm binary when wiring up the store cache.
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v4.4.0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
# Cache the pnpm store so weekly cold-cache runs don't re-fetch
|
||||
# the full dependency set every Monday.
|
||||
cache: "pnpm"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Read baseline
|
||||
id: baseline
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Emit `count|hash` so we can ratchet on both; surface "SET DRIFTED"
|
||||
# when the count matches but the hash differs.
|
||||
set +e
|
||||
baseline_tuple=$(node -e "
|
||||
let v;
|
||||
try {
|
||||
v = require('./showcase/scripts/fail-baseline.json');
|
||||
} catch (e) {
|
||||
console.error('fail-baseline.json: JSON syntax error: ' + e.message);
|
||||
process.exit(2);
|
||||
}
|
||||
const c = v.validatePinsFailCount;
|
||||
const h = v.validatePinsFailHash;
|
||||
if (typeof c !== 'number' || !Number.isInteger(c) || c < 0) {
|
||||
console.error('fail-baseline.json: schema failure: validatePinsFailCount must be a non-negative integer');
|
||||
process.exit(3);
|
||||
}
|
||||
if (typeof h !== 'string' || !/^[0-9a-f]{64}$/.test(h)) {
|
||||
console.error('fail-baseline.json: schema failure: validatePinsFailHash must be a 64-char lowercase hex SHA-256');
|
||||
process.exit(3);
|
||||
}
|
||||
console.log(c + '|' + h);
|
||||
")
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
# Preserve node's distinct rc (2=JSON syntax, 3=schema) in the
|
||||
# annotation so the weekly log pinpoints the cause.
|
||||
echo "::error::Failed to read baseline from showcase/scripts/fail-baseline.json (node exit=$rc; 2=JSON syntax, 3=schema)"
|
||||
exit "$rc"
|
||||
fi
|
||||
baseline_count="${baseline_tuple%%|*}"
|
||||
baseline_hash="${baseline_tuple##*|}"
|
||||
echo "count=$baseline_count" >> "$GITHUB_OUTPUT"
|
||||
echo "hash=$baseline_hash" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run validate-pins
|
||||
id: validate
|
||||
working-directory: showcase/scripts
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Temporarily disable -e around validate-pins so we can capture its
|
||||
# exit code ourselves (we expect 0 or 1 as legitimate outcomes and
|
||||
# need to distinguish 2+ as an internal crash). pipefail stays on:
|
||||
# the `{ grep || true; }` scoping below relies on pipefail being
|
||||
# enabled to actually deliver on its stated protection (scope no-
|
||||
# match tolerance to grep alone without swallowing producer/head
|
||||
# failures). -u stays on to catch accidental unset-var reads.
|
||||
set +e
|
||||
stderr_file=$(mktemp)
|
||||
stdout=$(pnpm exec tsx validate-pins.ts 2>"$stderr_file")
|
||||
rc=$?
|
||||
stderr=$(cat "$stderr_file")
|
||||
rm -f "$stderr_file"
|
||||
set -e
|
||||
printf '%s\n' "$stdout"
|
||||
printf '%s\n' "$stderr" >&2
|
||||
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
||||
# Preserve validate-pins.ts's distinct exit code (2=EXIT_INTERNAL,
|
||||
# 3=EXIT_UNREADABLE, 4+=future) so the weekly job signal
|
||||
# distinguishes "validator crashed" from "pin drift found"
|
||||
# (rc=1). Matches the same pattern in showcase_validate.yml.
|
||||
echo "::error::validate-pins.ts exited with unexpected code $rc"
|
||||
exit "$rc"
|
||||
fi
|
||||
# Scope grep's no-match tolerance to grep alone: a trailing
|
||||
# `|| true` on the whole pipeline would defeat `pipefail` (if it
|
||||
# were enabled) and swallow producer/head failures too. Wrapping
|
||||
# just grep in `{ ... || true; }` keeps other stages' errors
|
||||
# visible while still tolerating "Summary line absent" (which is
|
||||
# then reported explicitly by the `[ -z "$summary_line" ]` check).
|
||||
summary_line=$(printf '%s\n' "$stdout" | { grep -E '^[[:space:]]*Summary:' || true; } | head -n 1)
|
||||
if [ -z "$summary_line" ]; then
|
||||
echo "::error::Could not find validate-pins 'Summary:' line"
|
||||
exit 1
|
||||
fi
|
||||
actual=$(printf '%s\n' "$summary_line" | grep -oE '\bFAIL=[0-9]+\b' | head -n 1 | cut -d= -f2)
|
||||
if [ -z "${actual:-}" ] || ! [[ "$actual" =~ ^[0-9]+$ ]]; then
|
||||
echo "::error::Could not parse FAIL=<int> from Summary line"
|
||||
exit 1
|
||||
fi
|
||||
# Hash the sorted, deduplicated `[FAIL] ...` lines (stderr only),
|
||||
# matching the identical computation in showcase_validate.yml. This
|
||||
# catches the "count equal but set drifted" case: one FAIL healed
|
||||
# while another regressed.
|
||||
#
|
||||
# Scope grep's no-match tolerance to grep alone by wrapping just
|
||||
# the grep stage in `{ ... || true; }`. Without this, a zero-FAIL
|
||||
# week (grep exit 1 = no match) either fails the pipeline (if
|
||||
# pipefail is enabled) or — more insidiously — silently yields
|
||||
# the empty-input SHA-256 (`e3b0c44...`), which then mismatches
|
||||
# the recorded baseline hash and spuriously fires a "SET DRIFTED"
|
||||
# alert on a clean week. A trailing `|| true` on the whole
|
||||
# pipeline would defeat `pipefail` and also swallow real
|
||||
# sort/shasum/cut failures; we only tolerate grep's no-match.
|
||||
actual_hash=$(printf '%s\n' "$stderr" | { grep -E '^\[FAIL\]' || true; } | LC_ALL=C sort -u | shasum -a 256 | cut -d' ' -f1)
|
||||
# Classify the week into one of three states for the weekly Slack
|
||||
# payload. "FAIL=N" was reframed to "DRIFT=N" because the old
|
||||
# wording read as a regression even when N matched the ratchet
|
||||
# baseline exactly (a stable week). Presentation-only — the
|
||||
# underlying ratchet logic, baseline file format, and hashing are
|
||||
# unchanged.
|
||||
# stable — count matches baseline AND hash matches
|
||||
# REGRESSION — count grew OR hash mismatch (set drifted)
|
||||
# IMPROVED — count shrank (baseline needs ratcheting)
|
||||
baseline_count="${{ steps.baseline.outputs.count }}"
|
||||
baseline_hash="${{ steps.baseline.outputs.hash }}"
|
||||
run_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
if [ "$actual" -gt "$baseline_count" ] || { [ "$actual" -eq "$baseline_count" ] && [ "$actual_hash" != "$baseline_hash" ]; }; then
|
||||
set_status="REGRESSION"
|
||||
delta=$((actual - baseline_count))
|
||||
slack_text=":rotating_light: *Showcase pin-drift (weekly)*: DRIFT=${actual} (baseline ${baseline_count}, +${delta}) [REGRESSION] | <${run_url}|View run>"
|
||||
elif [ "$actual" -lt "$baseline_count" ]; then
|
||||
set_status="IMPROVED"
|
||||
delta=$((baseline_count - actual))
|
||||
slack_text=":chart_with_downwards_trend: *Showcase pin-drift (weekly)*: DRIFT=${actual} (baseline ${baseline_count}, -${delta}) [IMPROVED, ratchet me]"
|
||||
else
|
||||
set_status="stable"
|
||||
slack_text=":chart_with_downwards_trend: *Showcase pin-drift (weekly)*: DRIFT=${actual} (baseline ${baseline_count}) [stable]"
|
||||
fi
|
||||
echo "actual=$actual" >> "$GITHUB_OUTPUT"
|
||||
echo "actual_hash=$actual_hash" >> "$GITHUB_OUTPUT"
|
||||
echo "set_status=$set_status" >> "$GITHUB_OUTPUT"
|
||||
# Emit Slack text via EOF delimiter so embedded special chars
|
||||
# (colons, pipes, angle brackets) can't break $GITHUB_OUTPUT parsing.
|
||||
{
|
||||
echo "slack_text<<__PIN_DRIFT_EOF__"
|
||||
printf '%s\n' "$slack_text"
|
||||
echo "__PIN_DRIFT_EOF__"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Notify Slack (weekly drift report)
|
||||
# Skip cleanly when the webhook secret is unset (forks / pre-provision)
|
||||
# rather than failing the job. Counts still appear in the job log.
|
||||
if: ${{ env.SLACK_WEBHOOK != '' }}
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
# Defensive: wrap the pre-composed Slack text via toJSON(...) so
|
||||
# that any characters that would break the JSON payload (quotes,
|
||||
# backslashes, newlines) are safely JSON-encoded instead of
|
||||
# injected as raw text. The text itself is composed in the
|
||||
# validate step above so we can emit one of three state-specific
|
||||
# messages (stable / REGRESSION / IMPROVED).
|
||||
payload: |
|
||||
{ "text": ${{ toJSON(steps.validate.outputs.slack_text) }} }
|
||||
|
||||
- name: Log result (no Slack)
|
||||
if: ${{ env.SLACK_WEBHOOK == '' }}
|
||||
run: |
|
||||
echo "::warning::SLACK_WEBHOOK_OSS_ALERTS not set; weekly drift report not sent to Slack."
|
||||
echo "Weekly pin-drift report: DRIFT=${{ steps.validate.outputs.actual }} baseline=${{ steps.baseline.outputs.count }} set_status=${{ steps.validate.outputs.set_status }} actual_hash=${{ steps.validate.outputs.actual_hash }} baseline_hash=${{ steps.baseline.outputs.hash }}"
|
||||
|
||||
- name: Notify Slack (job failure)
|
||||
# Surface silent crashes (baseline read, validate-pins internal error,
|
||||
# Slack post failure) so the weekly report doesn't fail invisibly.
|
||||
if: ${{ failure() && env.SLACK_WEBHOOK != '' }}
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
# Defensive: wrap dynamic values via toJSON(format(...)) so that
|
||||
# if repository/run_id ever contains characters that would break
|
||||
# the JSON payload (quotes, backslashes, newlines), the value is
|
||||
# safely JSON-encoded instead of injected as raw text. Mirrors
|
||||
# the success payload above for consistency.
|
||||
payload: |
|
||||
{ "text": ${{ toJSON(format(':x: *Showcase drift report*: job failed | <https://github.com/{0}/actions/runs/{1}|View run>', github.repository, github.run_id)) }} }
|
||||
@@ -1,60 +0,0 @@
|
||||
name: "Showcase: SEO Redirect Decommission Report"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 1st of each month at 9am UTC
|
||||
- cron: "0 9 1 * *"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
decommission-report:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
cache: pnpm
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run decommission report
|
||||
id: report
|
||||
env:
|
||||
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
|
||||
POSTHOG_PROJECT_ID: ${{ secrets.POSTHOG_PROJECT_ID }}
|
||||
run: |
|
||||
# Run report in Slack format, capture output
|
||||
set +e
|
||||
npx tsx showcase/scripts/redirect-decommission-report.ts --slack > report.txt 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
|
||||
if [ $EXIT_CODE -eq 2 ]; then
|
||||
echo "No decommission candidates — skipping Slack notification"
|
||||
echo "has_candidates=false" >> "$GITHUB_OUTPUT"
|
||||
elif [ $EXIT_CODE -eq 0 ]; then
|
||||
echo "has_candidates=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Report failed with exit code $EXIT_CODE"
|
||||
cat report.txt
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Build Slack payload
|
||||
if: steps.report.outputs.has_candidates == 'true'
|
||||
run: |
|
||||
jq -n --rawfile text report.txt '{"text": $text}' > slack-payload.json
|
||||
|
||||
- name: Post to Slack
|
||||
if: steps.report.outputs.has_candidates == 'true'
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload-file-path: slack-payload.json
|
||||
@@ -1,463 +0,0 @@
|
||||
name: "Showcase: Smoke Monitor"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "*/15 * * * *"
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
smoke-check:
|
||||
name: Smoke Check
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
packages: read
|
||||
actions: write
|
||||
steps:
|
||||
- name: Checkout (starters dir only)
|
||||
# Sparse checkout of showcase/starters/ so the "Check image drift"
|
||||
# step can enumerate starter slugs from the filesystem. This keeps
|
||||
# the starter list a single source of truth (directory names under
|
||||
# showcase/starters/) instead of a literal list duplicated in this
|
||||
# workflow.
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: showcase/starters
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Restore state from cache
|
||||
id: cache-restore
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: smoke-state.json
|
||||
key: smoke-monitor-state-impossible-match
|
||||
restore-keys: |
|
||||
smoke-monitor-state-
|
||||
|
||||
- name: Initialize state if missing
|
||||
run: |
|
||||
if [ ! -f smoke-state.json ]; then
|
||||
cat > smoke-state.json <<'INIT'
|
||||
{
|
||||
"langgraph-python": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"langgraph-typescript": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"langgraph-fastapi": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"mastra": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"crewai-crews": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"pydantic-ai": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"google-adk": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"agno": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"ag2": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"llamaindex": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"strands": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"ms-agent-python": { "status": "ok", "fail_count": 0, "first_failure_at": "" },
|
||||
"ms-agent-dotnet": { "status": "ok", "fail_count": 0, "first_failure_at": "" }
|
||||
}
|
||||
INIT
|
||||
fi
|
||||
|
||||
- name: Run smoke checks
|
||||
id: smoke
|
||||
run: |
|
||||
declare -A URLS
|
||||
URLS[langgraph-python]="https://showcase-langgraph-python-production.up.railway.app"
|
||||
URLS[langgraph-typescript]="https://showcase-langgraph-typescript-production.up.railway.app"
|
||||
URLS[langgraph-fastapi]="https://showcase-langgraph-fastapi-production.up.railway.app"
|
||||
URLS[mastra]="https://showcase-mastra-production.up.railway.app"
|
||||
URLS[crewai-crews]="https://showcase-crewai-crews-production.up.railway.app"
|
||||
URLS[pydantic-ai]="https://showcase-pydantic-ai-production.up.railway.app"
|
||||
URLS[google-adk]="https://showcase-google-adk-production.up.railway.app"
|
||||
URLS[agno]="https://showcase-agno-production.up.railway.app"
|
||||
URLS[ag2]="https://showcase-ag2-production.up.railway.app"
|
||||
URLS[llamaindex]="https://showcase-llamaindex-production.up.railway.app"
|
||||
URLS[strands]="https://showcase-strands-production.up.railway.app"
|
||||
URLS[ms-agent-python]="https://showcase-ms-agent-python-production.up.railway.app"
|
||||
URLS[ms-agent-dotnet]="https://showcase-ms-agent-dotnet-production.up.railway.app"
|
||||
|
||||
NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
ALERTS=""
|
||||
STATE="$(cat smoke-state.json)"
|
||||
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
|
||||
for SLUG in "${!URLS[@]}"; do
|
||||
URL="${URLS[$SLUG]}/api/smoke"
|
||||
PREV_STATUS=$(echo "$STATE" | jq -r --arg s "$SLUG" '.[$s].status // "ok"')
|
||||
PREV_FAIL_COUNT=$(echo "$STATE" | jq -r --arg s "$SLUG" '.[$s].fail_count // 0')
|
||||
PREV_FIRST_FAILURE=$(echo "$STATE" | jq -r --arg s "$SLUG" '.[$s].first_failure_at // ""')
|
||||
|
||||
# Hit the endpoint
|
||||
HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" --max-time 45 "$URL" 2>&1) || true
|
||||
HTTP_BODY=$(echo "$HTTP_RESPONSE" | head -n -1)
|
||||
HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -n 1)
|
||||
|
||||
# Determine if healthy
|
||||
HEALTHY=false
|
||||
if [[ "$HTTP_CODE" =~ ^2[0-9][0-9]$ ]]; then
|
||||
# Try to parse JSON and check status field
|
||||
SMOKE_STATUS=$(echo "$HTTP_BODY" | jq -r '.status // empty' 2>/dev/null) || true
|
||||
if [ "$SMOKE_STATUS" = "ok" ] || [ "$SMOKE_STATUS" = "healthy" ]; then
|
||||
HEALTHY=true
|
||||
elif [ -z "$SMOKE_STATUS" ]; then
|
||||
# No status field but 2xx — treat as healthy
|
||||
HEALTHY=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$HEALTHY" = true ]; then
|
||||
# Recovery case
|
||||
if [ "$PREV_STATUS" = "failing" ]; then
|
||||
ALERTS="${ALERTS}:white_check_mark: *${SLUG}* recovered (was down since ${PREV_FIRST_FAILURE})\n"
|
||||
fi
|
||||
STATE=$(echo "$STATE" | jq --arg s "$SLUG" '.[$s] = {"status":"ok","fail_count":0,"first_failure_at":""}')
|
||||
else
|
||||
# Failure case
|
||||
NEW_FAIL_COUNT=$((PREV_FAIL_COUNT + 1))
|
||||
FIRST_FAILURE="$PREV_FIRST_FAILURE"
|
||||
if [ -z "$FIRST_FAILURE" ] || [ "$FIRST_FAILURE" = "" ]; then
|
||||
FIRST_FAILURE="$NOW"
|
||||
fi
|
||||
|
||||
# Build error description
|
||||
if [[ "$HTTP_CODE" =~ ^[0-9]+$ ]] && [ "$HTTP_CODE" -gt 0 ] 2>/dev/null; then
|
||||
ERROR_DESC="HTTP ${HTTP_CODE}"
|
||||
else
|
||||
ERROR_DESC="connection failed"
|
||||
fi
|
||||
|
||||
SVC_URL="${URLS[$SLUG]}"
|
||||
ALERTS="${ALERTS}:red_circle: *${SLUG}* — attempt: ${NEW_FAIL_COUNT}, error: ${ERROR_DESC} (<${SVC_URL}/api/smoke|smoke> · <${SVC_URL}/api/health|health>)\n"
|
||||
|
||||
# Escalation at 4 consecutive failures (1 hour at 15-min intervals)
|
||||
if [ "$NEW_FAIL_COUNT" -eq 4 ]; then
|
||||
ALERTS="${ALERTS}<!channel> :rotating_light: *${SLUG}* has been failing for 1 hour (since ${FIRST_FAILURE})\n"
|
||||
fi
|
||||
|
||||
STATE=$(echo "$STATE" | jq \
|
||||
--arg s "$SLUG" \
|
||||
--argjson fc "$NEW_FAIL_COUNT" \
|
||||
--arg ff "$FIRST_FAILURE" \
|
||||
'.[$s] = {"status":"failing","fail_count":$fc,"first_failure_at":$ff}')
|
||||
fi
|
||||
done
|
||||
|
||||
# Write updated state
|
||||
echo "$STATE" | jq '.' > smoke-state.json
|
||||
|
||||
# Write alerts for the Slack step
|
||||
if [ -n "$ALERTS" ]; then
|
||||
{
|
||||
echo "has_alerts=true"
|
||||
echo "run_url=${RUN_URL}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Write alerts to a file to avoid escaping issues
|
||||
printf "%b" "$ALERTS" > alerts.txt
|
||||
echo "" >> alerts.txt
|
||||
echo "<https://showcase.copilotkit.ai|Showcase> · <${RUN_URL}|Workflow run>" >> alerts.txt
|
||||
else
|
||||
echo "has_alerts=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build Slack payload
|
||||
if: steps.smoke.outputs.has_alerts == 'true'
|
||||
run: |
|
||||
jq -n --rawfile text alerts.txt '{"text": $text}' > slack-payload.json
|
||||
|
||||
- name: Post to Slack
|
||||
if: steps.smoke.outputs.has_alerts == 'true'
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload-file-path: slack-payload.json
|
||||
|
||||
- name: Check image drift
|
||||
id: image_drift
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Skip if the latest main commit is <20 min old (deploy is probably still building)
|
||||
COMMIT_DATE=$(gh api "/repos/${{ github.repository }}/commits/main" --jq '.commit.committer.date' 2>/dev/null) || true
|
||||
if [ -n "$COMMIT_DATE" ]; then
|
||||
COMMIT_TS=$(date -d "$COMMIT_DATE" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$COMMIT_DATE" +%s 2>/dev/null) || true
|
||||
NOW_TS=$(date +%s)
|
||||
if [ -z "$COMMIT_TS" ]; then COMMIT_TS=$NOW_TS; fi
|
||||
AGE_MIN=$(( (NOW_TS - COMMIT_TS) / 60 ))
|
||||
echo "Latest main commit is ${AGE_MIN}m old"
|
||||
if [ "$AGE_MIN" -lt 20 ]; then
|
||||
echo "Skipping drift check — deploy likely still in progress"
|
||||
echo "has_stale=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
STALE=""
|
||||
STALE_LIST=""
|
||||
# Non-starter services stay literal — they don't live under
|
||||
# showcase/starters/ and each has bespoke provisioning elsewhere.
|
||||
SERVICES=(
|
||||
shell langgraph-python langgraph-typescript langgraph-fastapi
|
||||
mastra crewai-crews pydantic-ai google-adk ag2 agno llamaindex
|
||||
strands ms-agent-python ms-agent-dotnet claude-sdk-python
|
||||
claude-sdk-typescript langroid spring-ai aimock
|
||||
)
|
||||
# Starter slugs are derived from showcase/starters/*/ so adding a
|
||||
# new starter directory automatically extends drift detection.
|
||||
# `template/` is scaffolding, not a deployed service — keep in sync
|
||||
# with showcase/scripts/validate-workflow-starters.ts EXCLUDED_DIRS.
|
||||
# nullglob guards against an empty showcase/starters/ tree expanding
|
||||
# the literal "showcase/starters/*/" pattern into the iteration,
|
||||
# which would corrupt SERVICES with a "starter-*" entry.
|
||||
shopt -s nullglob
|
||||
for dir in showcase/starters/*/; do
|
||||
slug=$(basename "$dir")
|
||||
[ "$slug" = "template" ] && continue
|
||||
SERVICES+=("starter-$slug")
|
||||
done
|
||||
shopt -u nullglob
|
||||
# Count appended starter-* entries independently of the non-starter
|
||||
# list size. Using a magic `-eq 19` sentinel tied us to a specific
|
||||
# non-starter count; any add/remove there would have silently
|
||||
# disabled this guard. `grep -c '^starter-'` stays correct under
|
||||
# arbitrary churn to the literal non-starter list.
|
||||
STARTER_COUNT=$(printf '%s\n' "${SERVICES[@]}" | grep -c '^starter-' || true)
|
||||
if [ "$STARTER_COUNT" -eq 0 ]; then
|
||||
# Sparse checkout failed to populate showcase/starters/, or every
|
||||
# directory under it was `template/` — fail loudly instead of
|
||||
# silently under-checking.
|
||||
echo "::error::No starter directories found under showcase/starters/ — sparse checkout failed?"
|
||||
exit 1
|
||||
fi
|
||||
# Compare against the last commits that touched showcase-related paths,
|
||||
# NOT main HEAD. Deploys only trigger on showcase/ and examples/integrations/
|
||||
# changes, so non-showcase commits shouldn't make images appear stale.
|
||||
SHOWCASE_SHA=$(gh api "repos/${{ github.repository }}/commits?sha=main&path=showcase&per_page=1" --jq '.[0].sha // empty') || {
|
||||
echo "::warning::Failed to fetch showcase/ commit SHA"
|
||||
SHOWCASE_SHA=""
|
||||
}
|
||||
EXAMPLES_SHA=$(gh api "repos/${{ github.repository }}/commits?sha=main&path=examples/integrations&per_page=1" --jq '.[0].sha // empty') || {
|
||||
echo "::warning::Failed to fetch examples/integrations/ commit SHA"
|
||||
EXAMPLES_SHA=""
|
||||
}
|
||||
echo "Last showcase/ SHA: ${SHOWCASE_SHA:0:8}, Last examples/integrations/ SHA: ${EXAMPLES_SHA:0:8}"
|
||||
|
||||
if [ -z "$SHOWCASE_SHA" ] && [ -z "$EXAMPLES_SHA" ]; then
|
||||
echo "::warning::Could not resolve any path-specific SHAs — skipping drift check"
|
||||
echo "has_stale=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
|
||||
for SVC in "${SERVICES[@]}"; do
|
||||
PKG="showcase-${SVC}"
|
||||
# Query GHCR for the latest package version. The previous
|
||||
# implementation used `gh api ... || true` + `[ -z "$TAGS" ]`
|
||||
# to skip, which silently conflated three distinct cases:
|
||||
# - new service, not yet published (legitimate, expected)
|
||||
# - GHCR 404 (legitimate — same as above)
|
||||
# - GHCR 401/403/5xx (transient API error — NOT legitimate;
|
||||
# should surface so a broken drift run doesn't look clean)
|
||||
# Split them: capture HTTP status from gh api -i, treat 200+empty
|
||||
# and 404 as "no versions yet" (quiet continue), and warn on any
|
||||
# other non-200 so the run log shows a ::warning:: without
|
||||
# failing the whole drift detector over one flaky service.
|
||||
# Capture stderr separately from stdout. Merging them with 2>&1
|
||||
# can splice gh error lines (auth failures, rate limits, network
|
||||
# errors) ahead of the HTTP header block, which then corrupts
|
||||
# HTTP_STATUS and API_BODY parsing — a drift run can look "clean"
|
||||
# while actually failing every call. Keep stderr in a temp file
|
||||
# and surface it only when we need to diagnose a non-zero RC.
|
||||
API_STDERR=$(mktemp)
|
||||
set +e
|
||||
API_RESPONSE=$(gh api -i "/orgs/copilotkit/packages/container/${PKG}/versions?per_page=1" 2>"$API_STDERR")
|
||||
API_RC=$?
|
||||
set -e
|
||||
# gh api -i emits HTTP headers on stdout followed by a blank
|
||||
# line then the body. Extract the status line (first line
|
||||
# beginning with HTTP/) even if redirects prepended extras.
|
||||
HTTP_STATUS=$(printf '%s\n' "$API_RESPONSE" | awk '/^HTTP\// { status=$2 } END { print status }')
|
||||
# Body is everything after the first blank line.
|
||||
API_BODY=$(printf '%s\n' "$API_RESPONSE" | awk 'blank { print; next } /^\r?$/ { blank=1 }')
|
||||
|
||||
if [ "$API_RC" -ne 0 ] && [ -z "$HTTP_STATUS" ]; then
|
||||
# gh itself failed (network, missing binary) with no response
|
||||
# at all — surface as a warning and move on. A brief outage
|
||||
# shouldn't mark every service stale on the next clean run.
|
||||
GH_ERR=$(tr '\n' ' ' < "$API_STDERR" | sed 's/ */ /g' | sed 's/^ *//;s/ *$//')
|
||||
rm -f "$API_STDERR"
|
||||
echo "::warning::${SVC}: gh api call failed (rc=$API_RC): ${GH_ERR:-no stderr captured}, skipping drift check"
|
||||
continue
|
||||
fi
|
||||
rm -f "$API_STDERR"
|
||||
|
||||
case "$HTTP_STATUS" in
|
||||
200)
|
||||
TAGS=$(printf '%s' "$API_BODY" | jq -r '.[0].metadata.container.tags | join(" ")' 2>/dev/null) || TAGS=""
|
||||
if [ -z "$TAGS" ]; then
|
||||
echo " ${SVC}: no package versions yet, skipping"
|
||||
continue
|
||||
fi
|
||||
;;
|
||||
404)
|
||||
echo " ${SVC}: no GHCR package yet (404), skipping"
|
||||
continue
|
||||
;;
|
||||
*)
|
||||
# 401/403 (auth drift), 5xx (transient), any other surprise.
|
||||
# Warn visibly but don't fail the whole drift run — one
|
||||
# flaky service shouldn't break the monitor.
|
||||
echo "::warning::${SVC}: GHCR returned HTTP ${HTTP_STATUS:-<no status>}, skipping"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Image is up to date if it matches EITHER the last showcase/ or
|
||||
# examples/integrations/ commit (deploy triggers on both paths).
|
||||
# Use `grep -w` (word boundary) instead of `grep` (substring)
|
||||
# so one SHA isn't accidentally matched as a prefix of another.
|
||||
UP_TO_DATE=false
|
||||
if [ -n "$SHOWCASE_SHA" ] && echo "$TAGS" | grep -qw "$SHOWCASE_SHA"; then
|
||||
UP_TO_DATE=true
|
||||
fi
|
||||
if [ -n "$EXAMPLES_SHA" ] && echo "$TAGS" | grep -qw "$EXAMPLES_SHA"; then
|
||||
UP_TO_DATE=true
|
||||
fi
|
||||
if [ "$UP_TO_DATE" = true ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo " ${SVC}: stale (tags: ${TAGS:0:60})"
|
||||
STALE="${STALE}:warning: *${SVC}* — image stale\n"
|
||||
STALE_LIST="${STALE_LIST} ${SVC}"
|
||||
done
|
||||
|
||||
# Compare against previous drift state to avoid alerting repeatedly
|
||||
PREV_STALE=$(jq -r '.image_drift_services // ""' smoke-state.json 2>/dev/null) || true
|
||||
SORTED_STALE=$(echo "$STALE_LIST" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs)
|
||||
|
||||
if [ -n "$STALE_LIST" ]; then
|
||||
echo "Image drift detected:${STALE_LIST}"
|
||||
echo "stale_services=${STALE_LIST}" >> "$GITHUB_OUTPUT"
|
||||
printf "%b" "$STALE" > image-drift.txt
|
||||
|
||||
if [ "$SORTED_STALE" != "$PREV_STALE" ]; then
|
||||
echo "Stale set changed (was: '${PREV_STALE}', now: '${SORTED_STALE}') — alerting"
|
||||
echo "has_stale=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Same stale set as last run — suppressing alert"
|
||||
echo "has_stale=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Persist current stale set
|
||||
STATE=$(cat smoke-state.json)
|
||||
echo "$STATE" | jq --arg ds "$SORTED_STALE" '. + {"image_drift_services": $ds}' > smoke-state.json
|
||||
else
|
||||
echo "All images up to date"
|
||||
echo "has_stale=false" >> "$GITHUB_OUTPUT"
|
||||
# Clear drift state
|
||||
STATE=$(cat smoke-state.json)
|
||||
echo "$STATE" | jq 'del(.image_drift_services)' > smoke-state.json
|
||||
fi
|
||||
|
||||
fi # end SHA guard
|
||||
|
||||
- name: Trigger rebuild for stale services
|
||||
id: rebuild
|
||||
if: steps.image_drift.outputs.has_stale == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
FAILED=""
|
||||
FAILED_COUNT=0
|
||||
: > rebuild-failures.txt
|
||||
COUNT=0
|
||||
for SVC in ${{ steps.image_drift.outputs.stale_services }}; do
|
||||
echo "Triggering rebuild for ${SVC}..."
|
||||
RC=0
|
||||
ERR_OUTPUT=$(gh workflow run showcase_deploy.yml --repo "${{ github.repository }}" -f service="${SVC}" 2>&1) || RC=$?
|
||||
if [ "$RC" -eq 0 ]; then
|
||||
COUNT=$((COUNT + 1))
|
||||
else
|
||||
echo "::warning::Failed to trigger rebuild for ${SVC}: ${ERR_OUTPUT}"
|
||||
# Collapse whitespace/newlines in the error so it fits on one Slack line
|
||||
REASON=$(echo "$ERR_OUTPUT" | tr '\n' ' ' | sed 's/ */ /g' | sed 's/^ *//;s/ *$//')
|
||||
if [ -z "$REASON" ]; then
|
||||
REASON="unknown error"
|
||||
fi
|
||||
FAILED="${FAILED} ${SVC}"
|
||||
FAILED_COUNT=$((FAILED_COUNT + 1))
|
||||
printf ":x: *%s* — %s\n" "$SVC" "$REASON" >> rebuild-failures.txt
|
||||
fi
|
||||
done
|
||||
echo "triggered_count=${COUNT}" >> "$GITHUB_OUTPUT"
|
||||
echo "failed_count=${FAILED_COUNT}" >> "$GITHUB_OUTPUT"
|
||||
if [ "$FAILED_COUNT" -gt 0 ]; then
|
||||
# Surface failures via annotation and fail the job so the run shows
|
||||
# RED in GitHub Actions UI / `gh run list`. Slack dedup is handled
|
||||
# by guarding the failure() notifier with
|
||||
# `steps.image_drift.outputs.has_stale != 'true'`, so this exit 1
|
||||
# does NOT cause double-posting — the detailed drift-alert step
|
||||
# covers the drift path.
|
||||
echo "::error::Failed to trigger rebuilds for:${FAILED}"
|
||||
echo "has_failures=true" >> "$GITHUB_OUTPUT"
|
||||
exit 1
|
||||
else
|
||||
echo "has_failures=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Alert image drift to Slack
|
||||
if: always() && steps.image_drift.outputs.has_stale == 'true'
|
||||
run: |
|
||||
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
COUNT="${{ steps.rebuild.outputs.triggered_count }}"
|
||||
FAILED_COUNT="${{ steps.rebuild.outputs.failed_count }}"
|
||||
if [ -z "$COUNT" ]; then
|
||||
# Fallback: rebuild step didn't run or didn't set the output; count from stale_services
|
||||
COUNT=$(echo "${{ steps.image_drift.outputs.stale_services }}" | wc -w | tr -d ' ')
|
||||
fi
|
||||
if [ -z "$FAILED_COUNT" ]; then
|
||||
FAILED_COUNT=0
|
||||
fi
|
||||
NOUN="rebuilds"
|
||||
if [ "$COUNT" = "1" ]; then NOUN="rebuild"; fi
|
||||
|
||||
if [ "${{ steps.rebuild.outputs.has_failures }}" = "true" ] && [ -s rebuild-failures.txt ]; then
|
||||
# Failure case: list only the services that failed to rebuild, with reasons.
|
||||
# COUNT reflects only successfully-triggered rebuilds; FAILED_COUNT is the rest.
|
||||
{
|
||||
printf ":package: *Image drift detected — %s %s triggered, %s failed:*\n" "$COUNT" "$NOUN" "$FAILED_COUNT"
|
||||
cat rebuild-failures.txt
|
||||
printf "<%s|Workflow run>\n" "$RUN_URL"
|
||||
} > drift-message.txt
|
||||
else
|
||||
# Success case: just summarize the count with a link to the run
|
||||
printf ":package: Image drift detected — %s %s triggered (<%s|run>)\n" "$COUNT" "$NOUN" "$RUN_URL" > drift-message.txt
|
||||
fi
|
||||
jq -n --rawfile text drift-message.txt '{"text": $text}' > drift-payload.json
|
||||
|
||||
- name: Post image drift to Slack
|
||||
if: always() && steps.image_drift.outputs.has_stale == 'true'
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload-file-path: drift-payload.json
|
||||
|
||||
- name: Notify Slack (workflow failure)
|
||||
# Suppress when the drift-alert path is active — that step already
|
||||
# posts a detailed Slack message covering per-service rebuild results.
|
||||
# This generic notifier still fires for non-drift failures (e.g., the
|
||||
# smoke-check step itself fails before drift detection runs).
|
||||
if: failure() && steps.image_drift.outputs.has_stale != 'true'
|
||||
uses: slackapi/slack-github-action@v2.1.0
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
{ "text": ":x: *Smoke monitor*: workflow failed | <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" }
|
||||
|
||||
- name: Save state to cache
|
||||
if: always()
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: smoke-state.json
|
||||
key: smoke-monitor-state-${{ github.run_id }}
|
||||
@@ -20,15 +20,6 @@ jobs:
|
||||
env:
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
# Fail on any unset variable, failed command in a pipeline, or
|
||||
# unhandled non-zero exit. The previous script would let `wc -c`
|
||||
# failing on a transient race (file deleted mid-diff) produce
|
||||
# an empty `$SIZE`, which then triggered `[ "$SIZE" -gt ... ]`
|
||||
# with the error `integer expression expected` written to
|
||||
# stderr while the script marched on — false-green runs
|
||||
# possible. With `set -euo pipefail` + the explicit numeric
|
||||
# check below, that failure path now fails the job loudly.
|
||||
set -euo pipefail
|
||||
VIOLATIONS=0
|
||||
|
||||
# Get list of added/modified files in the PR
|
||||
@@ -39,11 +30,8 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check for binary file extensions. Keep in rough sync with
|
||||
# `.gitignore` — any compiled/archive/debug artefact authored
|
||||
# by a build tool belongs here. Missing extensions historically
|
||||
# let by: .class, .jar, .pyd, .pyc, .node, .bin, .pdb, .zip, .whl.
|
||||
BINARY_FILES=$(echo "$CHANGED_FILES" | grep -iE '\.(exe|dll|so|dylib|o|obj|a|lib|wasm|class|jar|pyd|pyc|node|bin|pdb|zip|whl)$' || true)
|
||||
# Check for binary file extensions
|
||||
BINARY_FILES=$(echo "$CHANGED_FILES" | grep -iE '\.(exe|dll|so|dylib|o|obj|a|lib|wasm)$' || true)
|
||||
if [ -n "$BINARY_FILES" ]; then
|
||||
echo "::error::Binary files detected in PR:"
|
||||
echo "$BINARY_FILES"
|
||||
@@ -72,23 +60,12 @@ jobs:
|
||||
while IFS= read -r file; do
|
||||
if [ -f "$file" ]; then
|
||||
case "$file" in
|
||||
pnpm-lock.yaml|*/package-lock.json|*/poetry.lock) continue ;;
|
||||
pnpm-lock.yaml|*/pnpm-lock.yaml|*/package-lock.json|*/poetry.lock) continue ;;
|
||||
assets/*|docs/public/*|examples/*/preview.gif|examples/*/assets/*) continue ;;
|
||||
.github/actions/*/dist/*) continue ;;
|
||||
showcase/shell/src/data/*|showcase/shell-docs/src/data/*|showcase/shell-dojo/src/data/demo-content.json) continue ;;
|
||||
esac
|
||||
# Guard `wc -c`: if the read fails or returns a non-numeric
|
||||
# value, skip this file rather than letting the subsequent
|
||||
# arithmetic comparison spew "integer expression expected"
|
||||
# to stderr while the script silently continues.
|
||||
if ! SIZE=$(wc -c < "$file" 2>/dev/null | tr -d ' '); then
|
||||
echo "::warning::Could not size $file; skipping"
|
||||
continue
|
||||
fi
|
||||
if ! [[ "$SIZE" =~ ^[0-9]+$ ]]; then
|
||||
echo "::warning::wc returned non-numeric size for $file ($SIZE); skipping"
|
||||
continue
|
||||
fi
|
||||
SIZE=$(wc -c < "$file" | tr -d ' ')
|
||||
if [ "$SIZE" -gt 1048576 ]; then
|
||||
LARGE_FILES="${LARGE_FILES}${file} ($(( SIZE / 1024 )) KB)\n"
|
||||
fi
|
||||
|
||||
@@ -6,14 +6,14 @@ on:
|
||||
paths:
|
||||
- "packages/**"
|
||||
- "sdk-python/**"
|
||||
- ".github/workflows/e2e_dojo.yml"
|
||||
- ".github/workflows/test_e2e-dojo.yml"
|
||||
- ".changeset"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "packages/**"
|
||||
- "sdk-python/**"
|
||||
- ".github/workflows/e2e_dojo.yml"
|
||||
- ".github/workflows/test_e2e-dojo.yml"
|
||||
- ".changeset"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
filters: |
|
||||
ts:
|
||||
- 'packages/**'
|
||||
- '.github/workflows/e2e_dojo.yml'
|
||||
- '.github/workflows/test_e2e-dojo.yml'
|
||||
- '.changeset'
|
||||
python:
|
||||
- 'sdk-python/**'
|
||||
@@ -1,17 +1,17 @@
|
||||
name: test / e2e / examples
|
||||
name: test / e2e / legacy-v1
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "examples/**"
|
||||
- ".github/workflows/e2e_examples.yml"
|
||||
- ".github/workflows/test_e2e-legacy-v1.yml"
|
||||
- ".changeset"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "examples/**"
|
||||
- ".github/workflows/e2e_examples.yml"
|
||||
- ".github/workflows/test_e2e-legacy-v1.yml"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: "Showcase: Aimock E2E Tests"
|
||||
name: "test / e2e / showcase / on-demand"
|
||||
|
||||
# SECURITY — residual trust model (read before editing):
|
||||
#
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: test / doc-examples
|
||||
name: test / integration / docs
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
+3
-3
@@ -1,16 +1,16 @@
|
||||
name: test / integration
|
||||
name: test / integration / runtime
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "packages/runtime/**"
|
||||
- ".github/workflows/test_runtime-servers.yml"
|
||||
- ".github/workflows/test_integration-runtime.yml"
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "packages/runtime/**"
|
||||
- ".github/workflows/test_runtime-servers.yml"
|
||||
- ".github/workflows/test_integration-runtime.yml"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
name: Starter Deployed Smoke Tests
|
||||
name: test / smoke / starter-deployed
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -198,7 +198,7 @@ jobs:
|
||||
- name: Log (no Slack — webhook unset)
|
||||
if: failure() && (github.event_name == 'schedule' || github.event_name == 'workflow_run') && env.SLACK_WEBHOOK == ''
|
||||
run: |
|
||||
echo "::warning::starter_deployed_smoke failed but SLACK_WEBHOOK_OSS_ALERTS is not set; no Slack notification sent."
|
||||
echo "::warning::test_smoke-starter-deployed failed but SLACK_WEBHOOK_OSS_ALERTS is not set; no Slack notification sent."
|
||||
|
||||
# Compute outcome + state transition for the next run. Runs on
|
||||
# success and failure alike (but not when the job is cancelled —
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Starter Smoke Tests
|
||||
name: test / smoke / starter
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "examples/integrations/**"
|
||||
- ".github/workflows/starter-smoke.yml"
|
||||
- ".github/workflows/test_smoke-starter.yml"
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
@@ -48,6 +48,12 @@ jobs:
|
||||
working-directory: examples/integrations/${{ matrix.starter }}
|
||||
env:
|
||||
STARTER: ${{ matrix.starter }}
|
||||
# Starters whose SDK can't be intercepted by aimock (currently
|
||||
# google-adk — google-genai ignores endpoint overrides) need a
|
||||
# real provider key. docker-compose.test.yml for those starters
|
||||
# reads this via `${GOOGLE_API_KEY:-test-key-for-aimock}` so
|
||||
# unaffected starters still run offline against aimock.
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
# ESLint + Prettier → oxlint + oxfmt Migration
|
||||
|
||||
**Status:** Executed 2026-03-05 — path references reflect pre-rename `interrupts-langraph` spelling; the rename to `interrupts-langgraph` happened after this plan ran.
|
||||
|
||||
> **Note on paths:** This plan was written before `examples/v2/interrupts-langraph`
|
||||
> was renamed to `interrupts-langgraph` (spelling fix). Path references below
|
||||
> intentionally preserve the pre-rename name as a historical record of what
|
||||
> was touched when this migration ran; they are NOT a source of truth for
|
||||
> the current tree layout.
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Replace ESLint and Prettier with oxlint and oxfmt for faster linting and formatting across the entire
|
||||
|
||||
@@ -119,7 +119,7 @@ If an example auto-opens Copilot UI / triggers calls, prefer adding a query para
|
||||
|
||||
Workflow:
|
||||
|
||||
- `.github/workflows/e2e_examples.yml`
|
||||
- `.github/workflows/test_e2e-legacy-v1.yml`
|
||||
|
||||
It runs a matrix of:
|
||||
|
||||
@@ -153,4 +153,4 @@ Artifacts:
|
||||
3. Run locally:
|
||||
- `EXAMPLE=<example> pnpm test`
|
||||
4. Add the example name to the CI matrix in:
|
||||
- `.github/workflows/e2e_examples.yml`
|
||||
- `.github/workflows/test_e2e-legacy-v1.yml`
|
||||
|
||||
@@ -11,7 +11,13 @@ services:
|
||||
context: ./agent
|
||||
dockerfile: ../docker/Dockerfile.agent
|
||||
environment:
|
||||
- GOOGLE_API_KEY=test-key-for-aimock
|
||||
# google-genai SDK does not honor a custom endpoint the way the
|
||||
# OpenAI / Anthropic clients do, so aimock can't intercept calls
|
||||
# from the adk agent — the SDK hits generativelanguage.googleapis.com
|
||||
# directly. Pass a real key via CI secret (see .github/workflows/
|
||||
# test_smoke-starter.yml). Fall back to the placeholder for local
|
||||
# dev runs where the local harness may provide its own stub.
|
||||
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-test-key-for-aimock}
|
||||
- GOOGLE_GENAI_USE_VERTEXAI=false
|
||||
depends_on:
|
||||
aimock:
|
||||
|
||||
+4
@@ -20,6 +20,9 @@ node_modules
|
||||
# turbo
|
||||
.turbo
|
||||
|
||||
# Build outputs
|
||||
dist/
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
@@ -35,6 +38,7 @@ yarn-error.log*
|
||||
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
!.env.example
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
@@ -0,0 +1,135 @@
|
||||
# langgraph-js-starter
|
||||
|
||||
## 0.1.7 — 2026-04-21
|
||||
|
||||
### Fixed
|
||||
|
||||
- Hardened `emit_unknown_tools_notice` and `intercept_frontend_tools`
|
||||
against OpenAI tool_call invariant violations and prior-stash
|
||||
clobbering.
|
||||
- `useCopilotAction` deps on `setThemeColor` and `getWeather`.
|
||||
- `next lint` removal in Next 16 — replaced with direct ESLint invocation.
|
||||
- `route.ts` LANGSMITH warn now gates on NODE_ENV like DEPLOYMENT_URL.
|
||||
- `route.ts` log prefixes distinguish runtime-construction from dispatch
|
||||
failures.
|
||||
- `parseInterruptPayload` single-return-value, caller-owned log.
|
||||
|
||||
### Changed
|
||||
|
||||
- LICENSE copyright attribution: `2025-2026 CopilotKit` (was: individual).
|
||||
- README: fixed broken `pnpm --filter` example in troubleshooting;
|
||||
replaced inline `echo > .env` with `cp .env.example .env` + edit;
|
||||
corrected project-structure diagram comment to reference
|
||||
`pnpm-workspace.yaml`.
|
||||
- `apps/web/tsconfig.json`: dropped dead `.next/dev/types/**` include.
|
||||
- `apps/web/.env.example`: clarified that LANGSMITH\_\* vars forward to the
|
||||
agent.
|
||||
- Root `.gitignore`: ignore `dist/`.
|
||||
- `turbo.json`: add `start` task.
|
||||
- `apps/web/project.json`: `cache: false` on `build` target for Nx parity
|
||||
with package.json.
|
||||
- `apps/agent/tsconfig.json`: set `noEmit: true` so direct `tsc`
|
||||
invocation matches script-driven builds.
|
||||
|
||||
## 0.1.6 — 2026-04-21
|
||||
|
||||
### Changed
|
||||
|
||||
- Added `turbo.json` at the starter root so `turbo run dev/build/lint`
|
||||
works when the starter is extracted standalone (root scripts delegate
|
||||
to turbo; missing config previously broke extraction).
|
||||
- Added `pnpm-workspace.yaml` so pnpm v9+ reliably links
|
||||
`workspace:*` deps on standalone extraction; the `workspaces` array in
|
||||
the root `package.json` is not honored by pnpm on its own.
|
||||
- Bumped the agent `tsconfig.json` `target` from `es2016` to `ES2022`
|
||||
(with explicit `lib: ["ES2022"]`, no DOM) to match the declared
|
||||
Node 20+ runtime baseline.
|
||||
- Added `build` and `lint` scripts to `apps/agent/package.json`
|
||||
(`tsc -p tsconfig.json --noEmit` — identical to what `project.json`
|
||||
declares) so `turbo run build/lint` and nx targets agree.
|
||||
- Filled in the empty agent `description` and `author` fields.
|
||||
- Added `LANGSMITH_TRACING=true` (commented) to `apps/web/.env.example`
|
||||
to match the agent's `.env.example`.
|
||||
- Added `"baseUrl": "."` to `apps/web/tsconfig.json` so the `@/*` path
|
||||
alias resolves reliably across IDEs and tooling.
|
||||
- Removed Python-specific entries (`venv/`, `__pycache__/`, `*.pyc`)
|
||||
from `apps/agent/.gitignore` — leftovers from a forked Python
|
||||
LangGraph starter; this is a TypeScript project.
|
||||
- Added a copyright year to `LICENSE` (MIT convention requires one).
|
||||
|
||||
### Versioning
|
||||
|
||||
- Root, `apps/agent`, and `apps/web` all bumped from `0.1.5` to `0.1.6`
|
||||
in lockstep per the starter's shared-version convention.
|
||||
|
||||
## 0.1.5 — 2026-04-17
|
||||
|
||||
### Renamed
|
||||
|
||||
- Renamed directory from `interrupts-langraph` to `interrupts-langgraph`
|
||||
(spelling fix). The rename itself is code-neutral; hardening described
|
||||
below.
|
||||
|
||||
### Changed
|
||||
|
||||
- Tightened types in the agent graph (no `as any`, structural message
|
||||
narrowing) and added an explicit `bindTools` capability guard.
|
||||
- `shouldContinue` now evaluates ALL tool calls on an AIMessage and
|
||||
routes to `tool_node` whenever ANY call targets a registered backend
|
||||
tool. Previously a mixed batch (frontend action + backend tool) could
|
||||
silently drop the backend call. Each unknown tool-call name emits its
|
||||
own `console.warn`; routing then keeps the batch on `tool_node` when
|
||||
any known backend tool is present (ToolNode will emit an error
|
||||
ToolMessage for unknown tool names; the graph then loops back to
|
||||
`chat_node` with that error in context) and falls through to `END`
|
||||
otherwise.
|
||||
- Validated the interrupt payload on the web side via
|
||||
`parseInterruptPayload`; malformed payloads now render a cancellation
|
||||
fallback instead of crashing the renderer. Arrays are explicitly
|
||||
rejected (previously passed the `typeof === "object"` check and were
|
||||
coerced to the `Record` lookup path).
|
||||
- Validated the resumed interrupt value on the agent side via a zod
|
||||
schema (`ApprovalResumeSchema`); an out-of-band Client resuming with
|
||||
the wrong shape now fails loudly at the tool boundary instead of
|
||||
silently branching to "cancelled".
|
||||
- `deleteProverb` on the web side now uses a functional `setState`
|
||||
updater so concurrent state writes don't race.
|
||||
- React key for the proverb list is `${index}-${proverb}` (index + content
|
||||
composite). Chosen because plain `proverb` collides on duplicates and
|
||||
plain `index` destabilizes rows during agent-driven inserts. Migrating
|
||||
the underlying state to `{id, text}` objects is the proper long-term
|
||||
fix and is deferred.
|
||||
- Removed the unused `starterAgent` alias; extracted the model name
|
||||
to a `MODEL` constant for single-point swaps.
|
||||
- Wrapped `handleRequest` with a structured error response in the web
|
||||
runtime route (`apps/web/src/app/api/copilotkit/route.ts`), so
|
||||
unhandled exceptions surface as a structured 500 JSON response rather
|
||||
than the raw Next.js error page.
|
||||
- Corrected port references in the README (8125 everywhere).
|
||||
- Replaced the placeholder Next metadata with a real title/description.
|
||||
|
||||
### Dependencies
|
||||
|
||||
- `@types/node` ^20 → ^22.19.11
|
||||
- `typescript` ^5 → ^5.9.3
|
||||
- `zod` ^3.24.4 → ^3.25.76
|
||||
- Agent `@langchain/langgraph` → `1.1.5` (previously pinned via a root
|
||||
`overrides` entry at `1.0.2`; the override has been removed and the
|
||||
version is now declared directly on `apps/agent/package.json`).
|
||||
- Agent `@langchain/core` → `^1.1.26`.
|
||||
- Dropped the dead `@langchain/core` override from the starter root; it
|
||||
had no effect inside the monorepo pnpm workspace and would cap the
|
||||
agent's `^1.1.26` requirement if the starter were extracted.
|
||||
- Removed root `overrides` entry for `@langchain/langgraph` (was `1.0.2`);
|
||||
the agent owns its own version now.
|
||||
|
||||
### Versioning
|
||||
|
||||
- Sub-app versions synced to the root: `apps/agent` `0.0.1` → `0.1.5` and
|
||||
`apps/web` `0.1.0` → `0.1.5`. Convention: sub-app versions track the
|
||||
root starter version so changelog entries and `package.json` reads
|
||||
stay consistent across the workspace.
|
||||
|
||||
## 0.1.1 – 0.1.4
|
||||
|
||||
Internal only (dependency sync / tooling bumps).
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License
|
||||
|
||||
Copyright (c) Atai Barkai
|
||||
Copyright (c) 2025-2026 CopilotKit
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
+18
-17
@@ -2,7 +2,7 @@
|
||||
|
||||
This is a starter template for building AI agents using [LangGraph](https://www.langchain.com/langgraph) and [CopilotKit](https://copilotkit.ai). It provides a modern Next.js application with an integrated LangGraph agent to be built on top of.
|
||||
|
||||
This project is organized as a monorepo using [Turborepo](https://turbo.build) and [pnpm workspaces](https://pnpm.io/workspaces).
|
||||
This project is organized as a monorepo using [pnpm workspaces](https://pnpm.io/workspaces).
|
||||
|
||||
## Project Structure
|
||||
|
||||
@@ -11,14 +11,12 @@ This project is organized as a monorepo using [Turborepo](https://turbo.build) a
|
||||
├── apps/
|
||||
│ ├── web/ # Next.js frontend application
|
||||
│ └── agent/ # LangGraph agent
|
||||
├── pnpm-workspace.yaml
|
||||
├── turbo.json
|
||||
└── package.json
|
||||
└── package.json # pnpm workspaces via pnpm-workspace.yaml
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js 18+
|
||||
- Node.js 20+
|
||||
- [pnpm](https://pnpm.io/installation) 9.15.0 or later
|
||||
- OpenAI API Key (for the LangGraph agent)
|
||||
|
||||
@@ -30,27 +28,29 @@ This project is organized as a monorepo using [Turborepo](https://turbo.build) a
|
||||
pnpm install
|
||||
```
|
||||
|
||||
2. Set up your OpenAI API key:
|
||||
2. Set up your OpenAI API key by copying the example env file and editing it:
|
||||
|
||||
```bash
|
||||
cd apps/agent
|
||||
echo "OPENAI_API_KEY=your-openai-api-key-here" > .env
|
||||
cp apps/agent/.env.example apps/agent/.env
|
||||
```
|
||||
|
||||
Then open `apps/agent/.env` in your editor and fill in `OPENAI_API_KEY` with your key.
|
||||
|
||||
For production, also copy `apps/web/.env.example` to `apps/web/.env` and set `LANGGRAPH_DEPLOYMENT_URL` to your deployed agent URL.
|
||||
|
||||
3. Start the development servers:
|
||||
|
||||
```bash
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
This will start both the Next.js app (on port 3000) and the LangGraph agent (on port 8123) using Turborepo.
|
||||
This will start both the Next.js app (on port 3000) and the LangGraph agent (on port 8125) via Turbo (installed as a dev dependency).
|
||||
|
||||
## Available Scripts
|
||||
|
||||
All scripts use Turborepo to run tasks across the monorepo:
|
||||
All scripts use Turbo to run tasks across the workspace:
|
||||
|
||||
- `pnpm dev` - Starts both the web app and agent servers in development mode
|
||||
- `pnpm dev:studio` - Starts the web app and agent with LangGraph Studio UI
|
||||
- `pnpm build` - Builds all apps for production
|
||||
- `pnpm lint` - Runs linting across all apps
|
||||
|
||||
@@ -60,10 +60,10 @@ You can also run scripts for individual apps using pnpm's filter flag:
|
||||
|
||||
```bash
|
||||
# Run dev for just the web app
|
||||
pnpm --filter web dev
|
||||
pnpm --filter web-langgraph-interrupt dev
|
||||
|
||||
# Run dev for just the agent
|
||||
pnpm --filter agent dev
|
||||
pnpm --filter agent-langgraph-interrupt dev
|
||||
|
||||
# Or navigate to the app directory
|
||||
cd apps/web
|
||||
@@ -99,8 +99,9 @@ This project is licensed under the MIT License - see the LICENSE file for detail
|
||||
|
||||
### Agent Connection Issues
|
||||
|
||||
If you see "I'm having trouble connecting to my tools", make sure:
|
||||
**If the chat returns an "Internal error while dispatching CopilotKit request" 500:**
|
||||
Check the Next.js server logs for `[copilotkit/route] runtime construction failed:` or `[copilotkit/route] handleRequest dispatch failed:`. Common causes:
|
||||
|
||||
1. The LangGraph agent is running on port 8000
|
||||
2. Your OpenAI API key is set correctly
|
||||
3. Both servers started successfully
|
||||
- `LANGGRAPH_DEPLOYMENT_URL` unset in production (required — check `apps/web/.env`)
|
||||
- LangGraph server not running at the configured URL (check `pnpm --filter agent-langgraph-interrupt dev` started cleanly)
|
||||
- `OPENAI_API_KEY` missing from `apps/agent/.env`
|
||||
@@ -0,0 +1,7 @@
|
||||
# OpenAI API key (required)
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# LangSmith tracing (optional)
|
||||
# LANGSMITH_API_KEY=
|
||||
# LANGSMITH_TRACING=true
|
||||
# LANGSMITH_PROJECT=interrupts-langgraph
|
||||
+1
-4
@@ -1,9 +1,6 @@
|
||||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
.vercel
|
||||
|
||||
# LangGraph API
|
||||
.langgraph_api
|
||||
node_modules/
|
||||
node_modules/
|
||||
+12
-9
@@ -1,25 +1,28 @@
|
||||
{
|
||||
"name": "agent-langraph-interrupt",
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"name": "agent-langgraph-interrupt",
|
||||
"version": "0.1.7",
|
||||
"private": true,
|
||||
"description": "LangGraph agent for the CopilotKit interrupts starter",
|
||||
"keywords": [],
|
||||
"license": "ISC",
|
||||
"author": "",
|
||||
"main": "index.js",
|
||||
"license": "MIT",
|
||||
"author": "CopilotKit",
|
||||
"scripts": {
|
||||
"dev": "npx @langchain/langgraph-cli dev --port 8125 --no-browser"
|
||||
"dev": "npx @langchain/langgraph-cli dev --port 8125 --no-browser",
|
||||
"build": "tsc -p tsconfig.json --noEmit",
|
||||
"lint": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@copilotkit/sdk-js": "workspace:*",
|
||||
"@langchain/core": "^1.1.26",
|
||||
"@langchain/langgraph": "1.1.5",
|
||||
"@langchain/langgraph-checkpoint": "1.0.0",
|
||||
"@langchain/openai": "^1.2.8",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/html-to-text": "^9.0.4",
|
||||
"@types/node": "^22.19.11",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
}
|
||||
+6
-4
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "agent-langraph-interrupt",
|
||||
"name": "agent-langgraph-interrupt",
|
||||
"$schema": "../../node_modules/nx/schemas/project-schema.json",
|
||||
"sourceRoot": "apps/agent/src",
|
||||
"projectType": "application",
|
||||
@@ -7,19 +7,21 @@
|
||||
"dev": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "pnpm --filter agent-langraph-interrupt dev"
|
||||
"command": "pnpm --filter agent-langgraph-interrupt dev"
|
||||
}
|
||||
},
|
||||
"build": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "node -e \"console.log('Agent app has no separate build target yet.')\""
|
||||
"command": "tsc -p tsconfig.json --noEmit",
|
||||
"cwd": "examples/v2/interrupts-langgraph/apps/agent"
|
||||
}
|
||||
},
|
||||
"lint": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "node -e \"console.log('Agent app has no lint target yet.')\""
|
||||
"command": "tsc -p tsconfig.json --noEmit",
|
||||
"cwd": "examples/v2/interrupts-langgraph/apps/agent"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -0,0 +1,924 @@
|
||||
/**
|
||||
* This is the main entry point for the agent.
|
||||
* It defines the workflow graph, state, tools, nodes and edges.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { z } from "zod";
|
||||
import type { RunnableConfig } from "@langchain/core/runnables";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { ToolRunnableConfig } from "@langchain/core/tools";
|
||||
import { ToolNode } from "@langchain/langgraph/prebuilt";
|
||||
import type { BaseMessage, ToolCall } from "@langchain/core/messages";
|
||||
import {
|
||||
AIMessage,
|
||||
isAIMessage,
|
||||
SystemMessage,
|
||||
ToolMessage,
|
||||
} from "@langchain/core/messages";
|
||||
|
||||
import {
|
||||
Annotation,
|
||||
Command,
|
||||
END,
|
||||
getCurrentTaskInput,
|
||||
interrupt,
|
||||
MemorySaver,
|
||||
START,
|
||||
StateGraph,
|
||||
} from "@langchain/langgraph";
|
||||
import { ChatOpenAI } from "@langchain/openai";
|
||||
import {
|
||||
convertActionsToDynamicStructuredTools,
|
||||
CopilotKitStateAnnotation,
|
||||
} from "@copilotkit/sdk-js/langgraph";
|
||||
|
||||
// Include CopilotKitStateAnnotation so the frontend can attach actions and
|
||||
// so messages flow through the same channel the SDK expects.
|
||||
//
|
||||
// `interceptedToolCalls` + `originalAIMessageId` mirror
|
||||
// `@copilotkit/sdk-js/langgraph`'s `copilotkitMiddleware.afterModel`
|
||||
// intercept pattern (see node_modules/@copilotkit/sdk-js/src/langgraph/middleware.ts):
|
||||
// on a mixed batch (backend tool call + frontend-action call in the same
|
||||
// AIMessage), we strip the frontend calls out of the AIMessage before
|
||||
// ToolNode runs (otherwise ToolNode errors on "Tool not found" for the
|
||||
// frontend action names), stash them here, then restore them onto the
|
||||
// original AIMessage before the graph ends so the frontend runtime still
|
||||
// dispatches them. Raw-StateGraph starters like this one don't use
|
||||
// createAgent+middleware, so we reproduce the pattern inline.
|
||||
const AgentStateAnnotation = Annotation.Root({
|
||||
...CopilotKitStateAnnotation.spec,
|
||||
proverbs: Annotation<string[]>,
|
||||
interceptedToolCalls: Annotation<ToolCall[] | undefined>,
|
||||
originalAIMessageId: Annotation<string | undefined>,
|
||||
});
|
||||
|
||||
export type AgentState = typeof AgentStateAnnotation.State;
|
||||
|
||||
// The renderer in apps/web/src/app/page.tsx validates the *emitted* interrupt
|
||||
// payload against its own `parseInterruptPayload` shape. We validate the
|
||||
// *resumed* payload here so an out-of-band Client that resumes with the wrong
|
||||
// shape fails loudly at the tool boundary instead of silently branching to
|
||||
// "cancelled".
|
||||
const ApprovalResumeSchema = z.object({
|
||||
approved: z.boolean(),
|
||||
});
|
||||
|
||||
const getWeather = tool(
|
||||
(args) => {
|
||||
return `The weather for ${args.location} is 70 degrees, clear skies, 45% humidity, 5 mph wind, and feels like 72 degrees.`;
|
||||
},
|
||||
{
|
||||
name: "getWeather",
|
||||
description: "Get the weather for a given location.",
|
||||
schema: z.object({
|
||||
location: z.string().describe("The location to get weather for"),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
// HITL tool: triggers an interrupt that the frontend resolves with
|
||||
// `{ approved: boolean }`. Validated with zod so a malformed resume value
|
||||
// surfaces as a deterministic tool message rather than throwing through
|
||||
// ToolNode.
|
||||
//
|
||||
// On approval, returns a `Command` that BOTH emits a ToolMessage (so the
|
||||
// model sees the tool result) AND applies a state update that removes the
|
||||
// matching proverb from `state.proverbs`. Without the state update, the
|
||||
// UI (which reads `state.proverbs` via CopilotKit) would still show the
|
||||
// "deleted" proverb, making the HITL demo a sham.
|
||||
const deleteProverb = tool(
|
||||
async (args, config: ToolRunnableConfig) => {
|
||||
// `config.toolCall.id` is the canonical id accessor when a tool is
|
||||
// invoked by ToolNode. ToolNode calls `tool.invoke({...call, type:
|
||||
// "tool_call"}, config)` (see
|
||||
// node_modules/@langchain/langgraph/dist/prebuilt/tool_node.js runTool),
|
||||
// and @langchain/core's StructuredTool.invoke then copies the call
|
||||
// onto `enrichedConfig.toolCall` (see
|
||||
// node_modules/@langchain/core/dist/tools/index.js lines 84-91) before
|
||||
// forwarding to the tool function. This is typed on `ToolRunnableConfig`
|
||||
// — typing `config` explicitly above is what gives us the safe accessor.
|
||||
//
|
||||
// We need the id here because returning a `Command` bypasses
|
||||
// `_formatToolOutput`'s automatic tool_call_id wiring. If the id is
|
||||
// somehow missing we throw loudly rather than silently emitting
|
||||
// `tool_call_id: ""`, which OpenAI rejects on the next turn with
|
||||
// "tool_call_id does not match any preceding tool_calls".
|
||||
const toolCallId = config.toolCall?.id;
|
||||
if (typeof toolCallId !== "string" || toolCallId.length === 0) {
|
||||
throw new Error(
|
||||
"deleteProverb: missing tool_call_id on ToolRunnableConfig.toolCall — " +
|
||||
"tool was invoked outside a ToolNode context. Refusing to emit a " +
|
||||
"ToolMessage with an empty tool_call_id (OpenAI rejects those).",
|
||||
);
|
||||
}
|
||||
|
||||
const rawApproval = interrupt({
|
||||
action: "delete_proverb",
|
||||
proverb: args.proverb,
|
||||
message: `Are you sure you want to delete the proverb: "${args.proverb}"?`,
|
||||
});
|
||||
|
||||
let approval: z.infer<typeof ApprovalResumeSchema>;
|
||||
try {
|
||||
approval = ApprovalResumeSchema.parse(rawApproval);
|
||||
} catch (err) {
|
||||
// Only swallow ZodError — any other throw (programming errors, runtime
|
||||
// failures, etc.) must propagate so we don't mask real bugs behind a
|
||||
// generic tool message.
|
||||
if (!(err instanceof z.ZodError)) {
|
||||
throw err;
|
||||
}
|
||||
// eslint-disable-next-line no-console
|
||||
console.error("[deleteProverb] resume payload rejected:", err.issues);
|
||||
// Don't let ZodError propagate through ToolNode. Return a
|
||||
// deterministic tool message so the graph can loop back to chat_node
|
||||
// with a readable result in context.
|
||||
return new ToolMessage({
|
||||
status: "error",
|
||||
name: "deleteProverb",
|
||||
tool_call_id: toolCallId,
|
||||
content:
|
||||
"Confirmation failed due to an unexpected resume payload shape; deletion was NOT performed.",
|
||||
});
|
||||
}
|
||||
|
||||
if (approval.approved) {
|
||||
// Read the current graph state via LangGraph's task-local accessor so
|
||||
// we can filter `proverbs` deterministically. We match by content
|
||||
// (the schema accepts the proverb text). If multiple proverbs tie
|
||||
// exactly, only the first matching entry is removed — consistent
|
||||
// with "delete the proverb the user named".
|
||||
//
|
||||
// AgentStateAnnotation's `proverbs` channel has no reducer, so
|
||||
// Annotation<string[]> defaults to last-write-wins: emitting a
|
||||
// filtered array replaces the channel wholesale (which is exactly
|
||||
// what chat_node reads on the next turn for the system prompt).
|
||||
const currentState = getCurrentTaskInput<AgentState>();
|
||||
const current = Array.isArray(currentState?.proverbs)
|
||||
? currentState.proverbs
|
||||
: [];
|
||||
const idx = current.indexOf(args.proverb);
|
||||
|
||||
// Approved-but-not-present: do not lie to the model. Return an
|
||||
// error ToolMessage so the model sees "nothing matched" and can
|
||||
// respond truthfully instead of confirming a deletion that never
|
||||
// happened.
|
||||
if (idx === -1) {
|
||||
return new Command({
|
||||
update: {
|
||||
messages: [
|
||||
new ToolMessage({
|
||||
status: "error",
|
||||
name: "deleteProverb",
|
||||
tool_call_id: toolCallId,
|
||||
content: `No proverb matching "${args.proverb}" was found; nothing was deleted.`,
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const filtered = [...current.slice(0, idx), ...current.slice(idx + 1)];
|
||||
|
||||
return new Command({
|
||||
update: {
|
||||
messages: [
|
||||
new ToolMessage({
|
||||
status: "success",
|
||||
name: "deleteProverb",
|
||||
tool_call_id: toolCallId,
|
||||
content: `Proverb "${args.proverb}" has been deleted.`,
|
||||
}),
|
||||
],
|
||||
proverbs: filtered,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Mirror the approved branch: return a Command wrapping a ToolMessage
|
||||
// so the model sees a well-formed tool result with the correct
|
||||
// tool_call_id (OpenAI rejects tool messages with mismatched ids).
|
||||
// Cancellation is not success — the tool did not complete its stated
|
||||
// intent — so the ToolMessage status is "error". The content string is
|
||||
// truthful as a user-cancelled message.
|
||||
return new Command({
|
||||
update: {
|
||||
messages: [
|
||||
new ToolMessage({
|
||||
status: "error",
|
||||
name: "deleteProverb",
|
||||
tool_call_id: toolCallId,
|
||||
content: `Deletion of proverb "${args.proverb}" was cancelled by the user.`,
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
},
|
||||
{
|
||||
name: "deleteProverb",
|
||||
description:
|
||||
"Delete a proverb from the list. This will ask the user for confirmation before deleting.",
|
||||
schema: z.object({
|
||||
proverb: z.string().describe("The proverb to delete"),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
const tools = [getWeather, deleteProverb];
|
||||
|
||||
// gpt-4o-mini: reliable tool-calling, low cost. Swap model here if you
|
||||
// need different tradeoffs.
|
||||
const MODEL = "gpt-4o-mini";
|
||||
|
||||
async function chat_node(state: AgentState, config: RunnableConfig) {
|
||||
const model = new ChatOpenAI({ model: MODEL });
|
||||
|
||||
// Bind tools to the model, including CopilotKit frontend actions.
|
||||
//
|
||||
// bindTools is optional on BaseChatModel's type; guard explicitly instead
|
||||
// of using the non-null `!` escape hatch so a model instance that doesn't
|
||||
// support tool binding fails loudly.
|
||||
if (typeof model.bindTools !== "function") {
|
||||
throw new Error(
|
||||
`ChatOpenAI instance for model "${MODEL}" does not expose bindTools; cannot bind tools for this model.`,
|
||||
);
|
||||
}
|
||||
const modelWithTools = model.bindTools([
|
||||
...convertActionsToDynamicStructuredTools(state.copilotkit?.actions ?? []),
|
||||
...tools,
|
||||
]);
|
||||
|
||||
const systemMessage = new SystemMessage({
|
||||
content: `You are a helpful assistant. The current proverbs are ${JSON.stringify(state.proverbs ?? [])}. If a user asks to delete a proverb, call deleteProverb to trigger a human-in-the-loop interrupt for confirmation.`,
|
||||
});
|
||||
|
||||
const response = await modelWithTools.invoke(
|
||||
[systemMessage, ...(state.messages ?? [])],
|
||||
config,
|
||||
);
|
||||
|
||||
return {
|
||||
messages: [response],
|
||||
};
|
||||
}
|
||||
|
||||
// intercept_frontend_tools: strips frontend-action tool_calls out of the
|
||||
// last AIMessage before ToolNode runs and stashes them in state.
|
||||
//
|
||||
// ToolNode only knows about backend `tools` (getWeather, deleteProverb); it
|
||||
// looks up each `tool_call.name` in its own registry and throws
|
||||
// "Tool not found" for any frontend-action name (see
|
||||
// node_modules/@langchain/langgraph/dist/prebuilt/tool_node.js, runTool).
|
||||
// On a mixed batch that would leave backend results AND a model-visible
|
||||
// error ToolMessage for the frontend action, and the frontend would never
|
||||
// see the frontend-action call at all.
|
||||
//
|
||||
// The intercept+restore pattern mirrors CopilotKit's own
|
||||
// `copilotkitMiddleware.afterModel` + `afterAgent` (see
|
||||
// node_modules/@copilotkit/sdk-js/src/langgraph/middleware.ts). The
|
||||
// `restore_frontend_tools` node below reattaches the stashed calls to the
|
||||
// original AIMessage (matched by id) before the graph ends so the
|
||||
// CopilotKit runtime still dispatches them to the frontend.
|
||||
//
|
||||
// Pure-frontend-only batches skip this node entirely — shouldContinue
|
||||
// routes them straight to END, and the AIMessage with their tool_calls
|
||||
// reaches the frontend as-is.
|
||||
// Rebuild an AIMessage with a different tool_calls set while preserving
|
||||
// every other field (additional_kwargs, response_metadata, usage_metadata,
|
||||
// name, invalid_tool_calls, id, content). Required for LangSmith tracing
|
||||
// + token accounting — naively constructing `new AIMessage({ content,
|
||||
// tool_calls, id })` drops everything else.
|
||||
//
|
||||
// Note: `tool_call_chunks` only exists on AIMessageChunk, and the AIMessage
|
||||
// constructor does not accept it — preserving it here was a no-op. Omitted.
|
||||
function rebuildAIMessageWithToolCalls(
|
||||
source: AIMessage,
|
||||
toolCalls: ToolCall[],
|
||||
): AIMessage {
|
||||
return new AIMessage({
|
||||
content: source.content,
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
additional_kwargs: source.additional_kwargs,
|
||||
response_metadata: source.response_metadata,
|
||||
usage_metadata: source.usage_metadata,
|
||||
invalid_tool_calls: source.invalid_tool_calls,
|
||||
tool_calls: toolCalls,
|
||||
});
|
||||
}
|
||||
|
||||
function intercept_frontend_tools(state: AgentState) {
|
||||
const frontendActionNames = new Set(
|
||||
(state.copilotkit?.actions ?? []).map((a: { name: string }) => a.name),
|
||||
);
|
||||
if (frontendActionNames.size === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Widen to our directly-imported `BaseMessage` type so `isAIMessage`
|
||||
// (1.1.27) can narrow a `state.messages[i]` (structurally identical
|
||||
// 1.1.40) without the pnpm nominal-mismatch error. Runtime values are
|
||||
// unaffected — see the matching annotation on `lastMessage` in
|
||||
// `shouldContinue`.
|
||||
let messages = (state.messages ?? []) as unknown as BaseMessage[];
|
||||
|
||||
// The per-turn intercept slot is a single pair (interceptedToolCalls +
|
||||
// originalAIMessageId) with no reducer on the annotation — it cannot
|
||||
// queue. If the graph re-enters this node for a second mixed batch in
|
||||
// the same thread before `restore_frontend_tools` has flushed the prior
|
||||
// stash, we must flush the previous stash onto its matching AIMessage
|
||||
// inline here. Otherwise last-write-wins would silently drop the
|
||||
// earlier frontend-action calls and the frontend would never see them.
|
||||
//
|
||||
// Flush strategy on re-entry:
|
||||
// (a) First pass: walk `messages` and reattach the prior stash onto
|
||||
// the AIMessage whose id matches `priorOriginalId` (pre-strip).
|
||||
// (b) If (a) finds no match AND this pass would otherwise overwrite
|
||||
// the slot with a new stash (the mixed-batch return below), we
|
||||
// make a second flush attempt against the newly-rewritten
|
||||
// `messages` array (post-strip) before committing the new stash.
|
||||
// (c) If both attempts fail, we emit a loud warn naming the lost
|
||||
// AIMessage id + tool-call ids and STILL write the new stash —
|
||||
// merging two different AIMessage ids into one slot would corrupt
|
||||
// `originalAIMessageId`. The warn is the escape valve.
|
||||
//
|
||||
// The `frontendToolCalls.length === 0` return branch applies a
|
||||
// flush-or-clear rule: if the pre-strip flush matched (prior_flushed),
|
||||
// the updated messages are emitted and the slot is cleared; if it
|
||||
// didn't match but a prior stash was present, the slot is cleared
|
||||
// with a warn (retaining it risks double-flushing onto the original
|
||||
// AIMessage on a subsequent non-stripping turn since the AIMessage
|
||||
// may still be in history).
|
||||
const priorIntercepted = state.interceptedToolCalls;
|
||||
const priorOriginalId = state.originalAIMessageId;
|
||||
const priorSlotPresent =
|
||||
!!priorIntercepted &&
|
||||
priorIntercepted.length > 0 &&
|
||||
typeof priorOriginalId === "string" &&
|
||||
priorOriginalId.length > 0;
|
||||
let prior_flushed = false;
|
||||
if (priorSlotPresent) {
|
||||
messages = messages.map((msg) => {
|
||||
if (isAIMessage(msg) && msg.id === priorOriginalId) {
|
||||
prior_flushed = true;
|
||||
const existing = msg.tool_calls ?? [];
|
||||
return rebuildAIMessageWithToolCalls(msg, [
|
||||
...existing,
|
||||
...priorIntercepted!,
|
||||
]);
|
||||
}
|
||||
return msg;
|
||||
});
|
||||
if (!prior_flushed) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[intercept_frontend_tools] prior intercept slot held id=${priorOriginalId} but no matching AIMessage was found to flush onto (pre-strip); downstream branches will flush-or-clear.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const lastMessage: BaseMessage | undefined = messages[messages.length - 1];
|
||||
if (lastMessage === undefined || !isAIMessage(lastMessage)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const toolCalls = lastMessage.tool_calls ?? [];
|
||||
const backendToolCalls: ToolCall[] = [];
|
||||
const frontendToolCalls: ToolCall[] = [];
|
||||
for (const call of toolCalls) {
|
||||
if (frontendActionNames.has(call.name)) {
|
||||
frontendToolCalls.push(call);
|
||||
} else {
|
||||
backendToolCalls.push(call);
|
||||
}
|
||||
}
|
||||
|
||||
// `AIMessage.id` is typed `string | undefined` in @langchain/core. If the
|
||||
// upstream provider (or a test fixture) produced an AIMessage without an
|
||||
// id AND this batch needs stripping, `restore_frontend_tools` would never
|
||||
// find a matching id on the later pass — frontend-action calls would be
|
||||
// silently dropped and the user would see nothing. Synthesize a stable
|
||||
// id in place so the strip/stash/restore chain can match. LangChain
|
||||
// AIMessage is mutable; the synthesized id survives `rebuildAIMessageWithToolCalls`
|
||||
// (which copies `id` from `source.id`) and lives on the same object
|
||||
// reference in `messages`.
|
||||
if (
|
||||
frontendToolCalls.length > 0 &&
|
||||
(typeof lastMessage.id !== "string" || lastMessage.id.length === 0)
|
||||
) {
|
||||
const synthesizedId = `synthesized-${randomUUID()}`;
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[intercept_frontend_tools] lastMessage.id is missing on an AIMessage with ${frontendToolCalls.length} frontend-action call(s); synthesizing id=${synthesizedId} so restore_frontend_tools can match. Upstream provider should supply stable AIMessage ids.`,
|
||||
);
|
||||
(lastMessage as AIMessage).id = synthesizedId;
|
||||
}
|
||||
|
||||
if (frontendToolCalls.length === 0) {
|
||||
// No frontend calls in the batch — nothing to strip.
|
||||
//
|
||||
// Three prior-slot cases to distinguish:
|
||||
// (a) prior_flushed === true: we reattached the stashed calls onto
|
||||
// a matching AIMessage above; emit the updated messages and
|
||||
// clear the slot.
|
||||
// (b) priorSlotPresent && !prior_flushed: no matching AIMessage was
|
||||
// found THIS pass. Over a sequence like
|
||||
// [mixed-stash] → [backend-only+unknown] → [pure-backend],
|
||||
// retaining the stash across multiple non-stripping turns would
|
||||
// eventually let `restore_frontend_tools` re-apply it to an
|
||||
// unrelated AIMessage (or let a future intercept pass
|
||||
// double-append onto the original AIMessage still in history).
|
||||
// Flush-or-clear discipline: if we had a stash and this path
|
||||
// isn't stripping, clear it. Emit a warn so operators can
|
||||
// debug the dropped frontend-action dispatch.
|
||||
// (c) No prior slot: no-op.
|
||||
if (prior_flushed) {
|
||||
return {
|
||||
messages,
|
||||
interceptedToolCalls: undefined,
|
||||
originalAIMessageId: undefined,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
if (priorSlotPresent) {
|
||||
const lostIds = priorIntercepted!
|
||||
.map((c) => c.id ?? "<no-id>")
|
||||
.join(", ");
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[intercept_frontend_tools] prior intercept slot held id=${priorOriginalId} but no matching AIMessage was found and this path isn't stripping; clearing stash to prevent later re-application onto an unrelated AIMessage. Lost tool-call ids: [${lostIds}]`,
|
||||
);
|
||||
return {
|
||||
interceptedToolCalls: undefined,
|
||||
originalAIMessageId: undefined,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// Rebuild the AIMessage preserving id (so restore_frontend_tools can
|
||||
// find it later) AND all other metadata (additional_kwargs,
|
||||
// response_metadata, usage_metadata, etc.) with only the backend calls.
|
||||
const strippedAIMessage = rebuildAIMessageWithToolCalls(
|
||||
lastMessage,
|
||||
backendToolCalls,
|
||||
);
|
||||
|
||||
// Compose the outgoing message list with the strip applied so any
|
||||
// post-strip flush attempt sees the final shape.
|
||||
let outgoingMessages: BaseMessage[] = [
|
||||
...messages.slice(0, -1),
|
||||
strippedAIMessage,
|
||||
];
|
||||
|
||||
// Mixed-batch overwrite guard: if a prior stash is still present and
|
||||
// was NOT flushed in the pre-strip pass above, we are about to
|
||||
// overwrite the slot. Try one more flush against the post-strip
|
||||
// `outgoingMessages` before giving up. If still unmatched, warn
|
||||
// loudly — merging different AIMessage ids into one slot would
|
||||
// corrupt `originalAIMessageId`, so we accept losing the prior stash
|
||||
// in exchange for a coherent new one. The warn mirrors the
|
||||
// "no matching AIMessage" style used by `restore_frontend_tools`.
|
||||
if (priorSlotPresent && !prior_flushed) {
|
||||
let lateFlushed = false;
|
||||
outgoingMessages = outgoingMessages.map((msg) => {
|
||||
if (isAIMessage(msg) && msg.id === priorOriginalId) {
|
||||
lateFlushed = true;
|
||||
const existing = msg.tool_calls ?? [];
|
||||
return rebuildAIMessageWithToolCalls(msg, [
|
||||
...existing,
|
||||
...priorIntercepted!,
|
||||
]);
|
||||
}
|
||||
return msg;
|
||||
});
|
||||
if (!lateFlushed) {
|
||||
const lostIds = priorIntercepted!
|
||||
.map((c) => c.id ?? "<no-id>")
|
||||
.join(", ");
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[intercept_frontend_tools] prior intercept slot held id=${priorOriginalId} but no matching AIMessage was found to flush onto (pre- or post-strip); overwriting stash with current mixed-batch intercept. Lost tool-call ids: [${lostIds}]`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The outer cast passes the return past a pre-existing pnpm monorepo
|
||||
// resolution quirk: `@langchain/langgraph@1.1.5` pins `@langchain/core`
|
||||
// at a different patch level than this agent's direct dep, so our
|
||||
// imported `AIMessage/BaseMessage` and the graph-state's internal
|
||||
// version are nominally distinct types though structurally identical
|
||||
// at runtime. chat_node's `return { messages: [response] }` hits the
|
||||
// same mismatch implicitly; cf. the baseline tsc errors on that line.
|
||||
return {
|
||||
messages: outgoingMessages,
|
||||
interceptedToolCalls: frontendToolCalls,
|
||||
originalAIMessageId: lastMessage.id,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
|
||||
// restore_frontend_tools: reattaches the stashed frontend-action tool_calls
|
||||
// to the original AIMessage (matched by id) so the CopilotKit runtime can
|
||||
// dispatch them to the frontend. Mirrors `copilotkitMiddleware.afterAgent`.
|
||||
function restore_frontend_tools(state: AgentState) {
|
||||
const interceptedToolCalls = state.interceptedToolCalls;
|
||||
const originalMessageId = state.originalAIMessageId;
|
||||
if (
|
||||
!interceptedToolCalls ||
|
||||
interceptedToolCalls.length === 0 ||
|
||||
!originalMessageId
|
||||
) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Widen to our directly-imported `BaseMessage` for `isAIMessage` —
|
||||
// see the matching cast in `intercept_frontend_tools` above.
|
||||
const messages = (state.messages ?? []) as unknown as BaseMessage[];
|
||||
let messageFound = false;
|
||||
const updatedMessages: BaseMessage[] = messages.map((msg) => {
|
||||
if (isAIMessage(msg) && msg.id === originalMessageId) {
|
||||
messageFound = true;
|
||||
const existing = msg.tool_calls ?? [];
|
||||
// Preserve all AIMessage metadata (additional_kwargs,
|
||||
// response_metadata, usage_metadata, name, invalid_tool_calls)
|
||||
// — a naive rebuild drops them and breaks LangSmith tracing +
|
||||
// token accounting.
|
||||
return rebuildAIMessageWithToolCalls(msg, [
|
||||
...existing,
|
||||
...interceptedToolCalls,
|
||||
]);
|
||||
}
|
||||
return msg;
|
||||
});
|
||||
|
||||
if (!messageFound) {
|
||||
// This node is terminal (edge goes to END). Clear both slots so a
|
||||
// stale stash can't be flushed onto an unrelated AIMessage on a
|
||||
// later intercept pass. The warn is the diagnostic signal —
|
||||
// persisting the slot would corrupt future turns rather than help
|
||||
// diagnose this one.
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[restore_frontend_tools] original AIMessage id=${originalMessageId} not found in messages; clearing stash to avoid cross-turn corruption`,
|
||||
);
|
||||
return {
|
||||
interceptedToolCalls: undefined,
|
||||
originalAIMessageId: undefined,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
|
||||
// See note on the matching return in `intercept_frontend_tools`.
|
||||
return {
|
||||
messages: updatedMessages,
|
||||
interceptedToolCalls: undefined,
|
||||
originalAIMessageId: undefined,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
|
||||
// The return type is the union of node names plus END, matching the
|
||||
// shape addConditionalEdges expects from its callback.
|
||||
function shouldContinue({
|
||||
messages,
|
||||
copilotkit,
|
||||
}: AgentState):
|
||||
| "intercept_frontend_tools"
|
||||
| "tool_node"
|
||||
| "restore_frontend_tools"
|
||||
| "emit_unknown_tools_notice"
|
||||
| typeof END {
|
||||
// Guard the tool-call-carrying variant structurally instead of casting
|
||||
// BaseMessage to AIMessage.
|
||||
const lastMessage: BaseMessage | undefined = messages[messages.length - 1];
|
||||
if (lastMessage === undefined) {
|
||||
return END;
|
||||
}
|
||||
// AIMessage is the only message variant that carries tool_calls. Use
|
||||
// the `isAIMessage` type predicate from @langchain/core/messages so the
|
||||
// narrowing is checked rather than cast.
|
||||
if (!isAIMessage(lastMessage)) {
|
||||
const kind = lastMessage._getType();
|
||||
// Log and fall through to END in both dev and prod so a graph-shape
|
||||
// bug doesn't crash the process. Tradeoff: the user sees the turn
|
||||
// end silently (no synthetic error message surfaced to the chat).
|
||||
// Follow-up: emit a synthetic AIMessage from chat_node (not this
|
||||
// routing function) the next time we observe unexpected internal
|
||||
// state, so the user sees "I hit an unexpected internal state —
|
||||
// please try rephrasing."
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn("[shouldContinue] unexpected last message type:", kind);
|
||||
return END;
|
||||
}
|
||||
|
||||
// Evaluate ALL tool calls. If ANY tool call targets a backend tool (i.e.
|
||||
// not a CopilotKit frontend action), we must route to `tool_node` so the
|
||||
// backend tool runs — returning END on mixed batches would silently
|
||||
// drop the backend call.
|
||||
const toolCalls = lastMessage.tool_calls ?? [];
|
||||
if (toolCalls.length > 0) {
|
||||
const actionNames = new Set((copilotkit?.actions ?? []).map((a) => a.name));
|
||||
// Widen to `Set<string>` because TypeScript's `Set<T>.has` parameter
|
||||
// is invariant on T — a `Set<"getWeather" | "deleteProverb">` would
|
||||
// reject a caller-supplied plain `string` at compile time even though
|
||||
// the runtime answer (`false` for unknown names) is exactly what we
|
||||
// want.
|
||||
const backendToolNames = new Set<string>(tools.map((t) => t.name));
|
||||
|
||||
let hasBackendTool = false;
|
||||
let hasFrontendAction = false;
|
||||
let hasUnknown = false;
|
||||
for (const toolCall of toolCalls) {
|
||||
const name = toolCall.name;
|
||||
if (actionNames.has(name)) {
|
||||
hasFrontendAction = true;
|
||||
// Frontend action — handled client-side.
|
||||
continue;
|
||||
}
|
||||
if (backendToolNames.has(name)) {
|
||||
hasBackendTool = true;
|
||||
continue;
|
||||
}
|
||||
// Unknown name: neither a frontend action nor a registered backend
|
||||
// tool. Track it so we can route unknown-bearing batches through
|
||||
// emit_unknown_tools_notice (below), which synthesizes error
|
||||
// ToolMessages for each unknown call and strips them off the
|
||||
// AIMessage so the frontend runtime never sees them.
|
||||
hasUnknown = true;
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[shouldContinue] unknown tool call name '${name}' — will route through emit_unknown_tools_notice unless a known backend tool is also present in this batch`,
|
||||
);
|
||||
}
|
||||
|
||||
// Mixed batch (backend + frontend action): route through the intercept
|
||||
// node first so ToolNode doesn't choke on the frontend-action call,
|
||||
// then tool_node executes backend calls, then chat_node (looped) will
|
||||
// reach END via the restore node.
|
||||
//
|
||||
// Note: if the batch also contains unknown calls, we still prefer
|
||||
// "tool_node" over the unknown-notice path when a backend call is
|
||||
// present — ToolNode itself emits an error ToolMessage for unknown
|
||||
// names and the graph loops back to chat_node with that context.
|
||||
// The unknown-notice path is reserved for batches that would otherwise
|
||||
// end the turn without running tool_node.
|
||||
if (hasBackendTool && hasFrontendAction) {
|
||||
return "intercept_frontend_tools";
|
||||
}
|
||||
if (hasBackendTool) {
|
||||
return "tool_node";
|
||||
}
|
||||
// No backend tool. If the batch carries ANY unknown calls (with or
|
||||
// without frontend actions), route through emit_unknown_tools_notice
|
||||
// so:
|
||||
// (a) error ToolMessages are synthesized for each unknown call,
|
||||
// keeping the AIMessage+ToolMessage sequence well-formed for
|
||||
// OpenAI on the next turn (no dangling tool_calls);
|
||||
// (b) the AIMessage retains its unknown tool_calls alongside the
|
||||
// knownCalls so every errorToolMessage has a matching
|
||||
// preceding tool_call.id. The frontend runtime does not
|
||||
// re-dispatch calls that carry a matching ToolMessage result.
|
||||
// (c) in the frontend-action + unknown mixed case, the surviving
|
||||
// frontend-action calls still reach the frontend via the
|
||||
// terminal restore path (emit_unknown_tools_notice goes to END
|
||||
// and the rebuilt AIMessage retains both the known frontend
|
||||
// calls and the unknown calls, the latter pre-resolved by
|
||||
// their error ToolMessages).
|
||||
if (hasUnknown) {
|
||||
return "emit_unknown_tools_notice";
|
||||
}
|
||||
}
|
||||
|
||||
// All paths that reach here are assistant replies with no pending backend tool_calls.
|
||||
// Route through restore_frontend_tools; it is a no-op when nothing was intercepted.
|
||||
return "restore_frontend_tools";
|
||||
}
|
||||
|
||||
// emit_unknown_tools_notice: when the model emits a tool_calls batch that
|
||||
// includes names the agent cannot dispatch (neither a registered backend
|
||||
// tool nor a frontend action), chat_node's conditional edge routes here.
|
||||
// Responsibilities:
|
||||
//
|
||||
// 1. Synthesize an error ToolMessage for each unknown tool_call that
|
||||
// carries a non-empty `call.id`. Without this, the AIMessage's
|
||||
// unresolved tool_calls leave a dangling tool-use turn — OpenAI
|
||||
// rejects any AIMessage with tool_calls not followed by matching
|
||||
// ToolMessages on the NEXT user turn, poisoning the conversation.
|
||||
// Unknown calls with a missing/empty id are DROPPED from both the
|
||||
// ToolMessage list AND the AIMessage's tool_calls (emitting
|
||||
// `tool_call_id: ""` would itself be rejected; keeping the call on
|
||||
// the AIMessage without a matching ToolMessage re-introduces the
|
||||
// dangling-reference bug).
|
||||
// 2. Rebuild the prior AIMessage with rebuildAIMessageWithToolCalls,
|
||||
// retaining BOTH the knownCalls AND every unknown call whose
|
||||
// tool_call_id was retained in (1). The retained unknowns are
|
||||
// what makes the transcript well-formed: each errorToolMessage
|
||||
// emitted in (1) has a matching `tool_call.id` on the immediately
|
||||
// preceding AIMessage, which is what OpenAI's chat-completions
|
||||
// API validates on the next user turn. Stripping the unknowns
|
||||
// here (as an earlier revision did) while still appending their
|
||||
// error ToolMessages produced orphaned `tool_call_id`s and
|
||||
// "tool_call_id does not match any preceding tool_calls" errors
|
||||
// on the next turn.
|
||||
// Dropped-id unknowns — those with no usable tool_call_id — are
|
||||
// still omitted from tool_calls since they have no matching
|
||||
// ToolMessage result; keeping them would reintroduce the dangling
|
||||
// reference on the next turn.
|
||||
// Safety for the frontend: tool_calls on the AIMessage are
|
||||
// dispatched by the CopilotKit frontend runtime only when the
|
||||
// model streams TOOL_CALL_START / TOOL_CALL_END events in the
|
||||
// current turn. This node does not emit those events — it only
|
||||
// writes to state.messages. On the next turn the frontend sees
|
||||
// each retained unknown paired with its error ToolMessage in the
|
||||
// snapshot; pairs that already carry a result are not
|
||||
// re-dispatched.
|
||||
// 3. In the PURE-unknown batch (`knownCalls.length === 0`), append a
|
||||
// user-visible AIMessage notice so the turn doesn't end silently,
|
||||
// and clear any stale intercept slot from a prior turn.
|
||||
// In the MIXED frontend-action + unknown batch, do NOT append the
|
||||
// notice — the surviving frontend-action tool_calls on
|
||||
// strippedAIMessage need matching ToolMessages on the next turn,
|
||||
// and a trailing AIMessage(notice) produces an ill-formed OpenAI
|
||||
// transcript. The surviving calls reach the frontend via the
|
||||
// outgoing `restore_frontend_tools` → END path.
|
||||
//
|
||||
// Routing: outgoing edge goes to `restore_frontend_tools` (not END).
|
||||
// That node no-ops when the slot is empty, so the pure-unknown case
|
||||
// still terminates cleanly, while the mixed case gets canonical
|
||||
// restore-then-END handling and any prior unflushed stash is cleared.
|
||||
//
|
||||
// Conditional edges are pure routing functions — they cannot mutate
|
||||
// state — so the state rewrite lives here.
|
||||
function emit_unknown_tools_notice(state: AgentState) {
|
||||
const messages = (state.messages ?? []) as unknown as BaseMessage[];
|
||||
const lastMessage: BaseMessage | undefined = messages[messages.length - 1];
|
||||
if (lastMessage === undefined || !isAIMessage(lastMessage)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Mirror shouldContinue's partition logic so the same known-set defines
|
||||
// what counts as unknown. `tools` is the backend registry; the frontend
|
||||
// action set comes from state.copilotkit.actions.
|
||||
const frontendActionNames = new Set(
|
||||
(state.copilotkit?.actions ?? []).map((a: { name: string }) => a.name),
|
||||
);
|
||||
const backendToolNames = new Set<string>(tools.map((t) => t.name));
|
||||
|
||||
const allCalls = lastMessage.tool_calls ?? [];
|
||||
const knownCalls: ToolCall[] = [];
|
||||
const unknownCalls: ToolCall[] = [];
|
||||
for (const call of allCalls) {
|
||||
if (frontendActionNames.has(call.name) || backendToolNames.has(call.name)) {
|
||||
knownCalls.push(call);
|
||||
} else {
|
||||
unknownCalls.push(call);
|
||||
}
|
||||
}
|
||||
|
||||
if (unknownCalls.length === 0) {
|
||||
// Nothing unknown to notify about — shouldContinue shouldn't have
|
||||
// routed here, but be defensive.
|
||||
return {};
|
||||
}
|
||||
|
||||
// Partition unknowns by whether they carry a usable tool_call_id.
|
||||
// OpenAI rejects ToolMessages whose `tool_call_id` doesn't match a
|
||||
// preceding AIMessage tool_call id — including empty strings. The
|
||||
// only safe handling for an unknown tool_call with a missing/empty
|
||||
// id is to DROP IT from both the error ToolMessage list AND the
|
||||
// AIMessage's tool_calls (so no dangling reference remains). This
|
||||
// mirrors `deleteProverb`'s refusal to emit `tool_call_id: ""`.
|
||||
const unknownWithId: ToolCall[] = [];
|
||||
for (const call of unknownCalls) {
|
||||
const id = call.id;
|
||||
if (typeof id === "string" && id.length > 0) {
|
||||
unknownWithId.push(call);
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[emit_unknown_tools_notice] unknown tool_call '${call.name}' has no id; dropping from both errorToolMessages and strippedAIMessage.tool_calls to avoid emitting a ToolMessage with empty tool_call_id`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// If every unknown call lacked an id, `unknownWithId` is empty and
|
||||
// `errorToolMessages` below will be empty — the AIMessage retains
|
||||
// only `knownCalls` and we still emit the notice in the pure-unknown
|
||||
// case below (drop-only is still a reportable turn).
|
||||
|
||||
// Rebuild the prior AIMessage preserving its id + metadata, retaining
|
||||
// BOTH knownCalls AND unknownWithId on `tool_calls`. Retaining the
|
||||
// unknowns is what keeps the OpenAI transcript well-formed on the
|
||||
// next user turn: every errorToolMessage below references an id
|
||||
// that still appears on the immediately preceding AIMessage's
|
||||
// `tool_calls`. Stripping the unknowns while still emitting their
|
||||
// error ToolMessages produced orphaned `tool_call_id`s — OpenAI's
|
||||
// chat completions API rejects a ToolMessage whose `tool_call_id`
|
||||
// does not match a preceding AIMessage `tool_call.id`, so the next
|
||||
// user turn failed. With the unknowns retained, the AIMessage →
|
||||
// ToolMessage(result) pairing is intact for every unknown.
|
||||
//
|
||||
// Safety for the frontend: the frontend action handler only
|
||||
// dispatches a tool_call when it receives a live TOOL_CALL_START /
|
||||
// TOOL_CALL_END event stream during the current agent turn. This
|
||||
// node DOES NOT emit those events — it only writes to state.messages
|
||||
// after the model stream is already finalized. On the next turn,
|
||||
// the frontend sees the pair (AIMessage.tool_call + ToolMessage
|
||||
// result) already present in the snapshot; pairs that carry a
|
||||
// result are treated as resolved and are not re-dispatched.
|
||||
//
|
||||
// Dropped-id unknowns (no usable tool_call_id) are still omitted
|
||||
// from tool_calls — they have no matching ToolMessage, so keeping
|
||||
// them would reintroduce the dangling-id problem on the next turn.
|
||||
const retainedCalls: ToolCall[] = [...knownCalls, ...unknownWithId];
|
||||
const strippedAIMessage = rebuildAIMessageWithToolCalls(
|
||||
lastMessage,
|
||||
retainedCalls,
|
||||
);
|
||||
|
||||
const errorToolMessages = unknownWithId.map((call) => {
|
||||
return new ToolMessage({
|
||||
status: "error",
|
||||
name: call.name,
|
||||
// Narrowed: unknownWithId only contains calls whose id is a
|
||||
// non-empty string.
|
||||
tool_call_id: call.id as string,
|
||||
content: `Tool '${call.name}' is not available in this environment.`,
|
||||
});
|
||||
});
|
||||
|
||||
// Mixed frontend-action + unknown batch: the surviving knownCalls
|
||||
// are frontend-action calls that still need to reach the frontend
|
||||
// runtime. Appending an `AIMessage(notice)` here would leave
|
||||
// strippedAIMessage's frontend-action tool_calls with no matching
|
||||
// ToolMessages before the trailing notice, producing an ill-formed
|
||||
// OpenAI transcript on replay. Instead, suppress the notice in the
|
||||
// mixed case and let the outgoing edge route the surviving calls
|
||||
// through `restore_frontend_tools` → END for normal dispatch.
|
||||
//
|
||||
// Pure-unknown batch (`knownCalls.length === 0`): emit the notice
|
||||
// as before so the turn doesn't end silently.
|
||||
const unknownNames = unknownCalls.map((c) => c.name);
|
||||
const trailingMessages: BaseMessage[] =
|
||||
knownCalls.length === 0
|
||||
? [
|
||||
new AIMessage({
|
||||
content: `I tried to call tools that aren't available in this environment (${unknownNames.join(
|
||||
", ",
|
||||
)}). Cancelling this turn.`,
|
||||
}),
|
||||
]
|
||||
: [];
|
||||
|
||||
// Always clear any prior-turn intercept slot before routing to
|
||||
// `restore_frontend_tools`. Gating the clear on `knownCalls.length === 0`
|
||||
// is incorrect: in a MIXED batch (knownCalls.length > 0) the surviving
|
||||
// frontend-action calls ride the stripped AIMessage to `restore_frontend_tools`,
|
||||
// which consumes `interceptedToolCalls` + `originalAIMessageId`. If a stale
|
||||
// stash from a PRIOR turn still sits in that slot, it would be grafted
|
||||
// onto an unrelated AIMessage in THIS turn. Clearing unconditionally
|
||||
// guarantees restore_frontend_tools only sees state stashed for the
|
||||
// current turn (which, from this node, is always empty — no stash is
|
||||
// written here).
|
||||
const slotClear: Partial<AgentState> = {
|
||||
interceptedToolCalls: undefined,
|
||||
originalAIMessageId: undefined,
|
||||
};
|
||||
|
||||
// Sequence on the channel (mixed case):
|
||||
// [...existing..., strippedAIMessage (replaces lastMessage),
|
||||
// ToolMessage(unknown1), ..., ToolMessage(unknownN)]
|
||||
// Pure-unknown case appends a trailing AIMessage(notice).
|
||||
return {
|
||||
messages: [
|
||||
...messages.slice(0, -1),
|
||||
strippedAIMessage,
|
||||
...errorToolMessages,
|
||||
...trailingMessages,
|
||||
],
|
||||
...slotClear,
|
||||
} as unknown as Partial<AgentState>;
|
||||
}
|
||||
|
||||
const workflow = new StateGraph(AgentStateAnnotation)
|
||||
.addNode("chat_node", chat_node)
|
||||
.addNode("tool_node", new ToolNode(tools))
|
||||
.addNode("intercept_frontend_tools", intercept_frontend_tools)
|
||||
.addNode("restore_frontend_tools", restore_frontend_tools)
|
||||
.addNode("emit_unknown_tools_notice", emit_unknown_tools_notice)
|
||||
.addEdge(START, "chat_node")
|
||||
.addEdge("intercept_frontend_tools", "tool_node")
|
||||
.addEdge("tool_node", "chat_node")
|
||||
.addEdge("restore_frontend_tools", END)
|
||||
// Route through `restore_frontend_tools` (not END) so any prior
|
||||
// unflushed intercept stash is cleared and any surviving
|
||||
// frontend-action tool_calls retained on the stripped AIMessage in
|
||||
// the mixed-batch case are reattached via the canonical restore
|
||||
// path before termination. `restore_frontend_tools` no-ops when the
|
||||
// slot is empty, so the pure-unknown case still terminates cleanly.
|
||||
.addEdge("emit_unknown_tools_notice", "restore_frontend_tools")
|
||||
.addConditionalEdges("chat_node", shouldContinue);
|
||||
|
||||
const memory = new MemorySaver();
|
||||
|
||||
export const graph = workflow.compile({
|
||||
checkpointer: memory,
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"lib": ["ES2022"],
|
||||
"module": "Node16",
|
||||
"moduleResolution": "node16",
|
||||
"resolvePackageJsonExports": true,
|
||||
"resolvePackageJsonImports": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# LangGraph deployment URL (required in production, defaults to http://localhost:8125 in dev)
|
||||
# LANGGRAPH_DEPLOYMENT_URL=
|
||||
|
||||
# LANGSMITH_* variables below are primarily consumed by the agent /
|
||||
# LangGraph deployment at deploy time (set them in the agent's
|
||||
# environment, e.g. the LangGraph Cloud deployment or `apps/agent/.env`).
|
||||
# The Next.js web app itself only observes LANGSMITH_API_KEY here to
|
||||
# emit a dev-only "missing key" warning and to forward the key through
|
||||
# to the LangGraphAgent client in apps/web/src/app/api/copilotkit/route.ts.
|
||||
|
||||
# LangSmith API key (optional, for production tracing)
|
||||
# LANGSMITH_API_KEY=
|
||||
|
||||
# Enable LangSmith tracing (optional)
|
||||
# LANGSMITH_TRACING=true
|
||||
|
||||
# LangSmith project name (optional; used by LangSmith tracing if LANGSMITH_TRACING=true)
|
||||
# LANGSMITH_PROJECT=interrupts-langgraph
|
||||
@@ -0,0 +1,23 @@
|
||||
import nextCoreWebVitals from "eslint-config-next/core-web-vitals";
|
||||
import nextTypescript from "eslint-config-next/typescript";
|
||||
|
||||
// Flat-config ESLint setup for Next 16 / React 19 / TS 5. Mirrors the
|
||||
// sibling `examples/showcases/open-mcp-client/apps/web/eslint.config.mjs`
|
||||
// so lint behaviour stays consistent across starters. Next 16 removed
|
||||
// the `next lint` command, so package.json now invokes `eslint .`
|
||||
// directly against this config.
|
||||
const eslintConfig = [
|
||||
...nextCoreWebVitals,
|
||||
...nextTypescript,
|
||||
{
|
||||
ignores: [
|
||||
"node_modules/**",
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export default eslintConfig;
|
||||
+12
-6
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "web-langraph-interrupt",
|
||||
"version": "0.1.0",
|
||||
"name": "web-langgraph-interrupt",
|
||||
"version": "0.1.7",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint"
|
||||
"lint": "eslint . --max-warnings=0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@copilotkit/react-core": "workspace:*",
|
||||
@@ -16,15 +16,21 @@
|
||||
"react": "^19.2.1",
|
||||
"react-dom": "^19.2.1",
|
||||
"shiki": "^3.22.0",
|
||||
"zod": "^3.24.4"
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/eslintrc": "^3",
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/node": "^20",
|
||||
"@types/node": "^22.19.11",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "16.0.8",
|
||||
"tailwindcss": "^4",
|
||||
"typescript": "^5"
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
},
|
||||
"nx": {
|
||||
"targets": {
|
||||
+5
-4
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "web-langraph-interrupt",
|
||||
"name": "web-langgraph-interrupt",
|
||||
"$schema": "../../node_modules/nx/schemas/project-schema.json",
|
||||
"sourceRoot": "apps/web/src",
|
||||
"projectType": "application",
|
||||
@@ -7,19 +7,20 @@
|
||||
"dev": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "pnpm --filter web-langraph-interrupt dev"
|
||||
"command": "pnpm --filter web-langgraph-interrupt dev"
|
||||
}
|
||||
},
|
||||
"build": {
|
||||
"executor": "nx:run-commands",
|
||||
"cache": false,
|
||||
"options": {
|
||||
"command": "pnpm --filter web-langraph-interrupt build"
|
||||
"command": "pnpm --filter web-langgraph-interrupt build"
|
||||
}
|
||||
},
|
||||
"lint": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "pnpm --filter web-langraph-interrupt lint"
|
||||
"command": "pnpm --filter web-langgraph-interrupt lint"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
Before Width: | Height: | Size: 391 B After Width: | Height: | Size: 391 B |
|
Before Width: | Height: | Size: 1.0 KiB After Width: | Height: | Size: 1.0 KiB |
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.3 KiB |
|
Before Width: | Height: | Size: 128 B After Width: | Height: | Size: 128 B |
|
Before Width: | Height: | Size: 385 B After Width: | Height: | Size: 385 B |
@@ -0,0 +1,149 @@
|
||||
import {
|
||||
CopilotRuntime,
|
||||
ExperimentalEmptyAdapter,
|
||||
copilotRuntimeNextJSAppRouterEndpoint,
|
||||
} from "@copilotkit/runtime";
|
||||
import { LangGraphAgent } from "@copilotkit/runtime/langgraph";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
// 1. Runtime adapter. Since this starter wires agent responses directly
|
||||
// (no OpenAI/Anthropic-compatible chat-completions fallback), we use
|
||||
// the empty adapter: it satisfies the runtime interface without
|
||||
// dispatching any LLM requests of its own.
|
||||
const serviceAdapter = new ExperimentalEmptyAdapter();
|
||||
|
||||
// 2. LANGGRAPH_DEPLOYMENT_URL handling. In development we fall back to
|
||||
// localhost:8125 (the port this starter's LangGraph dev server uses —
|
||||
// see apps/agent/package.json) and warn so the developer sees it. In
|
||||
// production a missing URL is almost always a misconfiguration — but
|
||||
// we defer the hard failure to the first request (see POST below)
|
||||
// rather than throwing at module load. `next build` evaluates route
|
||||
// modules with NODE_ENV=production during route collection, while
|
||||
// runtime-only env vars (Vercel secrets, Railway env, Docker ENV at
|
||||
// container start) are NOT injected at build time. Throwing at module
|
||||
// load would abort otherwise-valid production builds.
|
||||
if (
|
||||
!process.env.LANGGRAPH_DEPLOYMENT_URL &&
|
||||
process.env.NODE_ENV !== "production"
|
||||
) {
|
||||
console.warn(
|
||||
"[copilotkit/route] LANGGRAPH_DEPLOYMENT_URL is not set; falling back to http://localhost:8125. Set LANGGRAPH_DEPLOYMENT_URL in production.",
|
||||
);
|
||||
}
|
||||
|
||||
// LangSmith is optional; warn once at module load so a missing key
|
||||
// surfaces in logs. When absent we omit the langsmithApiKey field
|
||||
// entirely rather than passing "" — omitting the field disables
|
||||
// LangSmith tracing cleanly; passing an empty string may be forwarded
|
||||
// to the SDK. Gate the warn on NODE_ENV so it does not fire during
|
||||
// `next build` in CI (same rationale as the DEPLOYMENT_URL warn above).
|
||||
if (!process.env.LANGSMITH_API_KEY && process.env.NODE_ENV !== "production") {
|
||||
console.warn(
|
||||
"[copilotkit/route] LANGSMITH_API_KEY is not set; LangSmith tracing is disabled for this session.",
|
||||
);
|
||||
}
|
||||
|
||||
// 3. Lazy runtime construction. We build the LangGraphAgent + runtime +
|
||||
// handler on the first request rather than at module load, so that
|
||||
// `next build` (which evaluates this file with NODE_ENV=production but
|
||||
// without runtime env vars) does not bake the fallback localhost URL
|
||||
// into the compiled artifact. The cached closure keeps per-request
|
||||
// overhead at the same single-allocation cost as the eager form.
|
||||
// Matches the signature returned by copilotRuntimeNextJSAppRouterEndpoint:
|
||||
// `(req: Request) => Response | Promise<Response>`. NextRequest extends
|
||||
// Request, so the POST handler below can still pass its req through
|
||||
// without an explicit cast.
|
||||
let cachedHandleRequest:
|
||||
| ((req: Request) => Response | Promise<Response>)
|
||||
| null = null;
|
||||
|
||||
const getHandleRequest = () => {
|
||||
if (cachedHandleRequest) return cachedHandleRequest;
|
||||
|
||||
const deploymentUrl = process.env.LANGGRAPH_DEPLOYMENT_URL;
|
||||
if (!deploymentUrl && process.env.NODE_ENV === "production") {
|
||||
throw new Error(
|
||||
"[copilotkit/route] LANGGRAPH_DEPLOYMENT_URL is required in production. " +
|
||||
"Set it to the deployed LangGraph endpoint for this starter.",
|
||||
);
|
||||
}
|
||||
|
||||
const agent = new LangGraphAgent({
|
||||
deploymentUrl: deploymentUrl || "http://localhost:8125",
|
||||
graphId: "default",
|
||||
// Only pass langsmithApiKey when it's a non-empty string; omitting
|
||||
// the field disables LangSmith tracing without asking the SDK to
|
||||
// authenticate with an empty key.
|
||||
...(process.env.LANGSMITH_API_KEY
|
||||
? { langsmithApiKey: process.env.LANGSMITH_API_KEY }
|
||||
: {}),
|
||||
});
|
||||
|
||||
// Register the single LangGraph agent under the `default` name, which
|
||||
// matches the <CopilotKit agent="default"> prop in layout.tsx. If you
|
||||
// need to expose this agent under an additional id (e.g. when pointing
|
||||
// a second frontend at this runtime), add another entry here and
|
||||
// update the corresponding <CopilotKit agent="..."> prop.
|
||||
const runtime = new CopilotRuntime({
|
||||
agents: {
|
||||
default: agent,
|
||||
},
|
||||
});
|
||||
|
||||
const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
|
||||
runtime,
|
||||
serviceAdapter,
|
||||
endpoint: "/api/copilotkit",
|
||||
});
|
||||
|
||||
cachedHandleRequest = handleRequest;
|
||||
return handleRequest;
|
||||
};
|
||||
|
||||
// 4. POST handler. Resolves the cached handler (building it on first
|
||||
// request), then dispatches. Wrap in try/catch so unhandled exceptions
|
||||
// surface as a structured 500 rather than a raw Next.js error page,
|
||||
// and log the failure for observability. Errors inside the streaming
|
||||
// response are handled by the runtime itself.
|
||||
export const POST = async (req: NextRequest) => {
|
||||
// Redact `detail` in production: raw error messages can leak
|
||||
// internals (stack-adjacent strings, paths, env-var names). Keep
|
||||
// the full detail in non-production builds so developers see the
|
||||
// real cause locally.
|
||||
const isProd = process.env.NODE_ENV === "production";
|
||||
|
||||
// Split construction vs dispatch into two try/catch regions so
|
||||
// logs distinguish runtime-construction failures (bad config / env)
|
||||
// from dispatch failures inside handleRequest. Response shape is
|
||||
// unchanged between the two.
|
||||
let handleRequest: (req: Request) => Response | Promise<Response>;
|
||||
try {
|
||||
handleRequest = getHandleRequest();
|
||||
} catch (err) {
|
||||
console.error("[copilotkit/route] runtime construction failed:", err);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Internal error while dispatching CopilotKit request.",
|
||||
...(isProd
|
||||
? {}
|
||||
: { detail: err instanceof Error ? err.message : String(err) }),
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await handleRequest(req);
|
||||
} catch (err) {
|
||||
console.error("[copilotkit/route] handleRequest dispatch failed:", err);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Internal error while dispatching CopilotKit request.",
|
||||
...(isProd
|
||||
? {}
|
||||
: { detail: err instanceof Error ? err.message : String(err) }),
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
};
|
||||
|
Before Width: | Height: | Size: 25 KiB After Width: | Height: | Size: 25 KiB |
+3
-2
@@ -5,8 +5,9 @@ import "./globals.css";
|
||||
import "@copilotkit/react-ui/styles.css";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Create Next App",
|
||||
description: "Generated by create next app",
|
||||
title: "CopilotKit LangGraph Interrupts Starter",
|
||||
description:
|
||||
"A starter template for building AI agents with human-in-the-loop interrupts using LangGraph and CopilotKit.",
|
||||
};
|
||||
|
||||
export default function RootLayout({
|
||||
@@ -0,0 +1,398 @@
|
||||
"use client";
|
||||
|
||||
import { useCoAgent, useCopilotAction } from "@copilotkit/react-core";
|
||||
import { CopilotKitCSSProperties, CopilotSidebar } from "@copilotkit/react-ui";
|
||||
import { useInterrupt } from "@copilotkit/react-core/v2";
|
||||
import { useState } from "react";
|
||||
|
||||
export default function CopilotKitPage() {
|
||||
const [themeColor, setThemeColor] = useState("#6366f1");
|
||||
|
||||
// 🪁 Frontend Actions: https://docs.copilotkit.ai/guides/frontend-actions
|
||||
useCopilotAction(
|
||||
{
|
||||
name: "setThemeColor",
|
||||
description: "Set the theme color of the page.",
|
||||
parameters: [
|
||||
{
|
||||
name: "themeColor",
|
||||
type: "string",
|
||||
description: "The theme color to set. Make sure to pick nice colors.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
handler({ themeColor }) {
|
||||
// Defensive guard: during streaming, the LLM may invoke the handler
|
||||
// before the `themeColor` arg has fully arrived. Skipping the update
|
||||
// here avoids writing `undefined` into the CSS custom property, which
|
||||
// would collapse the --copilot-kit-primary-color variable.
|
||||
if (typeof themeColor !== "string" || themeColor.length === 0) return;
|
||||
setThemeColor(themeColor);
|
||||
},
|
||||
},
|
||||
[setThemeColor],
|
||||
);
|
||||
|
||||
// 🪁 Interrupts: Handle human-in-the-loop confirmations from the agent
|
||||
// https://docs.copilotkit.ai/coagents/human-in-the-loop (useInterrupt)
|
||||
//
|
||||
// The agent emits interrupt payloads shaped as
|
||||
// { action: "delete_proverb"; proverb: string; message: string }
|
||||
// Today the only supported action is delete_proverb. When future
|
||||
// actions are added, widen InterruptPayload to a discriminated union
|
||||
// on `action` and extend APPROVE_LABELS below — TypeScript will flag
|
||||
// the missing key as a compile error because APPROVE_LABELS is typed
|
||||
// as Record<InterruptPayload["action"], string>.
|
||||
useInterrupt({
|
||||
render: ({ event, resolve }) => {
|
||||
const parsed = parseInterruptPayload(event.value);
|
||||
if (!parsed.ok) {
|
||||
// Unknown/malformed payload shape. Surface a generic fallback
|
||||
// rather than crashing on an unchecked cast. Resolving with a
|
||||
// cancellation unblocks the agent in case the user dismisses it.
|
||||
// Log the reason + raw payload so developers can diagnose
|
||||
// schema drift from the agent side — the UI shows "unknown"
|
||||
// without this, which is useless for debugging. The parser
|
||||
// itself does not log, so this is the single log line for
|
||||
// the whole failure path.
|
||||
console.error(
|
||||
"[interrupts-langgraph] Unknown interrupt payload shape:",
|
||||
parsed.reason,
|
||||
event.value,
|
||||
);
|
||||
return (
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-4 my-2">
|
||||
<p className="text-sm text-red-800">
|
||||
Received an unknown interrupt payload. This will tell the agent to
|
||||
cancel.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => resolve({ approved: false })}
|
||||
className="mt-2 px-3 py-1.5 text-sm font-medium text-gray-700 bg-gray-100 hover:bg-gray-200 rounded-md transition-colors"
|
||||
>
|
||||
Cancel request
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve the button label for this action.
|
||||
const payload = parsed.value;
|
||||
const approveLabel = APPROVE_LABELS[payload.action];
|
||||
|
||||
return (
|
||||
<div className="bg-yellow-50 border border-yellow-200 rounded-lg p-4 my-2">
|
||||
<p className="text-sm font-medium text-yellow-800 mb-1">
|
||||
Confirmation Required
|
||||
</p>
|
||||
<p className="text-sm text-yellow-700 mb-3">{payload.message}</p>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => resolve({ approved: true })}
|
||||
className="px-3 py-1.5 text-sm font-medium text-white bg-red-500 hover:bg-red-600 rounded-md transition-colors"
|
||||
>
|
||||
{approveLabel}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => resolve({ approved: false })}
|
||||
className="px-3 py-1.5 text-sm font-medium text-gray-700 bg-gray-100 hover:bg-gray-200 rounded-md transition-colors"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<main
|
||||
style={
|
||||
{ "--copilot-kit-primary-color": themeColor } as CopilotKitCSSProperties
|
||||
}
|
||||
>
|
||||
<YourMainContent themeColor={themeColor} />
|
||||
<CopilotSidebar
|
||||
clickOutsideToClose={false}
|
||||
defaultOpen={true}
|
||||
labels={{
|
||||
title: "Popup Assistant",
|
||||
initial:
|
||||
'👋 Hi, there! You\'re chatting with an agent. This agent comes with a few tools to get you started.\n\nFor example you can try:\n- **Frontend Tools**: "Set the theme to orange"\n- **Shared State**: "Write a proverb about AI"\n- **Generative UI**: "Get the weather in SF"\n- **Interrupts**: "Delete the first proverb" (will ask for confirmation)\n\nAs you interact with the agent, you\'ll see the UI update in real-time to reflect the agent\'s **state**, **tool calls**, and **progress**.',
|
||||
}}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
// Partial view of the agent state the UI reads + writes. The agent's
|
||||
// full state is defined in apps/agent/src/agent.ts (AgentStateAnnotation);
|
||||
// this type intentionally declares only the subset the UI consumes. Keep
|
||||
// the field names and types in sync with the agent side.
|
||||
type AgentState = {
|
||||
proverbs: string[];
|
||||
};
|
||||
|
||||
// Shape of interrupt payloads produced by deleteProverb on the agent
|
||||
// side. Validated at runtime by `parseInterruptPayload` below. Kept
|
||||
// close to the consumer so divergence surfaces in the renderer where
|
||||
// it's used.
|
||||
type InterruptPayload = {
|
||||
action: "delete_proverb";
|
||||
proverb: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
// Approve-button label per interrupt action. Typed exhaustively against
|
||||
// InterruptPayload["action"] so adding a new action anywhere else in the
|
||||
// file is a compile error until this map is extended.
|
||||
const APPROVE_LABELS: Record<InterruptPayload["action"], string> = {
|
||||
delete_proverb: "Yes, delete it",
|
||||
};
|
||||
|
||||
// Result type surfaces the first failed predicate as a stable reason
|
||||
// string so the caller can log a single message containing both the
|
||||
// reason and the raw value. The parser itself does no logging — the
|
||||
// renderer owns the single log line (avoids the prior double-log).
|
||||
type ParseResult =
|
||||
| { ok: true; value: InterruptPayload }
|
||||
| { ok: false; reason: string };
|
||||
|
||||
function parseInterruptPayload(value: unknown): ParseResult {
|
||||
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
||||
return { ok: false, reason: "payload is not object" };
|
||||
}
|
||||
const v = value as Record<string, unknown>;
|
||||
if (v.action !== "delete_proverb") {
|
||||
return { ok: false, reason: "action mismatch" };
|
||||
}
|
||||
if (typeof v.proverb !== "string") {
|
||||
return { ok: false, reason: "proverb not string" };
|
||||
}
|
||||
if (typeof v.message !== "string") {
|
||||
return { ok: false, reason: "message not string" };
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
value: {
|
||||
action: "delete_proverb",
|
||||
proverb: v.proverb,
|
||||
message: v.message,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function YourMainContent({ themeColor }: { themeColor: string }) {
|
||||
// 🪁 Shared State: https://docs.copilotkit.ai/coagents/shared-state
|
||||
const { state, setState } = useCoAgent<AgentState>({
|
||||
name: "default",
|
||||
initialState: {
|
||||
proverbs: [
|
||||
"CopilotKit may be new, but it's the best thing since sliced bread.",
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// Defensive default: during transient state-sync, `state` or
|
||||
// `state.proverbs` can momentarily be undefined. Coalescing here
|
||||
// keeps the map/length checks below from falling into the
|
||||
// `undefined !== 0` trap that would hide the "No proverbs yet"
|
||||
// empty-state fallback.
|
||||
const proverbs = state?.proverbs ?? [];
|
||||
|
||||
// 🪁 Shared State action: writes into the shared agent state above
|
||||
// (not to be confused with a pure frontend action — this mutates
|
||||
// `proverbs`, which is part of the agent's CoAgent state and is
|
||||
// synced back to the graph on the next turn).
|
||||
// https://docs.copilotkit.ai/coagents/shared-state
|
||||
useCopilotAction(
|
||||
{
|
||||
name: "addProverb",
|
||||
description: "Add a proverb to the list.",
|
||||
parameters: [
|
||||
{
|
||||
name: "proverb",
|
||||
type: "string",
|
||||
description: "The proverb to add. Make it witty, short and concise.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
handler: ({ proverb }) => {
|
||||
// Defensive guard: during streaming, the LLM may invoke the handler
|
||||
// before the `proverb` arg has fully arrived. Skipping the update
|
||||
// here avoids pushing `undefined` into the proverbs array, which
|
||||
// would break React rendering and React key stability.
|
||||
if (typeof proverb !== "string" || proverb.length === 0) return;
|
||||
setState((prevState) => ({
|
||||
...prevState,
|
||||
proverbs: [...(prevState?.proverbs || []), proverb],
|
||||
}));
|
||||
},
|
||||
},
|
||||
[setState],
|
||||
);
|
||||
|
||||
//🪁 Generative UI: https://docs.copilotkit.ai/coagents/generative-ui
|
||||
//
|
||||
// `available: "disabled"` is the correct pairing with `render:` here.
|
||||
// In @copilotkit/react-core, useCopilotAction routes based on the
|
||||
// `available` value (see
|
||||
// node_modules/@copilotkit/react-core/src/hooks/use-copilot-action.ts
|
||||
// `getActionConfig`):
|
||||
// - "enabled" / "remote" → frontend tool (handler runs client-side)
|
||||
// - "frontend" / "disabled" → render-only (registers a tool-call
|
||||
// renderer; no handler)
|
||||
// We want the BACKEND `getWeather` tool in agent.ts to execute the
|
||||
// handler server-side AND have the streamed tool-call args drive a
|
||||
// client-side gen-UI render here. The render-only path (via
|
||||
// useRenderToolCall) still adds this renderer to
|
||||
// `copilotkit.renderToolCalls` regardless of `available`, so the
|
||||
// WeatherCard fires during tool-call streaming (see
|
||||
// examples/showcases/scene-creator/src/app/page.tsx for the same
|
||||
// pattern). Setting `"enabled"` here would register a FRONTEND handler
|
||||
// of the same name and collide with the backend tool.
|
||||
useCopilotAction(
|
||||
{
|
||||
name: "getWeather",
|
||||
description: "Get the weather for a given location.",
|
||||
available: "disabled",
|
||||
parameters: [{ name: "location", type: "string", required: true }],
|
||||
render: ({ args }) => {
|
||||
return <WeatherCard location={args.location} themeColor={themeColor} />;
|
||||
},
|
||||
},
|
||||
[themeColor],
|
||||
);
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{ backgroundColor: themeColor }}
|
||||
className="h-screen w-screen flex justify-center items-center flex-col transition-colors duration-300"
|
||||
>
|
||||
<div className="bg-white/20 backdrop-blur-md p-8 rounded-2xl shadow-xl max-w-2xl w-full">
|
||||
<h1 className="text-4xl font-bold text-white mb-2 text-center">
|
||||
Proverbs
|
||||
</h1>
|
||||
<p className="text-gray-200 text-center italic mb-6">
|
||||
This is a demonstrative page, but it could be anything you want! 🪁
|
||||
</p>
|
||||
<hr className="border-white/20 my-6" />
|
||||
<div className="flex flex-col gap-3">
|
||||
{proverbs.map((proverb, index) => (
|
||||
// Intentional index-and-content composite key: `${index}-${proverb}`.
|
||||
// Plain `proverb` collides on duplicates (React logs a
|
||||
// duplicate-key warning and collapses to one node); plain
|
||||
// `index` destabilizes rows across agent-side inserts/deletes.
|
||||
// The composite tolerates duplicates for this demo where rows
|
||||
// are append-only and reordering is not a concern. Migrating
|
||||
// to `{id, text}` objects (seeded with crypto.randomUUID())
|
||||
// is the proper fix and is deferred to a follow-up since it
|
||||
// requires widening AgentState.proverbs.
|
||||
<div
|
||||
key={`${index}-${proverb}`}
|
||||
className="bg-white/15 p-4 rounded-xl text-white relative group hover:bg-white/20 transition-all"
|
||||
>
|
||||
<p className="pr-8">{proverb}</p>
|
||||
<button
|
||||
onClick={() =>
|
||||
setState((prev) => ({
|
||||
...(prev ?? {}),
|
||||
// Filter by value identity rather than captured index:
|
||||
// between render and click, the agent may have
|
||||
// inserted/removed proverbs, shifting `index` to point
|
||||
// at a different entry. Matching on the string value
|
||||
// is stable (and consistent with the React key above).
|
||||
proverbs: (prev?.proverbs ?? []).filter(
|
||||
(p) => p !== proverb,
|
||||
),
|
||||
}))
|
||||
}
|
||||
className="absolute right-3 top-3 opacity-0 group-hover:opacity-100 transition-opacity
|
||||
bg-red-500 hover:bg-red-600 text-white rounded-full h-6 w-6 flex items-center justify-center"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{proverbs.length === 0 && (
|
||||
<p className="text-center text-white/80 italic my-8">
|
||||
No proverbs yet. Ask the assistant to add some!
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SunIcon() {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
className="w-14 h-14 text-yellow-200"
|
||||
>
|
||||
<circle cx="12" cy="12" r="5" />
|
||||
<path
|
||||
d="M12 1v2M12 21v2M4.22 4.22l1.42 1.42M18.36 18.36l1.42 1.42M1 12h2M21 12h2M4.22 19.78l1.42-1.42M18.36 5.64l1.42-1.42"
|
||||
strokeWidth="2"
|
||||
stroke="currentColor"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
// Weather card rendered by the getWeather action. `location` and
|
||||
// `themeColor` are driven by the agent's tool-call args + frontend state.
|
||||
function WeatherCard({
|
||||
location,
|
||||
themeColor,
|
||||
}: {
|
||||
location?: string;
|
||||
themeColor: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
style={{ backgroundColor: themeColor }}
|
||||
className="rounded-xl shadow-xl mt-6 mb-4 max-w-md w-full"
|
||||
>
|
||||
<div className="bg-white/20 p-4 w-full">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h3 className="text-xl font-bold text-white capitalize">
|
||||
{/* During streaming, the tool-call arg may not have arrived
|
||||
yet, leaving `location` undefined and producing a blank
|
||||
heading. Show a loading placeholder in that window. */}
|
||||
{location || "Loading…"}
|
||||
</h3>
|
||||
<p className="text-white">Current Weather</p>
|
||||
</div>
|
||||
<SunIcon />
|
||||
</div>
|
||||
|
||||
<div className="mt-4 flex items-end justify-between">
|
||||
<div className="text-3xl font-bold text-white">70°</div>
|
||||
<div className="text-sm text-white">Clear skies</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 pt-4 border-t border-white">
|
||||
<div className="grid grid-cols-3 gap-2 text-center">
|
||||
<div>
|
||||
<p className="text-white text-xs">Humidity</p>
|
||||
<p className="text-white font-medium">45%</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-white text-xs">Wind</p>
|
||||
<p className="text-white font-medium">5 mph</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-white text-xs">Feels Like</p>
|
||||
<p className="text-white font-medium">72°</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2017",
|
||||
"target": "ES2022",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
@@ -18,6 +18,7 @@
|
||||
"name": "next"
|
||||
}
|
||||
],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
+2
-6
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "langgraph-js-starter",
|
||||
"version": "0.1.4",
|
||||
"version": "0.1.7",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"apps/*"
|
||||
@@ -14,12 +14,8 @@
|
||||
"@langchain/langgraph-cli": "^1.0.4",
|
||||
"turbo": "^2.3.3"
|
||||
},
|
||||
"overrides": {
|
||||
"@langchain/core": "^1.0.1",
|
||||
"@langchain/langgraph": "1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
"node": ">=20.0.0"
|
||||
},
|
||||
"packageManager": "pnpm@9.15.0"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
packages:
|
||||
- "apps/*"
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"tasks": {
|
||||
"dev": {
|
||||
"cache": false,
|
||||
"persistent": true
|
||||
},
|
||||
"build": {
|
||||
"outputs": [".next/**", "!.next/cache/**", "dist/**"],
|
||||
"dependsOn": ["^build"]
|
||||
},
|
||||
"start": {
|
||||
"cache": false,
|
||||
"persistent": true,
|
||||
"dependsOn": ["^build"]
|
||||
},
|
||||
"lint": {}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
# langgraph-js-starter
|
||||
|
||||
## 0.1.4
|
||||
|
||||
## 0.1.3
|
||||
|
||||
## 0.1.2
|
||||
|
||||
## 0.1.1
|
||||
@@ -1,139 +0,0 @@
|
||||
/**
|
||||
* This is the main entry point for the agent.
|
||||
* It defines the workflow graph, state, tools, nodes and edges.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
import { RunnableConfig } from "@langchain/core/runnables";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import { ToolNode } from "@langchain/langgraph/prebuilt";
|
||||
import { AIMessage, SystemMessage } from "@langchain/core/messages";
|
||||
import {
|
||||
interrupt,
|
||||
MemorySaver,
|
||||
START,
|
||||
StateGraph,
|
||||
} from "@langchain/langgraph";
|
||||
import { ChatOpenAI } from "@langchain/openai";
|
||||
import {
|
||||
convertActionsToDynamicStructuredTools,
|
||||
CopilotKitStateAnnotation,
|
||||
} from "@copilotkit/sdk-js/langgraph";
|
||||
import { Annotation } from "@langchain/langgraph";
|
||||
|
||||
// 1. Define our agent state, which includes CopilotKit state to
|
||||
// provide actions to the state.
|
||||
const AgentStateAnnotation = Annotation.Root({
|
||||
...CopilotKitStateAnnotation.spec, // CopilotKit state annotation already includes messages, as well as frontend tools
|
||||
proverbs: Annotation<string[]>,
|
||||
});
|
||||
|
||||
// 2. Define the type for our agent state
|
||||
export type AgentState = typeof AgentStateAnnotation.State;
|
||||
|
||||
// 3. Define a simple tool to get the weather statically
|
||||
const getWeather = tool(
|
||||
(args) => {
|
||||
return `The weather for ${args.location} is 70 degrees, clear skies, 45% humidity, 5 mph wind, and feels like 72 degrees.`;
|
||||
},
|
||||
{
|
||||
name: "getWeather",
|
||||
description: "Get the weather for a given location.",
|
||||
schema: z.object({
|
||||
location: z.string().describe("The location to get weather for"),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
// 4. Define a tool that triggers a human-in-the-loop interrupt
|
||||
const deleteProverb = tool(
|
||||
async (args) => {
|
||||
const approval = interrupt({
|
||||
action: "delete_proverb",
|
||||
proverb: args.proverb,
|
||||
message: `Are you sure you want to delete the proverb: "${args.proverb}"?`,
|
||||
});
|
||||
|
||||
if (approval?.approved) {
|
||||
return `Proverb "${args.proverb}" has been deleted.`;
|
||||
}
|
||||
return `Deletion of proverb "${args.proverb}" was cancelled by the user.`;
|
||||
},
|
||||
{
|
||||
name: "deleteProverb",
|
||||
description:
|
||||
"Delete a proverb from the list. This will ask the user for confirmation before deleting.",
|
||||
schema: z.object({
|
||||
proverb: z.string().describe("The proverb to delete"),
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
// 5. Put our tools into an array
|
||||
const tools = [getWeather, deleteProverb];
|
||||
|
||||
// 5. Define the chat node, which will handle the chat logic
|
||||
async function chat_node(state: AgentState, config: RunnableConfig) {
|
||||
// 5.1 Define the model, lower temperature for deterministic responses
|
||||
const model = new ChatOpenAI({ model: "gpt-4o-mini" });
|
||||
|
||||
// 5.2 Bind the tools to the model, include CopilotKit actions. This allows
|
||||
// the model to call tools that are defined in CopilotKit by the frontend.
|
||||
const modelWithTools = model.bindTools!([
|
||||
...convertActionsToDynamicStructuredTools(state.copilotkit?.actions ?? []),
|
||||
...tools,
|
||||
]);
|
||||
|
||||
// 5.3 Define the system message, which will be used to guide the model, in this case
|
||||
// we also add in the language to use from the state.
|
||||
const systemMessage = new SystemMessage({
|
||||
content: `You are a helpful assistant. The current proverbs are ${JSON.stringify(state.proverbs)}. If a user asks to delete a proverb, call deleteProverb to trigger a human-in-the-loop interrupt for confirmation.`,
|
||||
});
|
||||
|
||||
// 5.4 Invoke the model with the system message and the messages in the state
|
||||
const response = await modelWithTools.invoke(
|
||||
[systemMessage, ...state.messages],
|
||||
config,
|
||||
);
|
||||
|
||||
// 5.5 Return the response, which will be added to the state
|
||||
return {
|
||||
messages: response,
|
||||
};
|
||||
}
|
||||
|
||||
// 6. Define the function that determines whether to continue or not,
|
||||
// this is used to determine the next node to run
|
||||
function shouldContinue({ messages, copilotkit }: AgentState) {
|
||||
// 6.1 Get the last message from the state
|
||||
const lastMessage = messages[messages.length - 1] as AIMessage;
|
||||
|
||||
// 7.2 If the LLM makes a tool call, then we route to the "tools" node
|
||||
if (lastMessage.tool_calls?.length) {
|
||||
// Actions are the frontend tools coming from CopilotKit
|
||||
const actions = copilotkit?.actions;
|
||||
const toolCallName = lastMessage.tool_calls![0].name;
|
||||
|
||||
// 7.3 Only route to the tool node if the tool call is not a CopilotKit action
|
||||
if (!actions || actions.every((action) => action.name !== toolCallName)) {
|
||||
return "tool_node";
|
||||
}
|
||||
}
|
||||
|
||||
// 6.4 Otherwise, we stop (reply to the user) using the special "__end__" node
|
||||
return "__end__";
|
||||
}
|
||||
|
||||
// Define the workflow graph
|
||||
const workflow = new StateGraph(AgentStateAnnotation)
|
||||
.addNode("chat_node", chat_node)
|
||||
.addNode("tool_node", new ToolNode(tools))
|
||||
.addEdge(START, "chat_node")
|
||||
.addEdge("tool_node", "chat_node")
|
||||
.addConditionalEdges("chat_node", shouldContinue as any);
|
||||
|
||||
const memory = new MemorySaver();
|
||||
|
||||
export const graph = workflow.compile({
|
||||
checkpointer: memory,
|
||||
});
|
||||
@@ -1,110 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
// "jsx": "preserve", /* Specify what JSX code is generated. */
|
||||
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
|
||||
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
|
||||
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
|
||||
|
||||
/* Modules */
|
||||
"module": "Node16" /* Specify what module code is generated. */,
|
||||
// "rootDir": "./", /* Specify the root folder within your source files. */
|
||||
"moduleResolution": "node16" /* Specify how TypeScript looks up a file from a given module specifier. */,
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
|
||||
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
|
||||
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
|
||||
"resolvePackageJsonExports": true /* Use the package.json 'exports' field when resolving package imports. */,
|
||||
"resolvePackageJsonImports": true /* Use the package.json 'imports' field when resolving imports. */,
|
||||
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
|
||||
// "noUncheckedSideEffectImports": true, /* Check side effect imports. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files. */
|
||||
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
|
||||
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
|
||||
// "outDir": "./", /* Specify an output folder for all emitted files. */
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
|
||||
// "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true /* Enable all strict type-checking options. */,
|
||||
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
|
||||
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
|
||||
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
|
||||
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
|
||||
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
|
||||
// "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
|
||||
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
|
||||
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
|
||||
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
|
||||
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
|
||||
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
|
||||
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true /* Skip type checking all .d.ts files. */
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import {
|
||||
CopilotRuntime,
|
||||
ExperimentalEmptyAdapter,
|
||||
copilotRuntimeNextJSAppRouterEndpoint,
|
||||
} from "@copilotkit/runtime";
|
||||
import { LangGraphAgent } from "@copilotkit/runtime/langgraph";
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
// 1. You can use any service adapter here for multi-agent support. We use
|
||||
// the empty adapter since we're only using one agent.
|
||||
const serviceAdapter = new ExperimentalEmptyAdapter();
|
||||
const agent = new LangGraphAgent({
|
||||
deploymentUrl:
|
||||
process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8125",
|
||||
graphId: "default",
|
||||
langsmithApiKey: process.env.LANGSMITH_API_KEY || "",
|
||||
});
|
||||
// 2. Create the CopilotRuntime instance and utilize the LangGraph AG-UI
|
||||
// integration to setup the connection.
|
||||
const runtime = new CopilotRuntime({
|
||||
agents: {
|
||||
default: agent,
|
||||
starterAgent: agent,
|
||||
},
|
||||
});
|
||||
|
||||
// 3. Build a Next.js API route that handles the CopilotKit runtime requests.
|
||||
export const POST = async (req: NextRequest) => {
|
||||
const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
|
||||
runtime,
|
||||
serviceAdapter,
|
||||
endpoint: "/api/copilotkit",
|
||||
});
|
||||
|
||||
return handleRequest(req);
|
||||
};
|
||||
@@ -1,243 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { useCoAgent, useCopilotAction } from "@copilotkit/react-core";
|
||||
import { CopilotKitCSSProperties, CopilotSidebar } from "@copilotkit/react-ui";
|
||||
import { useInterrupt } from "@copilotkit/react-core/v2";
|
||||
import { useState } from "react";
|
||||
|
||||
export default function CopilotKitPage() {
|
||||
const [themeColor, setThemeColor] = useState("#6366f1");
|
||||
|
||||
// 🪁 Frontend Actions: https://docs.copilotkit.ai/guides/frontend-actions
|
||||
useCopilotAction({
|
||||
name: "setThemeColor",
|
||||
description: "Set the theme color of the page.",
|
||||
parameters: [
|
||||
{
|
||||
name: "themeColor",
|
||||
description: "The theme color to set. Make sure to pick nice colors.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
handler({ themeColor }) {
|
||||
setThemeColor(themeColor);
|
||||
},
|
||||
});
|
||||
|
||||
// 🪁 Interrupts: Handle human-in-the-loop confirmations from the agent
|
||||
useInterrupt({
|
||||
render: ({ event, resolve }) => {
|
||||
const { message, proverb, action } = event.value as {
|
||||
message: string;
|
||||
proverb: string;
|
||||
action: string;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-yellow-50 border border-yellow-200 rounded-lg p-4 my-2">
|
||||
<p className="text-sm font-medium text-yellow-800 mb-1">
|
||||
Confirmation Required
|
||||
</p>
|
||||
<p className="text-sm text-yellow-700 mb-3">{message}</p>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => resolve({ approved: true })}
|
||||
className="px-3 py-1.5 text-sm font-medium text-white bg-red-500 hover:bg-red-600 rounded-md transition-colors"
|
||||
>
|
||||
Yes, delete it
|
||||
</button>
|
||||
<button
|
||||
onClick={() => resolve({ approved: false })}
|
||||
className="px-3 py-1.5 text-sm font-medium text-gray-700 bg-gray-100 hover:bg-gray-200 rounded-md transition-colors"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<main
|
||||
style={
|
||||
{ "--copilot-kit-primary-color": themeColor } as CopilotKitCSSProperties
|
||||
}
|
||||
>
|
||||
<YourMainContent themeColor={themeColor} />
|
||||
<CopilotSidebar
|
||||
clickOutsideToClose={false}
|
||||
defaultOpen={true}
|
||||
labels={{
|
||||
title: "Popup Assistant",
|
||||
initial:
|
||||
'👋 Hi, there! You\'re chatting with an agent. This agent comes with a few tools to get you started.\n\nFor example you can try:\n- **Frontend Tools**: "Set the theme to orange"\n- **Shared State**: "Write a proverb about AI"\n- **Generative UI**: "Get the weather in SF"\n- **Interrupts**: "Delete the first proverb" (will ask for confirmation)\n\nAs you interact with the agent, you\'ll see the UI update in real-time to reflect the agent\'s **state**, **tool calls**, and **progress**.',
|
||||
}}
|
||||
/>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
// State of the agent, make sure this aligns with your agent's state.
|
||||
type AgentState = {
|
||||
proverbs: string[];
|
||||
};
|
||||
|
||||
function YourMainContent({ themeColor }: { themeColor: string }) {
|
||||
// 🪁 Shared State: https://docs.copilotkit.ai/coagents/shared-state
|
||||
const { state, setState } = useCoAgent<AgentState>({
|
||||
name: "default",
|
||||
initialState: {
|
||||
proverbs: [
|
||||
"CopilotKit may be new, but its the best thing since sliced bread.",
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
// 🪁 Frontend Actions: https://docs.copilotkit.ai/coagents/frontend-actions
|
||||
useCopilotAction(
|
||||
{
|
||||
name: "addProverb",
|
||||
description: "Add a proverb to the list.",
|
||||
parameters: [
|
||||
{
|
||||
name: "proverb",
|
||||
description: "The proverb to add. Make it witty, short and concise.",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
handler: ({ proverb }) => {
|
||||
setState((prevState) => ({
|
||||
...prevState,
|
||||
proverbs: [...(prevState?.proverbs || []), proverb],
|
||||
}));
|
||||
},
|
||||
},
|
||||
[setState],
|
||||
);
|
||||
|
||||
//🪁 Generative UI: https://docs.copilotkit.ai/coagents/generative-ui
|
||||
useCopilotAction({
|
||||
name: "getWeather",
|
||||
description: "Get the weather for a given location.",
|
||||
available: "disabled",
|
||||
parameters: [{ name: "location", type: "string", required: true }],
|
||||
render: ({ args }) => {
|
||||
return <WeatherCard location={args.location} themeColor={themeColor} />;
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{ backgroundColor: themeColor }}
|
||||
className="h-screen w-screen flex justify-center items-center flex-col transition-colors duration-300"
|
||||
>
|
||||
<div className="bg-white/20 backdrop-blur-md p-8 rounded-2xl shadow-xl max-w-2xl w-full">
|
||||
<h1 className="text-4xl font-bold text-white mb-2 text-center">
|
||||
Proverbs
|
||||
</h1>
|
||||
<p className="text-gray-200 text-center italic mb-6">
|
||||
This is a demonstrative page, but it could be anything you want! 🪁
|
||||
</p>
|
||||
<hr className="border-white/20 my-6" />
|
||||
<div className="flex flex-col gap-3">
|
||||
{state.proverbs?.map((proverb, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className="bg-white/15 p-4 rounded-xl text-white relative group hover:bg-white/20 transition-all"
|
||||
>
|
||||
<p className="pr-8">{proverb}</p>
|
||||
<button
|
||||
onClick={() =>
|
||||
setState({
|
||||
...state,
|
||||
proverbs: state.proverbs?.filter((_, i) => i !== index),
|
||||
})
|
||||
}
|
||||
className="absolute right-3 top-3 opacity-0 group-hover:opacity-100 transition-opacity
|
||||
bg-red-500 hover:bg-red-600 text-white rounded-full h-6 w-6 flex items-center justify-center"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{state.proverbs?.length === 0 && (
|
||||
<p className="text-center text-white/80 italic my-8">
|
||||
No proverbs yet. Ask the assistant to add some!
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Simple sun icon for the weather card
|
||||
function SunIcon() {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
className="w-14 h-14 text-yellow-200"
|
||||
>
|
||||
<circle cx="12" cy="12" r="5" />
|
||||
<path
|
||||
d="M12 1v2M12 21v2M4.22 4.22l1.42 1.42M18.36 18.36l1.42 1.42M1 12h2M21 12h2M4.22 19.78l1.42-1.42M18.36 5.64l1.42-1.42"
|
||||
strokeWidth="2"
|
||||
stroke="currentColor"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
// Weather card component where the location and themeColor are based on what the agent
|
||||
// sets via tool calls.
|
||||
function WeatherCard({
|
||||
location,
|
||||
themeColor,
|
||||
}: {
|
||||
location?: string;
|
||||
themeColor: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
style={{ backgroundColor: themeColor }}
|
||||
className="rounded-xl shadow-xl mt-6 mb-4 max-w-md w-full"
|
||||
>
|
||||
<div className="bg-white/20 p-4 w-full">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h3 className="text-xl font-bold text-white capitalize">
|
||||
{location}
|
||||
</h3>
|
||||
<p className="text-white">Current Weather</p>
|
||||
</div>
|
||||
<SunIcon />
|
||||
</div>
|
||||
|
||||
<div className="mt-4 flex items-end justify-between">
|
||||
<div className="text-3xl font-bold text-white">70°</div>
|
||||
<div className="text-sm text-white">Clear skies</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 pt-4 border-t border-white">
|
||||
<div className="grid grid-cols-3 gap-2 text-center">
|
||||
<div>
|
||||
<p className="text-white text-xs">Humidity</p>
|
||||
<p className="text-white font-medium">45%</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-white text-xs">Wind</p>
|
||||
<p className="text-white font-medium">5 mph</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-white text-xs">Feels Like</p>
|
||||
<p className="text-white font-medium">72°</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+13
-1
@@ -21,7 +21,19 @@ pre-commit:
|
||||
stage_fixed: true
|
||||
lint-fix:
|
||||
tags: lint
|
||||
run: pnpm run lint && pnpm run format
|
||||
# Scope oxlint and oxfmt to just the files staged for commit — running
|
||||
# `--fix .` / `--write .` across the whole monorepo on every commit is
|
||||
# both slow and blurs the hook's purpose (touch files that aren't part
|
||||
# of this change). Guard against empty `{staged_files}` expansion: when
|
||||
# a commit touches only non-matching files (markdown, YAML), lefthook
|
||||
# still invokes this hook with an empty expansion, and oxlint/oxfmt
|
||||
# would default to operating on the current directory, defeating the
|
||||
# scoping entirely. stage_fixed re-stages whatever the hooks modify.
|
||||
glob: "*.{js,jsx,ts,tsx,mjs,cjs}"
|
||||
run: |
|
||||
if [ -n "{staged_files}" ]; then
|
||||
pnpm exec oxlint --fix {staged_files} && pnpm exec oxfmt --write {staged_files}
|
||||
fi
|
||||
stage_fixed: true
|
||||
test-and-check-packages:
|
||||
tags: test-packages
|
||||
|
||||
+2
-1
@@ -89,7 +89,8 @@
|
||||
"serve-static@<=1.16.0": "1.16.0",
|
||||
"prismjs@<=1.30.0": "1.30.0",
|
||||
"pino@<=10.1.1": "10.1.1",
|
||||
"@copilotkit/license-verifier": "0.2.0"
|
||||
"@copilotkit/license-verifier": "0.2.0",
|
||||
"next": "^16.0.10"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+810
-552
File diff suppressed because it is too large
Load Diff
@@ -9,5 +9,13 @@ packages:
|
||||
- "examples/showcases/generative-ui-playground"
|
||||
- "!examples/v1/_legacy"
|
||||
- "showcase/scripts"
|
||||
- "showcase/ops"
|
||||
# NOTE: `showcase/shell-dashboard` is intentionally NOT part of this
|
||||
# pnpm workspace. It's a flat, standalone Next.js app that ships its
|
||||
# own package-lock.json and is built with `npm ci` (see its Dockerfile).
|
||||
# Including it here would force it into the pnpm lockfile, which
|
||||
# breaks its independent deploy path and invalidates the npm-based
|
||||
# Docker build. `showcase/scripts` IS in the workspace because the
|
||||
# ops package imports from it and both use pnpm.
|
||||
onlyBuiltDependencies:
|
||||
- better-sqlite3
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
shared_python/
|
||||
shared_frontend/
|
||||
shared_typescript/
|
||||
|
||||
# Incremental TypeScript build state — regenerated on every tsc
|
||||
# invocation and not useful to track.
|
||||
**/tsconfig.tsbuildinfo
|
||||
|
||||
# Test fixture generated by showcase/scripts/__tests__/create-integration.test.ts.
|
||||
# If the test is killed before cleanup, the directory leaks into the worktree;
|
||||
# this entry prevents accidental `git add`.
|
||||
packages/test-integration-tmp/
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# Multi-Frontend Consolidation Strategy
|
||||
|
||||
This document records the intentional rationale for having multiple showcase shells in this repo, which shell each one replaces, and how packages target them during the transition.
|
||||
|
||||
> This is a transition-period strategy document. Once the rollouts below are complete, expect this doc to be updated or retired.
|
||||
|
||||
## Why multiple shells exist
|
||||
|
||||
Showcase is in the middle of **consolidating several external frontends into this repo** so they are built, deployed, versioned, and tested alongside the integration packages that back them. The fan-out in `showcase/` is intentional: each shell is the target replacement for a distinct external property, and they co-exist until their respective rollouts cut over.
|
||||
|
||||
The alternative — a single super-shell that hosts every audience — was rejected because the existing external frontends each have a different visual language, different audience framing, and different routing/embed conventions. Merging them into one shell would either lose each identity or require runtime mode switching that obscures what's actually rendered at a given URL.
|
||||
|
||||
## Current shells and their roles
|
||||
|
||||
| Directory | Package name | Role | Status |
|
||||
| ------------------ | -------------------------------------- | ------------------------------------------------------------------------------------- | ------------------------------ |
|
||||
| `shell/` | `@copilotkit/showcase-shell` | Public showcase integrations browser at **showcase.copilotkit.dev**. Canonical today. | Production |
|
||||
| `shell-dojo/` | `@copilotkit/showcase-shell-dojo` | Styled like the external ag-ui Dojo. Target replacement for the ag-ui Dojo property. | Deployed (rollout in progress) |
|
||||
| `shell-docs/` | `@copilotkit/showcase-shell-docs` | Docs-forward shell for the `docs.copilotkit.dev` consolidation. | Deployed (rollout in progress) |
|
||||
| `shell-dashboard/` | `@copilotkit/showcase-shell-dashboard` | Internal feature × integration grid (ops / QA audience). | Internal |
|
||||
|
||||
All shells:
|
||||
|
||||
- Consume the same `shared/` registry / constraints / manifest schema
|
||||
- Pull from the same `registry.json` (generated by `scripts/generate-registry.ts`)
|
||||
- Embed package demos via **iframe pointing at `integration.backend_url + demo.route`** (each package's own deployed Next.js app) — they do not import package code
|
||||
|
||||
They differ in chrome, nav, and audience framing, not in the underlying demo surface.
|
||||
|
||||
### `shell/` — public integrations browser
|
||||
|
||||
The canonical public site. Routes under `src/app/`:
|
||||
|
||||
- `integrations/` — list and per-slug profile, plus `[slug]/[demo]/page.tsx` with Preview / Code / Docs tabs
|
||||
- `docs/[[...slug]]/` — MDX docs served from `src/content/docs/`
|
||||
- `ag-ui/[[...slug]]/` — ag-ui reference content
|
||||
- `matrix/`, `reference/` — matrix and reference surfaces
|
||||
|
||||
Build pipeline runs `generate-registry.ts` + `bundle-demo-content.ts` + `bundle-starter-content.ts` + `generate-search-index.ts` before `next build`. Docker image `showcase-shell`, Railway service `40eea0da-6071-4ea8-bdb9-39afb19225ec`.
|
||||
|
||||
### `shell-dojo/` — Dojo replacement
|
||||
|
||||
A single-page Dojo-style viewer (integration selector + demo list + preview iframe + code pane) styled to match the external ag-ui Dojo. This is **not abandoned spike or cleanup waste** — it is the rollout vehicle for replacing the external ag-ui Dojo site with an in-repo shell fed by the same registry every other showcase surface uses. Reference visuals are kept at `shell-dojo/agent-notes/reference-dojo.png` (external Dojo target) and `shell-dojo/agent-notes/v-final-handcrafted.png` (current iteration).
|
||||
|
||||
Built standalone (no monorepo scripts), Docker image `showcase-shell-dojo`, Railway service `7ad1ece7-2228-49cd-8a78-bddf30322907`. CI builds both shells independently in `.github/workflows/showcase_deploy.yml`.
|
||||
|
||||
### Adding future shells
|
||||
|
||||
The expected pattern for any additional consolidation target:
|
||||
|
||||
1. New directory at `showcase/<shell-name>/` with its own `package.json`, `Dockerfile`, `next.config.ts`
|
||||
2. Consume `shared/` registry + `data/registry.json` generated by `scripts/generate-registry.ts`
|
||||
3. Add a matching entry in `.github/workflows/showcase_deploy.yml` (`workflow_dispatch` option, change-detection filter, build-matrix entry) with its own Railway service id
|
||||
4. Update this document with the target external property, deploy URL, and rollout status
|
||||
5. If the shell needs its own demo-content bundling, extend `scripts/bundle-demo-content.ts` rather than forking it
|
||||
|
||||
## Rollout order
|
||||
|
||||
1. **ag-ui Dojo → `shell-dojo/`** (target #1, in progress).
|
||||
- Build in-repo against the live reference (`agent-notes/reference-dojo.png`)
|
||||
- Deploy Railway service `7ad1ece7-2228-49cd-8a78-bddf30322907` continuously from `main`
|
||||
- Visual parity pass against the reference screenshot
|
||||
- Domain cutover: point the external Dojo domain at the Railway service (see Open Questions)
|
||||
- Archive / redirect the external Dojo repo
|
||||
2. **Further consolidations** (TBD). Any other external frontend that is semantically a view over the showcase registry is a candidate. Per-shell criteria:
|
||||
- Audience and visual language distinct enough that folding into `shell/` would lose identity
|
||||
- Currently lives outside this repo (the point of this exercise is consolidation)
|
||||
- Can be expressed as a view over the existing registry, or the registry schema can be extended to support it
|
||||
|
||||
Order is driven by which external property is most worth consolidating next — not by code readiness in this repo.
|
||||
|
||||
## Per-package shell selection
|
||||
|
||||
**There is currently no explicit shell-selection field on packages.** The relationship is one-to-many in the other direction: every shell can embed every package.
|
||||
|
||||
- **At runtime, in every shell:** each integration package is embedded via `<iframe src={integration.backend_url + demo.route}>`. Both `shell/` (`src/app/integrations/[slug]/[demo]/page.tsx`) and `shell-dojo/` (`src/app/page.tsx`) do this using the exact same `backend_url` and `route` fields from `manifest.yaml`. Which shell a user sees is determined by which **domain** they visit, not by anything on the package.
|
||||
- **For per-package E2E:** tests in `packages/<slug>/tests/e2e/` run against the **package's own dev server** on `http://localhost:3000/demos/<id>` — not against any shell. `scripts/run-e2e-with-aimock.sh <slug>` starts aimock + the package's `pnpm dev` + Playwright; no shell is involved. See `QA-COVERAGE.md` for the current per-package/shared E2E split.
|
||||
- **For shared E2E** (`scripts/__tests__/e2e/`): these target whatever is running at `BASE_URL` (defaults to `http://localhost:3000`). The starter hero tests (`starter-e2e.spec.ts`) expect the starter app; demo tests (`demo-e2e.spec.ts`) expect a package dev server. Again, no shell is mounted.
|
||||
- **`NEXT_PUBLIC_BASE_URL`** on packages points at `showcase.copilotkit.dev` for production links back to the canonical shell. It does not gate which shell embeds a given demo.
|
||||
|
||||
**Implication:** as long as a package's `manifest.yaml` is valid and its backend is deployed with `deployed: true`, both shells pick it up automatically on the next `generate-registry.ts` run. No per-package change is required to appear in a new shell.
|
||||
|
||||
**If per-package shell targeting is ever needed** (e.g., a package should only appear in Dojo-replacement but not the canonical browser), this will require a new manifest field — something like `shells: ["shell", "shell-dojo"]` — plus a filter in each shell's registry consumer. That structural change is not in place today.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Domain / DNS cutover plan for ag-ui Dojo replacement.** Which domain does `shell-dojo/` land on, and is the external Dojo repo archived or redirected on cutover?
|
||||
- **Shared vs. per-shell content.** `shell/` has substantial MDX docs under `src/content/`; `shell-dojo/` has none. When we add a third shell, is documentation shell-specific or hoisted into `shared/`?
|
||||
- **Visibility filtering.** Do we ever need a package to appear in one shell but not another? If yes, add a manifest field now rather than after the second rollout makes the lack of one painful.
|
||||
- **`generate-starters` and preview capture.** These scripts assume the `shell/` layout (`showcase/shell/public/previews/`, `showcase/shell/src/data/registry.json`). When a future shell wants previews, decide whether to hoist these assets into `shared/` or dual-write.
|
||||
- **E2E strategy once shells proliferate.** Per-package E2E still tests the package dev server directly, which is correct. But do we want shell-level E2E (iframe load, nav, search) per shell, and if so, where do those specs live?
|
||||
@@ -61,6 +61,57 @@ One per declared feature. Each demo must:
|
||||
|
||||
---
|
||||
|
||||
## Source of Truth: `examples/integrations/*` vs `showcase/packages/*`
|
||||
|
||||
Two directories hold integration code, and they play different roles. Understanding the relationship is critical before adding or modifying a package.
|
||||
|
||||
### Roles
|
||||
|
||||
- **`examples/integrations/<name>/`** — the **Dojo example**. This is the dep-pinning source of truth: minimal, focused agent code used to prove a framework works against CopilotKit/AG-UI. The weekly drift-detection workflow and the "Always pin agent framework and SDK versions to exact versions from the working Dojo example" rule (see "Dependency Pinning" below) both treat this directory as canonical.
|
||||
- **`showcase/packages/<slug>/`** — the **full triple-duty integration**:
|
||||
1. Partner-facing demo (lives on `showcase.copilotkit.dev`)
|
||||
2. Cloneable starter source (composed into `showcase/starters/<slug>/` by `generate-starters.ts`)
|
||||
3. Iframe-embedded experience inside the public showcase shell
|
||||
|
||||
### Automation Direction (one-way)
|
||||
|
||||
```
|
||||
examples/integrations/<name>/ ──(migrate-integration-examples.ts)──▶ showcase/packages/<slug>/src/agents/
|
||||
showcase/packages/<slug>/ ──(generate-starters.ts)────────────▶ showcase/starters/<slug>/
|
||||
```
|
||||
|
||||
- `showcase/scripts/migrate-integration-examples.ts` copies agent code **from** `examples/integrations/<name>/` **into** `showcase/packages/<slug>/src/agents/`. It never runs in reverse.
|
||||
- `showcase/scripts/generate-starters.ts` composes a template frontend plus the showcase package into a self-contained starter under `showcase/starters/<slug>/`.
|
||||
- Do not hand-edit agent code inside `showcase/packages/<slug>/src/agents/` if the package has a Dojo counterpart — fix it upstream in `examples/integrations/<name>/` and re-run the migration script.
|
||||
|
||||
### Born-in-Showcase Packages (no Dojo counterpart)
|
||||
|
||||
Five packages exist only in showcase and have no `examples/integrations/<name>/` sibling:
|
||||
|
||||
- `ag2`
|
||||
- `claude-sdk-python`
|
||||
- `claude-sdk-typescript`
|
||||
- `langroid`
|
||||
- `spring-ai`
|
||||
|
||||
These are authored directly in `showcase/packages/<slug>/` and are **exempt from the pin-to-Dojo rule** — there is no Dojo to pin to. They still must pin exact versions (see "Dependency Pinning"), but the reference is whatever the framework's own examples or release notes recommend, not a sibling `examples/integrations/` directory.
|
||||
|
||||
### Slug Aliasing
|
||||
|
||||
Several packages have different names in `examples/integrations/` vs `showcase/packages/`. The aliasing is historical — showcase standardized on shorter, marketing-friendly slugs while the Dojo kept the original framework-canonical names.
|
||||
|
||||
| `showcase/packages/` slug | `examples/integrations/` name | Why different |
|
||||
| ------------------------- | ----------------------------- | --------------------------------------------------------- |
|
||||
| `google-adk` | `adk` | Showcase prefixes with vendor for disambiguation |
|
||||
| `langgraph-typescript` | `langgraph-js` | Showcase prefers full language name (`-typescript`) |
|
||||
| `ms-agent-dotnet` | `ms-agent-framework-dotnet` | Showcase shortens `-framework-` out of the slug |
|
||||
| `ms-agent-python` | `ms-agent-framework-python` | Same — shorter slug in showcase |
|
||||
| `strands` | `strands-python` | Showcase drops the language suffix (no TS variant exists) |
|
||||
|
||||
When running `migrate-integration-examples.ts` or reasoning about drift, remember that the script internally maps these aliases — don't "fix" them by renaming one side.
|
||||
|
||||
---
|
||||
|
||||
## B. External Setup (after the package is ready)
|
||||
|
||||
### 1. Railway Service
|
||||
@@ -130,6 +181,39 @@ One per declared feature. Each demo must:
|
||||
|
||||
---
|
||||
|
||||
## LangGraph: Prebuilt vs Node-Based
|
||||
|
||||
LangGraph supports two agent authoring styles, and showcase uses both. When touching a LangGraph package — or adding a new one — decide the style explicitly and match the existing sibling's idioms.
|
||||
|
||||
### The Two Styles
|
||||
|
||||
- **Node-based** — hand-rolled `StateGraph` with `addNode(...)`, explicit edges, and custom routing logic. Maximum control; more code to maintain.
|
||||
- **Prebuilt** — `create_react_agent` / `create_agent` helpers that wrap the common ReAct pattern. Minimal code; less flexibility.
|
||||
|
||||
### Current Showcase State
|
||||
|
||||
| Package | Style | Evidence |
|
||||
| ---------------------------------------- | ---------- | ----------------------------------------------------- |
|
||||
| `showcase/packages/langgraph-python` | Prebuilt | `create_react_agent` in `src/agents/main.py:53` |
|
||||
| `showcase/packages/langgraph-fastapi` | Prebuilt | `create_react_agent` in `src/agents/src/agent.py:166` |
|
||||
| `showcase/packages/langgraph-typescript` | Node-based | `StateGraph` in `src/agent/graph.ts:271` |
|
||||
|
||||
### Dojo Coverage Gap
|
||||
|
||||
The `ag-ui/apps/dojo/` e2e tests exclusively exercise **node-based** graphs. This means prebuilt-agent coverage is thin in the Dojo even though two of the three LangGraph packages users clone from showcase are prebuilt.
|
||||
|
||||
Cross-reference the action inventory for the full breakdown of which AG-UI features are exercised where: <https://www.notion.so/3443aa38185281b5a1dfc6e0890264e1>.
|
||||
|
||||
### Guidance
|
||||
|
||||
- **When adding a new LangGraph-based package**, decide the authoring style explicitly and match the idioms of the corresponding showcase sibling (Python → prebuilt, TypeScript → node-based) unless you have a concrete reason to diverge.
|
||||
- If you do diverge, document why in the package's README and add an entry to the table above.
|
||||
- Do not silently convert a package between styles — it's a public API change for anyone who cloned the starter.
|
||||
|
||||
This distinction only applies to LangGraph today. Other frameworks (CrewAI, Mastra, etc.) have their own framework-specific authoring idioms — out of scope for this section.
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference: Common Gotchas
|
||||
|
||||
| Gotcha | Fix |
|
||||
|
||||
+33
-32
@@ -4,43 +4,43 @@ This matrix tracks what testing exists for each demo and the Sales Dashboard sta
|
||||
|
||||
**Legend:**
|
||||
|
||||
- ✅ Covered -- tests exist and verify this demo
|
||||
- ⚠️ Partial -- some coverage exists but gaps remain
|
||||
- ❌ None -- no tests exist for this demo
|
||||
- 🔧 Needs aimock -- tests exist but require aimock fixtures that are missing or incomplete
|
||||
- PASS Covered -- tests exist and verify this demo
|
||||
- WARN Partial -- some coverage exists but gaps remain
|
||||
- FAIL None -- no tests exist for this demo
|
||||
- STUB Needs aimock -- tests exist but require aimock fixtures that are missing or incomplete
|
||||
|
||||
## Demo Coverage
|
||||
|
||||
| Demo | Manual QA | Vitest Unit | Playwright E2E (smoke) | Playwright E2E (interaction) | Per-Package E2E | Aimock Fixtures | CI Auto |
|
||||
| ---------------------------- | ------------------------------- | ----------- | ---------------------- | ---------------------------- | ---------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------------- |
|
||||
| **Agentic Chat** | ✅ 17 packages | ❌ | ✅ load + suggestions | ⚠️ suggestion click only | ✅ weather card, background change, multi-turn | ⚠️ `background`, `weather` matches | ⚠️ validate only (no Playwright in CI by default) |
|
||||
| **Human in the Loop** | ✅ 17 packages | ❌ | ✅ load + suggestions | ⚠️ suggestion click only | ✅ step selector, approve/reject | ⚠️ `plan`/`steps`/`mars` matches (text only, no interrupt) | ⚠️ validate only |
|
||||
| **Tool Rendering** | ✅ 17 packages | ❌ | ✅ load + suggestions | ⚠️ suggestion click only | ✅ WeatherCard with stats grid | ⚠️ `weather` match (tool call) | ⚠️ validate only |
|
||||
| **Gen UI (Tool-Based)** | ✅ 17 packages | ❌ | ❌ | ❌ | ✅ sidebar, haiku card, pie/bar chart | ❌ no haiku-specific fixture | ⚠️ validate only |
|
||||
| **Gen UI (Agent)** | ✅ 1 package (langgraph-python) | ❌ | ❌ | ❌ | ✅ task progress tracker, progress bar | ❌ no gen-ui-agent fixture | ⚠️ validate only |
|
||||
| **Shared State (Read)** | ✅ 1 package (langgraph-python) | ❌ | ❌ | ❌ | ✅ recipe card, sidebar, pipeline | ❌ no shared-state fixture | ⚠️ validate only |
|
||||
| **Shared State (Write)** | ✅ 1 package (langgraph-python) | ❌ | ❌ | ❌ | ✅ pipeline, deal CRUD, agent state writes | ❌ no shared-state fixture | ⚠️ validate only |
|
||||
| **Shared State (Streaming)** | ✅ 1 package (langgraph-python) | ❌ | ❌ | ❌ | ✅ document editor, confirm/reject changes | ❌ no streaming fixture | ⚠️ validate only |
|
||||
| **Sub-Agents** | ✅ 1 package (langgraph-python) | ❌ | ❌ | ❌ | ✅ travel planner, agent indicators, sections | ❌ no subagent fixture | ⚠️ validate only |
|
||||
| Demo | Manual QA | Vitest Unit | Playwright E2E (smoke) | Playwright E2E (interaction) | Per-Package E2E | Aimock Fixtures | CI Auto |
|
||||
| ---------------------------- | ----------------------- | ----------- | ----------------------- | ---------------------------- | ------------------------------------------------ | ------------------------------------------------------------ | --------------------------------------------------- |
|
||||
| **Agentic Chat** | PASS 17 packages | FAIL | PASS load + suggestions | WARN suggestion click only | PASS weather card, background change, multi-turn | WARN `background`, `weather` matches | WARN validate only (no Playwright in CI by default) |
|
||||
| **Human in the Loop** | PASS 17 packages | FAIL | PASS load + suggestions | WARN suggestion click only | PASS step selector, approve/reject | WARN `plan`/`steps`/`mars` matches (text only, no interrupt) | WARN validate only |
|
||||
| **Tool Rendering** | PASS 17 packages | FAIL | PASS load + suggestions | WARN suggestion click only | PASS WeatherCard with stats grid | WARN `weather` match (tool call) | WARN validate only |
|
||||
| **Gen UI (Tool-Based)** | PASS 17 packages | FAIL | FAIL | FAIL | PASS sidebar, haiku card, pie/bar chart | FAIL no haiku-specific fixture | WARN validate only |
|
||||
| **Gen UI (Agent)** | PASS 17 packages | FAIL | FAIL | FAIL | PASS task progress tracker, progress bar | FAIL no gen-ui-agent fixture | WARN validate only |
|
||||
| **Shared State (Read)** | PASS 17 packages | FAIL | FAIL | FAIL | PASS recipe card, sidebar, pipeline | FAIL no shared-state fixture | WARN validate only |
|
||||
| **Shared State (Write)** | PASS 17 packages (stub) | FAIL | FAIL | FAIL | PASS pipeline, deal CRUD, agent state writes | FAIL no shared-state fixture | WARN validate only |
|
||||
| **Shared State (Streaming)** | PASS 17 packages (stub) | FAIL | FAIL | FAIL | PASS document editor, confirm/reject changes | FAIL no streaming fixture | WARN validate only |
|
||||
| **Sub-Agents** | PASS 17 packages (stub) | FAIL | FAIL | FAIL | PASS travel planner, agent indicators, sections | FAIL no subagent fixture | WARN validate only |
|
||||
|
||||
## Starter Hero Coverage
|
||||
|
||||
| Feature | Manual QA | Vitest Unit | Playwright E2E (smoke) | Playwright E2E (interaction) | Aimock Fixtures | CI Auto |
|
||||
| ------------------------------- | --------- | -------------------------------------------------------------- | ------------------------------------- | --------------------------------------------- | -------------------------------- | ----------------------------------------- |
|
||||
| **Sales Dashboard (page load)** | ❌ | ✅ generate-starters tests (17 starters exist, file structure) | ✅ header, 4 renderer pills | ✅ pill switching, content verification | ⚠️ `sales`/`todo`/`deal` matches | ⚠️ validate + aimock-e2e (manual trigger) |
|
||||
| **Renderer Selector** | ❌ | ❌ | ✅ 4 pills visible, default selection | ✅ mutual exclusion, content changes per mode | ❌ | ⚠️ validate only |
|
||||
| **Tool-Based mode** | ❌ | ❌ | ✅ pipeline heading, KPI cards | ✅ Add a deal, multiple deals, empty state | ⚠️ `sales`/`todo` matches | ⚠️ validate only |
|
||||
| **A2UI Catalog mode** | ❌ | ❌ | ✅ same pipeline content | ❌ | ❌ | ⚠️ validate only |
|
||||
| **json-render mode** | ❌ | ❌ | ✅ fallback note + pipeline | ❌ | ❌ | ⚠️ validate only |
|
||||
| **HashBrown mode** | ❌ | ❌ | ✅ pipeline content | ❌ | ❌ | ⚠️ validate only |
|
||||
| Feature | Manual QA | Vitest Unit | Playwright E2E (smoke) | Playwright E2E (interaction) | Aimock Fixtures | CI Auto |
|
||||
| ------------------------------- | --------- | ---------------------------------------------------------------- | --------------------------------------- | ----------------------------------------------- | ---------------------------------- | ------------------------------------------- |
|
||||
| **Sales Dashboard (page load)** | FAIL | PASS generate-starters tests (17 starters exist, file structure) | PASS header, 4 renderer pills | PASS pill switching, content verification | WARN `sales`/`todo`/`deal` matches | WARN validate + aimock-e2e (manual trigger) |
|
||||
| **Renderer Selector** | FAIL | FAIL | PASS 4 pills visible, default selection | PASS mutual exclusion, content changes per mode | FAIL | WARN validate only |
|
||||
| **Tool-Based mode** | FAIL | FAIL | PASS pipeline heading, KPI cards | PASS Add a deal, multiple deals, empty state | WARN `sales`/`todo` matches | WARN validate only |
|
||||
| **A2UI Catalog mode** | FAIL | FAIL | PASS same pipeline content | FAIL | FAIL | WARN validate only |
|
||||
| **json-render mode** | FAIL | FAIL | PASS fallback note + pipeline | FAIL | FAIL | WARN validate only |
|
||||
| **HashBrown mode** | FAIL | FAIL | PASS pipeline content | FAIL | FAIL | WARN validate only |
|
||||
|
||||
## Test Infrastructure Details
|
||||
|
||||
### Manual QA Checklists (`showcase/packages/*/qa/*.md`)
|
||||
|
||||
- 73 files across 17 packages
|
||||
- All 17 packages have checklists for: agentic-chat, hitl, tool-rendering, gen-ui-tool-based
|
||||
- Only langgraph-python has checklists for: gen-ui-agent, shared-state-read, shared-state-write, shared-state-streaming, subagents
|
||||
- 153 files across 17 packages (17 × 9 demos)
|
||||
- All 17 packages have checklists for all 9 demos: agentic-chat, hitl-in-chat, tool-rendering, gen-ui-tool-based, gen-ui-agent, shared-state-read, shared-state-write, shared-state-streaming, subagents
|
||||
- Authored across all packages: agentic-chat, hitl-in-chat, tool-rendering, gen-ui-tool-based, gen-ui-agent, shared-state-read. Stub-only across all 17 packages (not yet authored): shared-state-write, shared-state-streaming, subagents (3 demos × 17 packages = 51 stub files).
|
||||
|
||||
### Vitest Unit Tests (`showcase/scripts/__tests__/*.test.ts`)
|
||||
|
||||
@@ -55,15 +55,16 @@ This matrix tracks what testing exists for each demo and the Sales Dashboard sta
|
||||
### Playwright E2E -- Shared (`showcase/scripts/__tests__/e2e/`)
|
||||
|
||||
- `starter-e2e.spec.ts` -- Sales Dashboard starter (15 tests: pills, modes, content switching, add deals)
|
||||
- `demo-e2e.spec.ts` -- agentic-chat, hitl, tool-rendering only (9 tests: load, suggestions, click)
|
||||
- `demo-e2e.spec.ts` -- agentic-chat, hitl-in-chat, tool-rendering only (9 tests: load, suggestions, click)
|
||||
- `screenshots.spec.ts` -- screenshot capture
|
||||
- **Gap:** No shared E2E tests for gen-ui-tool-based, gen-ui-agent, shared-state-\*, subagents
|
||||
|
||||
### Playwright E2E -- Per-Package (`showcase/packages/langgraph-python/tests/e2e/`)
|
||||
### Playwright E2E -- Per-Package (`showcase/packages/*/tests/e2e/`)
|
||||
|
||||
- 10 spec files covering all 9 demos + renderer-selector
|
||||
- Tests require a running dev server and (for interaction tests) an agent backend or aimock
|
||||
- **Gap:** These tests only exist for langgraph-python, not other 16 packages
|
||||
- Every one of the 17 packages ships 9 per-demo spec files (agentic-chat, hitl-in-chat, tool-rendering, gen-ui-tool-based, gen-ui-agent, shared-state-read, shared-state-write, shared-state-streaming, subagents).
|
||||
- `langgraph-python` additionally ships a 10th spec (`renderer-selector.spec.ts`) covering the Sales Dashboard renderer-selector flow; no other package has this spec.
|
||||
- Tests require a running dev server and (for interaction tests) an agent backend or aimock, and are **not** wired into default CI — they run on demand locally or via the manual `test_e2e-showcase-on-demand.yml` trigger.
|
||||
- **Gap:** The renderer-selector per-package coverage is langgraph-python-only; replicating it to at least one TypeScript package would verify cross-framework parity.
|
||||
|
||||
### Aimock Fixtures (`showcase/aimock/`)
|
||||
|
||||
@@ -74,7 +75,7 @@ This matrix tracks what testing exists for each demo and the Sales Dashboard sta
|
||||
### CI Workflows (`.github/workflows/showcase_*.yml`)
|
||||
|
||||
- `showcase_validate.yml` -- runs `npx vitest run` on PR (unit tests only)
|
||||
- `showcase_aimock-e2e.yml` -- runs aimock-backed Playwright E2E, **manual trigger only** (`/test-aimock` comment or workflow_dispatch)
|
||||
- `test_e2e-showcase-on-demand.yml` -- runs aimock-backed Playwright E2E, **manual trigger only** (`/test-aimock` comment or workflow_dispatch)
|
||||
- `showcase_drift-detection.yml` -- template drift detection
|
||||
- `showcase_template-drift.yml` -- template synchronization
|
||||
- `showcase_deploy.yml` -- deployment pipeline
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Test-Gating Matrix
|
||||
|
||||
This matrix documents which CI workflows fire on which triggers, what they test, and whether they gate merges. Companion to [`QA-COVERAGE.md`](./QA-COVERAGE.md) -- that document tracks per-demo coverage; this one tracks per-workflow gating.
|
||||
|
||||
Scope: all testing-related workflows (unit, integration, e2e, smoke) across the monorepo. Data read directly from `.github/workflows/*.yml` on the current branch.
|
||||
|
||||
## Matrix
|
||||
|
||||
| Workflow file | Name (CI UI) | Trigger | Path filter | Required? | What it tests |
|
||||
| --------------------------------------------------- | --------------------------------- | ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | --------- | ------------------------------------------------------------------------------------------------ |
|
||||
| `.github/workflows/test_unit.yml` | test / unit | push (main), pull_request (main), workflow_dispatch | paths-ignore: `docs/**`, `README.md`, `examples/**` | No | Vitest unit suite across Node 20/22/24 for all TS packages |
|
||||
| `.github/workflows/test_unit-python-sdk.yml` | test / unit / python-sdk | push (main), pull_request (main) | `sdk-python/**`, this workflow | No | pytest against `sdk-python/` under Python 3.12 + Poetry |
|
||||
| `.github/workflows/test_integration-runtime.yml` | test / integration / runtime | push (main), pull_request (main), workflow_dispatch | `packages/runtime/**`, this workflow | No | Runtime server integration tests (Node, possibly others) |
|
||||
| `.github/workflows/test_integration-docs.yml` | test / integration / docs | push (main), pull_request | `docs/**` | No | Extracts code blocks from docs, runs them against aimock (model-name + doc-test) |
|
||||
| `.github/workflows/test_e2e-dojo.yml` | test / e2e / dojo | push (main), pull_request (main), workflow_dispatch | `packages/**`, `sdk-python/**`, this workflow, `.changeset` | No | ag-ui dojo end-to-end matrix on Depot runners |
|
||||
| `.github/workflows/test_e2e-legacy-v1.yml` | test / e2e / legacy-v1 | push (main), pull_request (main), workflow_dispatch | `examples/**`, this workflow, `.changeset` | No | Legacy v1.x examples (form-filling, travel, research-canvas, chat-with-your-data, state-machine) |
|
||||
| `.github/workflows/showcase_validate.yml` | Showcase: Validate | push (main), pull_request | `showcase/**`, `examples/integrations/**/fixtures/**`, `scripts/doc-tests/fixtures/**` | No | Build-pipeline Vitest + manifest/registry validation + shell build |
|
||||
| `.github/workflows/test_e2e-showcase-on-demand.yml` | test / e2e / showcase / on-demand | issue_comment (`/test-aimock`), workflow_dispatch | n/a (comment-gated) | No | aimock-backed Playwright E2E on demand per-package |
|
||||
| `.github/workflows/test_smoke-starter.yml` | test / smoke / starter | schedule (`0 */6 * * *`), workflow_run (publish / release), pull_request, workflow_dispatch | `examples/integrations/**`, this workflow | No | Docker-compose smoke for 12 starter integrations (build + curl) |
|
||||
| `.github/workflows/test_smoke-starter-deployed.yml` | test / smoke / starter-deployed | schedule (`0 */6 * * *`), workflow_run (Showcase: Build & Deploy), workflow_dispatch | n/a (scheduled / post-deploy) | No | Playwright E2E against live deployed starter URLs (@starter-health/-agent/-chat) |
|
||||
|
||||
## Which tests run on a typical PR?
|
||||
|
||||
- `packages/**` (runtime/SDK): `test / unit`, `test / integration`, `test / e2e / dojo`, `static / quality`, `static / check binaries`, plus `static / danger` if `packages/sdk-js/src/langgraph.ts` is touched.
|
||||
- `sdk-python/**`: `test / unit / python-sdk`, `test / e2e / dojo`, `static / check binaries`, plus `static / danger` if `copilotkit/langgraph_agent.py` is touched. `test / unit` also fires (paths-ignore does not exclude sdk-python).
|
||||
- `showcase/**`: `Showcase: Validate`, `test / unit` (paths-ignore does not exclude showcase), `static / quality`, `static / check binaries`. No Playwright E2E runs automatically -- comment `/test-aimock` on the PR to trigger `test / e2e / showcase / on-demand`.
|
||||
- `examples/**` (legacy v1.x): `test / e2e / legacy-v1`, `static / check binaries`. `test / unit` is excluded via paths-ignore.
|
||||
- `examples/integrations/**` (starters): `test / smoke / starter` (Docker), `Showcase: Validate` (for fixtures only), `static / check binaries`.
|
||||
- `docs/**`: `test / integration / docs` only. `test / unit` and `static / quality` are excluded via paths-ignore.
|
||||
- `.github/workflows/**`: each workflow that lists its own path in its trigger runs (most do). No single "workflows changed" catch-all.
|
||||
|
||||
## Required status checks
|
||||
|
||||
None of the workflows above are enforced as required status checks. The active `PROTECT_OUR_MAIN` ruleset on `main` requires zero status contexts -- merges are gated only by review approval, not by CI outcome.
|
||||
|
||||
The legacy classic branch protection `required_status_checks.contexts` array contains stale entries (e.g. `test / unit`, `Showcase: Validate`) that appear in GitHub's API responses but are not evaluated by the active ruleset. If you see a CI workflow marked as "required" in an older doc or script, treat that as ghost data: nothing in GitHub's current enforcement path consumes it.
|
||||
|
||||
Practical consequence: a red CI run does not block merge. Reviewers must eyeball `gh pr checks` before approving.
|
||||
|
||||
## Footnotes
|
||||
|
||||
- `test / unit` matrix is Node 20/22/24; the other workflows pin a single Node version each (22 most common).
|
||||
- `test / e2e / dojo` uses Depot runners (`depot-ubuntu-24.04`); all others use standard GitHub runners.
|
||||
- `test / smoke / starter` and `test / smoke / starter-deployed` both run every 6h; the former validates Docker-build integrity of `examples/integrations/`, the latter validates the deployed Railway services.
|
||||
- `workflow_run` triggers fire after another workflow completes -- they do not gate the triggering PR, they run post-merge.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Showcase aimock
|
||||
|
||||
Deterministic LLM fixture server for showcase E2E testing. Replaces real LLM API calls (OpenAI, Anthropic, Gemini) with pre-recorded responses so Playwright tests can run PR-gated in CI without API keys and without rate limits or non-determinism.
|
||||
|
||||
The image built from this directory is [`ghcr.io/copilotkit/showcase-aimock`](https://github.com/orgs/CopilotKit/packages/container/package/showcase-aimock) and is deployed to Railway as a sidecar alongside the showcase packages.
|
||||
|
||||
## What aimock is
|
||||
|
||||
aimock ([`@copilotkit/aimock`](https://www.npmjs.com/package/@copilotkit/aimock)) is a general-purpose LLM mock server. It speaks the OpenAI, Anthropic, and Gemini REST shapes (including SSE streaming), loads fixtures from disk at startup, and responds to incoming chat completions by matching the user's message text against fixture `match` criteria.
|
||||
|
||||
The showcase deployment runs aimock in proxy mode — `--proxy-only` with real upstream URLs configured for each provider. Unmatched requests are forwarded to the real API; matched requests short-circuit with the fixture response. This makes the sidecar safe to deploy as a general-purpose smoke-test aid: tests that hit fixture-matched prompts get deterministic responses, and anything else just falls through.
|
||||
|
||||
## Fixtures in this directory
|
||||
|
||||
- **`feature-parity.json`** — 35+ fixtures covering the nine showcase demos across 17 packages: agentic chat (weather, backgrounds, themes), tool rendering (pie/bar charts, weather cards), HITL (plans, steps, approvals), Sales Dashboard (deals, pipelines, todos), and assorted meeting/flight/greeting prompts. Consumed by the per-package `test_e2e-showcase-on-demand` Playwright suites and by the Dockerfile-baked image.
|
||||
- **`smoke.json`** — a single minimal fixture (`userMessage: "Respond with exactly: OK"` → `content: "OK"`). Used by `/api/smoke` endpoints in each package to verify the aimock → package → UI round-trip without depending on a real agent.
|
||||
- **`Dockerfile`** — pins the upstream `ghcr.io/copilotkit/aimock:latest`, copies both fixtures into `/fixtures/`, and boots with `--proxy-only --validate-on-load` plus the three provider upstream URLs.
|
||||
|
||||
Fixture match semantics: `userMessage` is a substring match against the last user turn. First fixture to match wins, so more specific prompts should appear before more generic ones (see the `"Based on the following context, write a concise"` entry that precedes the generic `report` / `plan` fixtures to protect CrewAI's startup probe).
|
||||
|
||||
## Sync policy
|
||||
|
||||
**Fixtures are hand-maintained.** There is no automated capture, no scheduled re-recording, and no drift-detection job that compares fixture responses against what a real LLM would say. The authoritative behavior is whatever is checked in.
|
||||
|
||||
The safety net is two-layered load-time validation, not behavioral verification:
|
||||
|
||||
1. **Load-time schema validation** (`--validate-on-load` in the `Dockerfile` and in every test entrypoint that boots aimock) — the container refuses to start if any fixture uses an unrecognized response key (e.g. `text` instead of `content`). See [#3973](https://github.com/CopilotKit/CopilotKit/pull/3973).
|
||||
2. **CI schema validation** (`showcase/scripts/__tests__/aimock-fixtures.test.ts`) — the `showcase_validate` workflow runs `loadFixtureFile` + `validateFixtures` from `@copilotkit/aimock` against every `showcase/aimock/*.json` on every PR. A broken fixture fails the PR before merge.
|
||||
|
||||
Neither layer catches **behavioral drift** — if a package's agent code changes what it asks the LLM (new prompt, new tool, renamed tool), the existing fixture keeps matching and keeps returning the old response. The test either keeps passing (wrong assertion) or fails at the UI-assertion layer (missing tool call, missing text), and a human has to trace it back to the fixture.
|
||||
|
||||
## Adding or updating a fixture
|
||||
|
||||
The process is manual. There is no CLI for this directory specifically — aimock's upstream `--record` mode can proxy real API calls and write fixtures, but the showcase repo does not wire it up and does not commit recorded fixtures.
|
||||
|
||||
1. Identify the user prompt your test issues and decide what response you need (plain text, a tool call, an error).
|
||||
2. Add an entry to `feature-parity.json` under `fixtures`. Keep more specific `userMessage` matches above more generic ones. Valid response keys: `content`, `toolCalls`, `error`, `embedding`.
|
||||
3. Run the fixture-validation suite locally:
|
||||
```
|
||||
pnpm --filter @copilotkit/showcase-scripts test aimock-fixtures
|
||||
```
|
||||
4. Run the per-package E2E against the new fixture:
|
||||
```
|
||||
./showcase/scripts/run-e2e-with-aimock.sh <slug> [test-filter]
|
||||
```
|
||||
5. Ship it. The `showcase_deploy` workflow picks up `showcase/aimock/**` changes and rebuilds the Railway image.
|
||||
|
||||
When a package's agent code changes in a way that changes its LLM calls, the person making the change is responsible for updating the corresponding fixture. There is no automation to remind you.
|
||||
|
||||
## Drift risk
|
||||
|
||||
Drift surfaces as **flaky or silently-wrong E2E tests**, not as a dedicated signal. Symptoms and how to respond:
|
||||
|
||||
- **Playwright assertion fails** on a UI element that depends on a tool call (`WeatherCard` missing, chart not rendering) → the agent is now calling a differently-named tool than the fixture has; update the fixture's `toolCalls[].name` / `arguments`.
|
||||
- **Assertion on assistant text fails** → the agent's prompt changed; either update the fixture's `match.userMessage` to the new prompt substring or update the fixture's `content`.
|
||||
- **`smoke.json` healthcheck fails** against a deployed package (`/api/smoke` returns non-OK) → either the package's smoke route changed or aimock is down; check the Railway service and the smoke-monitor workflow.
|
||||
- **Container fails to start post-deploy** → load-time validation caught a broken fixture; CI should have caught it first, investigate why it didn't.
|
||||
|
||||
There is no scheduled drift-detection job that compares fixture responses against live LLM output. If this becomes a problem, the path forward is to wire aimock's `--record` mode into a periodic workflow that re-captures against real providers and diffs against checked-in fixtures — but that's not built today.
|
||||
|
||||
## Related workflows
|
||||
|
||||
- **`test_e2e-showcase-on-demand.yml`** (historically `showcase_aimock-e2e.yml`) — triggered by `/test-aimock <slug>` PR comments or `workflow_dispatch`. Installs `@copilotkit/aimock@latest`, boots it with `feature-parity.json`, spins up the target package's dev server against `OPENAI_BASE_URL=http://localhost:4010/v1`, and runs the package's Playwright suite. Posts pass/fail back to the PR.
|
||||
- **`showcase_validate.yml`** — runs fixture schema validation (`aimock-fixtures.test.ts`) on every PR that touches `showcase/**`.
|
||||
- **`showcase_deploy.yml`** — rebuilds and redeploys the `showcase-aimock` Railway service whenever `showcase/aimock/**` changes.
|
||||
- **`showcase_smoke-monitor.yml`** — every 15 minutes, polls `/api/smoke` on all deployed showcase packages. Those smoke endpoints internally hit aimock's `smoke.json` fixture to verify the full stack.
|
||||
@@ -0,0 +1,7 @@
|
||||
dist/
|
||||
node_modules/
|
||||
.env
|
||||
.env.local
|
||||
coverage/
|
||||
*.log
|
||||
*.tsbuildinfo
|
||||
@@ -0,0 +1,120 @@
|
||||
FROM node:22-alpine AS build
|
||||
WORKDIR /repo
|
||||
|
||||
RUN corepack enable
|
||||
|
||||
COPY pnpm-workspace.yaml package.json pnpm-lock.yaml .pnpmfile.cjs ./
|
||||
COPY showcase/ops/package.json ./showcase/ops/package.json
|
||||
# Copy every workspace package.json manifest (but NOT source / node_modules).
|
||||
# The version-drift probe's pnpm-packages discovery source parses these at
|
||||
# runtime. Doing this in the build stage (rather than COPY-ing packages/
|
||||
# directly into the runtime image from the host build context) keeps the
|
||||
# runtime stage hermetic and consistent — the final image is always
|
||||
# assembled strictly from build-stage artifacts.
|
||||
COPY packages ./packages-src-tmp
|
||||
RUN mkdir -p ./packages && \
|
||||
cd packages-src-tmp && \
|
||||
find . -maxdepth 2 -name package.json -not -path '*/node_modules/*' | \
|
||||
while read f; do \
|
||||
dir="../packages/$(dirname "$f")"; \
|
||||
mkdir -p "$dir" && cp "$f" "$dir/package.json"; \
|
||||
done && \
|
||||
cd .. && rm -rf packages-src-tmp
|
||||
|
||||
# `--ignore-scripts` skips the root `prepare` hook (lefthook install), which
|
||||
# requires `git` and is meaningless inside the build image. Deps themselves
|
||||
# don't rely on postinstall scripts in showcase-ops.
|
||||
RUN pnpm install --frozen-lockfile --ignore-scripts --filter @copilotkit/showcase-ops...
|
||||
|
||||
COPY showcase/ops ./showcase/ops
|
||||
|
||||
RUN pnpm --filter @copilotkit/showcase-ops build
|
||||
|
||||
# `pnpm deploy` materializes a standalone, hoisted node_modules with only
|
||||
# production deps into /deploy — no symlinks into /repo/node_modules/.pnpm.
|
||||
# Without this, the runtime stage would copy a pnpm-hoisted tree whose
|
||||
# symlinks point into paths that don't exist in the final image.
|
||||
# `--legacy` keeps pnpm v10+'s `deploy` usable without requiring
|
||||
# `inject-workspace-packages=true` across the repo — we don't use
|
||||
# injected workspace deps in showcase-ops.
|
||||
# Verified on pnpm 10.13.x — the `--legacy` flag semantics shifted in the
|
||||
# 10.x line (pre-10.x `deploy` was itself the legacy behavior and the flag
|
||||
# was a no-op). Pin the comment to the repo's committed pnpm version so
|
||||
# future upgrades surface the dependency.
|
||||
RUN pnpm --filter @copilotkit/showcase-ops --prod --legacy --ignore-scripts deploy /deploy
|
||||
|
||||
# Runtime stage: Debian-slim (not Alpine) because the e2e-smoke probe
|
||||
# driver launches chromium in-process via `playwright`. Playwright's
|
||||
# `install --with-deps` only supports apt-based distros — Alpine ships
|
||||
# musl libc, and the upstream chromium binaries Playwright downloads are
|
||||
# glibc-linked. Switching the runtime image to `node:22-bookworm-slim`
|
||||
# lets `playwright install --with-deps chromium` succeed without a
|
||||
# custom apk dance. Build stage stays Alpine (just compiles TS and
|
||||
# prunes node_modules — no browser needed there).
|
||||
FROM node:22-bookworm-slim
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
# Playwright cache lives outside /home/node so `chown` below doesn't
|
||||
# have to recurse over the ~300MB chromium tree on every build. Setting
|
||||
# PLAYWRIGHT_BROWSERS_PATH at this stage pins the install target; the
|
||||
# orchestrator reads the same env at runtime via `playwright`'s own
|
||||
# default resolution logic.
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
|
||||
COPY --from=build /repo/showcase/ops/dist ./dist
|
||||
COPY --from=build /deploy/node_modules ./node_modules
|
||||
COPY --from=build /deploy/package.json ./package.json
|
||||
COPY --from=build /repo/showcase/ops/config ./config
|
||||
# e2e-smoke probe: install chromium + its system deps. `--with-deps`
|
||||
# pulls in libnss3, libatk, libxkbcommon, libdrm, etc. via apt. Runs as
|
||||
# root because apt-get needs root. Call `playwright/cli.js` directly
|
||||
# (not via `npx playwright`) — pnpm's deploy output materialises a
|
||||
# hoisted node_modules but doesn't always produce a `.bin/playwright`
|
||||
# shim, so `npx playwright` would resolve to "not found".
|
||||
RUN node ./node_modules/playwright/cli.js install --with-deps chromium \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# version-drift probe: the pnpm-packages discovery source reads
|
||||
# pnpm-workspace.yaml + each workspace package manifest at probe-tick time.
|
||||
# Copy them into /app so the source resolves with rootDir=/app (the runtime
|
||||
# WORKDIR) without any further configuration. Only manifest files are
|
||||
# copied — node_modules and source trees stay out of the runtime image.
|
||||
# packages/ is the only workspace prefix version-drift.yml filters to today
|
||||
# (filter.pathPrefix: "packages/"); adding examples/ or sdk-python here
|
||||
# would be safe — the probe's filter is the authoritative gate — but we
|
||||
# keep the image lean until another probe config actually needs those trees.
|
||||
COPY --from=build /repo/pnpm-workspace.yaml ./pnpm-workspace.yaml
|
||||
COPY --from=build /repo/packages ./packages
|
||||
# chown /app AND the playwright browser cache so the runtime user can
|
||||
# read the chromium tree at launch. Orchestrator today writes only to
|
||||
# mounted volumes (PB data dir, S3 backup buffer), so the /app chown
|
||||
# is defensive hygiene — running as node with a root-owned /app would
|
||||
# silently break any future feature that wants to write a pid/lock
|
||||
# file next to the binary.
|
||||
RUN chown -R node:node /app /ms-playwright
|
||||
USER node
|
||||
EXPOSE 8080
|
||||
# Runtime healthcheck. Railway provides its own health check on the
|
||||
# `health_path` in ALL_SERVICES, so this is primarily for parity with
|
||||
# `docker run` locally (and CI integration harnesses that use
|
||||
# `docker inspect` to gate test starts on container health). 30s
|
||||
# start-period gives Node + config-load time before the first probe.
|
||||
#
|
||||
# NODE HEALTHCHECK: previously `wget -q --spider`, which was busybox-wget
|
||||
# on Alpine. After the base-image move to Debian-slim (needed for
|
||||
# Playwright chromium), we switched to a Node one-liner because Debian
|
||||
# slim doesn't ship wget / curl by default and adding them just for
|
||||
# healthcheck would bloat the image unnecessarily. Node's http module
|
||||
# gives us the same semantic: non-2xx status → exit 1 → Docker/Railway
|
||||
# mark unhealthy.
|
||||
#
|
||||
# 503 at /health (intentional response from orchestrator.ts when the
|
||||
# rule-loader / probe pipeline is in a broken state) still causes the
|
||||
# container to be marked unhealthy and restarted after 3 retries. That
|
||||
# restart loop is THE INTENDED OUTCOME during sustained-503 windows:
|
||||
# if /health is reporting broken for 90s straight, restarting the
|
||||
# orchestrator is the right remediation (faster than waiting for a
|
||||
# human to notice). Do not "soften" this by treating 503 as healthy —
|
||||
# the 503 is specifically how orchestrator.ts communicates "I cannot
|
||||
# serve" to its supervisor.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
|
||||
CMD node -e "require('http').get('http://127.0.0.1:8080/health',r=>process.exit(r.statusCode>=200&&r.statusCode<300?0:1)).on('error',()=>process.exit(1))" || exit 1
|
||||
CMD ["node", "dist/orchestrator.js"]
|
||||
@@ -0,0 +1,494 @@
|
||||
# showcase-ops
|
||||
|
||||
In-cluster observability service for the showcase fleet. Runs on Railway, receives signed webhooks from GitHub Actions, executes cron-driven probes, persists state to PocketBase, classifies state transitions, and delivers alerts to Slack.
|
||||
|
||||
Replaces four legacy GitHub Actions cron workflows (`showcase_smoke-monitor`, `showcase_drift-detection`, `showcase_drift-report`, `showcase_redirect-report`) with a single long-lived process that can hold transition state, dedupe, rate-limit, and render rich templates without each tick re-reading GitHub artifacts.
|
||||
|
||||
---
|
||||
|
||||
# Part 1 — Operate it
|
||||
|
||||
This section is for everyone who needs to add an alert rule, rotate a secret, or figure out why something did or didn't fire. You do not need the source tree checked out — only Railway access and the repo's `config/alerts/` YAMLs.
|
||||
|
||||
## 1.1 Inspect a running instance
|
||||
|
||||
Production URL: `https://showcase-ops-production.up.railway.app`
|
||||
|
||||
- **`GET /health`** — JSON: `{status, pb, loop, rules, schedulerJobs}`. `pb:"ok"` means PocketBase reachable; `loop:"ok"` means the scheduler tick has advanced in the last interval; `rules` is the count of successfully compiled YAMLs; `schedulerJobs` is the count of registered cron entries.
|
||||
- **`GET /metrics`** — Prometheus exposition. Key counters:
|
||||
- `showcase_ops_probe_runs{dimension=...}` — per-dimension probe executions
|
||||
- `showcase_ops_alert_matches{rule=...}` — rule match count
|
||||
- `showcase_ops_alert_sends{target=...}` — successful target deliveries
|
||||
- `showcase_ops_rule_reloads` — increments on SIGHUP / file watcher reload
|
||||
- `showcase_ops_webhook_rejections{reason=...}` — HMAC and payload-validation failures; `reason` is one of `stale`, `invalid-signature-format`, `invalid-signature`, `missing-signature`, `missing-timestamp`, `invalid-payload`, `unknown`
|
||||
- **`POST /webhooks/deploy`** — HMAC-signed webhook ingest for `deploy.result` events. Canonical payload: `METHOD|PATH|TS|sha256(body)` with `sha256=<hex>` signature in `X-Ops-Signature`. Path must be the route constant (`/webhooks/deploy`), not `c.req.path`. 300s skew tolerance.
|
||||
- **Logs** — `railway logs --service showcase-ops` or the Railway dashboard. All lines are structured JSON: `{level, msg, ts, ...fields}`. Grep targets: `alert-engine.bootstrap-suppress` (gate suppressed a send), `writer.failed` (PB persist failed, always re-emits on bus), `suppress.eval-failed` (a suppress DSL expression threw), `rules.reload.failed` (load-time validation rejected a YAML).
|
||||
|
||||
## 1.2 Environment variables
|
||||
|
||||
All read at boot unless marked otherwise. See `showcase/ops/src/orchestrator.ts`.
|
||||
|
||||
**Required in production:**
|
||||
|
||||
| Var | Meaning |
|
||||
| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `POCKETBASE_URL` | Internal PB endpoint (`http://showcase-pocketbase.railway.internal:8090`). Boot refuses to start if unset when `NODE_ENV=production`. |
|
||||
| `POCKETBASE_SUPERUSER_EMAIL` | Admin auth for ops to write status rows. |
|
||||
| `POCKETBASE_SUPERUSER_PASSWORD` | Paired. |
|
||||
| `SHARED_SECRET` | Current HMAC secret for `/webhooks/deploy`. 64-hex recommended. Signer side lives in repo secret `SHOWCASE_OPS_SHARED_SECRET`. |
|
||||
|
||||
**Optional:**
|
||||
|
||||
| Var | Meaning / default |
|
||||
| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `SHARED_SECRET_PREV` | Accepted during rotation. See `docs/rotation-drill.md`. |
|
||||
| `AIMOCK_URL` | Public aimock URL for the aimock-wiring probe to compare against. Probe disables itself if unset. |
|
||||
| `RAILWAY_TOKEN` | Service token with read scope on the `showcase` project. Required by aimock-wiring probe to query service env vars. |
|
||||
| `RAILWAY_PROJECT_ID` | Paired with token. |
|
||||
| `RAILWAY_ENVIRONMENT_ID` | Paired with token. |
|
||||
| `DASHBOARD_URL` | Rendered as `{{env.dashboardUrl}}` in Slack link markup. Default `https://dashboard.showcase.copilotkit.ai`. |
|
||||
| `REPO` | Rendered as `{{env.repo}}`. Default `CopilotKit/CopilotKit`. |
|
||||
| `S3_BACKUP_BUCKET` | Enables the nightly PB-backup cron. Bucket must be writable via default AWS credential chain. Init failure emits `internal.backup.init-failed` on the bus but does not block boot. |
|
||||
| `AWS_REGION` | Default `us-east-1`. |
|
||||
| `LOG_LEVEL` | `debug` / `info` / `warn` / `error`. Default `info`. Mutable at runtime via SIGHUP after editing `LOG_LEVEL` env. |
|
||||
| `PORT` | HTTP listen port. Default `8080`. |
|
||||
| `SLACK_WEBHOOK_<ALIAS>` | One env var per webhook alias referenced by any rule (`SLACK_WEBHOOK_OSS_ALERTS`, etc.). See §1.4. |
|
||||
|
||||
**Caller-side (GitHub Actions repo secrets, not this service):**
|
||||
|
||||
| Secret | Used by |
|
||||
| ---------------------------- | ------------------------------------------------- |
|
||||
| `SHOWCASE_OPS_URL` | `notify-ops` step in `showcase_deploy.yml`. |
|
||||
| `SHOWCASE_OPS_SHARED_SECRET` | Same — paired with the service's `SHARED_SECRET`. |
|
||||
|
||||
## 1.3 Alert rule YAMLs
|
||||
|
||||
Location: `showcase/ops/config/alerts/*.yml`. The loader picks up every `.yml`/`.yaml` file except `_defaults.yml`, merges defaults in, compiles each rule through a Zod schema + structural validators, and hot-reloads on file changes (`chokidar`) or SIGHUP.
|
||||
|
||||
### File layout
|
||||
|
||||
```
|
||||
config/alerts/
|
||||
├── _defaults.yml # merged into every rule
|
||||
├── aimock-wiring-drift.yml # invariant drift — @oss, weekly cron
|
||||
├── deploy-result.yml # transition rule for deploy webhooks
|
||||
├── e2e-smoke-failure.yml # e2e harness red-tick
|
||||
├── image-drift.yml # GHCR tag vs Railway running image
|
||||
├── pin-drift-weekly.yml # showcase starter pin freshness
|
||||
├── redirect-decommission-monthly.yml # legacy-host redirect stability
|
||||
├── smoke-red-tick.yml # smoke probe transition rule
|
||||
└── version-drift-weekly.yml # showcase package version pins
|
||||
```
|
||||
|
||||
### Rule skeleton
|
||||
|
||||
```yaml
|
||||
id: aimock-wiring-drift
|
||||
name: "aimock-universal invariant drift"
|
||||
owner: "@oss"
|
||||
severity: error # info | warn | error | critical (default warn)
|
||||
|
||||
signal:
|
||||
dimension: aimock_wiring # closed enum — see types/index.ts DIMENSIONS
|
||||
filter: # optional
|
||||
key: "smoke:mastra" # or glob: "smoke:*"
|
||||
dimension: smoke # optional narrowing, must be DIMENSIONS member
|
||||
slug: "mastra" # optional substring/glob on the key's slug part
|
||||
|
||||
triggers: # fires iff any listed trigger resolves true
|
||||
- set_drifted # signal-derived (see deriveSignalFlags)
|
||||
- red_to_green # state-transition
|
||||
- cron_only: # cron expression co-evaluated by the scheduler
|
||||
schedule: "0 8 * * 1"
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts # resolves to env var SLACK_WEBHOOK_OSS_ALERTS
|
||||
|
||||
conditions:
|
||||
guards: [] # optional: rule only fires when signal matches a guard
|
||||
rate_limit:
|
||||
window: 15m # parseDuration: Ns/Nm/Nh/Nd. `null` = off.
|
||||
perKey: "ruleId:slug" # optional, limits per-dimension-slug
|
||||
suppress: # optional DSL, fail-closed on eval error
|
||||
when: "signal.unwiredCount < 2 && trigger.set_drifted"
|
||||
escalations: [] # optional mention ladder — see _defaults.yml
|
||||
|
||||
template: # mustache
|
||||
text: |
|
||||
:warning: *drift — {{signal.unwiredCount}} bypassing aimock:*
|
||||
{{#signal.unwired}}• `{{.}}`
|
||||
{{/signal.unwired}}
|
||||
<{{{env.dashboardUrl}}}|Dashboard>
|
||||
|
||||
on_error: # optional separate template for probeErrored=true ticks
|
||||
template:
|
||||
text: ":rotating_light: probe errored: `{{signal.probeErrorDesc}}`"
|
||||
|
||||
actions: [] # reserved; no-op for now
|
||||
```
|
||||
|
||||
### Triggers (`src/rules/schema.ts`)
|
||||
|
||||
State-transition triggers: `green_to_red`, `red_to_green`, `sustained_red`, `sustained_green`, `first`, `stable`, `regressed`, `improved`.
|
||||
|
||||
Signal-derived triggers (set in `deriveSignalFlags`): `set_changed`, `set_drifted`, `set_errored`, `gate_skipped`, `cancelled_prebuild`, `cancelled_midmatrix`.
|
||||
|
||||
Plus `cron_only: {schedule}` for time-based invariant rules.
|
||||
|
||||
A rule fires when **any** listed trigger matches. The matched trigger name is exposed in the template as `{{#trigger.X}}...{{/trigger.X}}`.
|
||||
|
||||
### Templates — Mustache safety rules
|
||||
|
||||
Rules render via Mustache. The renderer gates triple-brace `{{{path}}}` at rule-load time to prevent injection:
|
||||
|
||||
- Triple-brace on `signal.*` is permitted **only** for fields declared in the probe's `*_SLACK_SAFE_FIELDS` export. Adding a new triple-brace-safe probe field requires extending the probe's export list + a rule-loader test.
|
||||
- Triple-brace on `event.*` is permitted for: `id`, `at`, `runId`, `runUrl`, `jobUrl`.
|
||||
- Triple-brace on `env.*` is permitted for: `dashboardUrl`, `repo`.
|
||||
- Anything else — use double-brace `{{path}}` (HTML-escaped). Triple-brace on an un-safelisted path fails `validateTripleBrace` at load and the rule is rejected.
|
||||
|
||||
Convention for Slack link markup: `<{{{url}}}|label>` — triple-brace the URL because Mustache would otherwise HTML-escape `&` inside query strings, breaking Slack's link parser.
|
||||
|
||||
### Filters
|
||||
|
||||
`{{path | filterName}}` or chained: `{{path | stripAnsi | slackEscape | truncateUtf8 2048}}`.
|
||||
|
||||
Available filters (`src/render/filters.ts`):
|
||||
|
||||
| Filter | Purpose |
|
||||
| -------------- | ------------------------------------------------------------------------------------- | ------------------------------ |
|
||||
| `stripAnsi` | Remove ANSI colour escapes. |
|
||||
| `truncateUtf8` | Byte-bounded truncation (codepoint-aware). `truncateUtf8 2000` caps at 2000 bytes. |
|
||||
| `truncateCsv` | Comma-separated truncation, drops whole entries. `truncateCsv 500` caps at 500 chars. |
|
||||
| `slackEscape` | Escape `&`, `<`, `>` for Slack mrkdwn label context. Does not escape ` | ` — use triple-brace for URLs. |
|
||||
|
||||
Unknown filters are rejected at rule-load. Output of a filter can never be re-parsed by Mustache (sentinel-fenced).
|
||||
|
||||
### Suppress DSL (`src/alerts/dsl.ts`)
|
||||
|
||||
Hand-written recursive-descent parser. No function calls, no member access beyond `x.y` dot notation. Fail-closed on eval error (treated as `true` — alert IS suppressed) and emits `suppress.eval-failed` on the bus so operators can route a watcher rule at it.
|
||||
|
||||
Identifier surface exposed in suppress expressions:
|
||||
|
||||
- `signal.*` — anything on the probe's signal object
|
||||
- `trigger.*` — boolean for each matched trigger name
|
||||
- `state.new`, `state.prev` — one of `green`, `red`, `degraded`, `error`, or `null` if no prior state
|
||||
- `lastAlertAgeMin` — minutes since this rule last fired for this dedupe key, or `undefined` on first match
|
||||
- `hasCandidates`, `probeErrored` — signal-derived booleans
|
||||
|
||||
Operators: `==`, `!=`, `<`, `<=`, `>`, `>=`, `&&`, `||`, `!`, literal strings, literal numbers, literal booleans. Validate at rule-load via a dry-run eval; malformed expressions fail the compile.
|
||||
|
||||
### Rate limit + escalation
|
||||
|
||||
- `rate_limit.window: 15m` — same `(rule, dedupe_key)` doesn't re-fire within the window. `null` disables. Fail-load on any spec that doesn't `parseDuration` cleanly.
|
||||
- `escalations: [{whenFailCount: N, mention: "@oncall"}, ...]` — ladder keyed on consecutive-failure count. Last-matching-threshold wins (ascending sort), rendered as `{{escalationMention}}`.
|
||||
|
||||
### Dedupe
|
||||
|
||||
Dedupe key is `alpha-sorted([rule.id, key, trigger1, trigger2, ...])` joined by `:`. A multi-target rule advances dedupe only when **all** targets succeed; a partial failure leaves the key unadvanced so the failing target retries next tick.
|
||||
|
||||
## 1.3a Probe configs
|
||||
|
||||
Location: `showcase/ops/config/probes/*.yml`. One YAML per probe. Loaded at startup + hot-reloaded via chokidar + SIGHUP, exactly like alert rules (`probes.reloaded` / `probes.reload.failed` emit on the bus on success/error).
|
||||
|
||||
A probe config binds a `kind` (driver) to a `schedule` (cron) and a target shape. At each tick the scheduler calls the driver with one input per target; every driver invocation produces one `ProbeResult` which flows through `writer.write()` → `status.changed` → the alert engine. One YAML = one scheduler entry = N target invocations per tick.
|
||||
|
||||
### Three YAML shapes
|
||||
|
||||
Exactly one of `targets` / `discovery` / `target` is required per config. The loader's Zod schema (`ProbeConfigSchema`) enforces this — a config with zero or more than one of these three fails the load.
|
||||
|
||||
**Static targets** — probes with a fixed, operator-authored list of endpoints. Used by `smoke` (and by `e2e_smoke` once its Playwright runner lands — the driver exists, the YAMLs are deferred):
|
||||
|
||||
```yaml
|
||||
kind: smoke
|
||||
id: smoke
|
||||
schedule: "*/15 * * * *"
|
||||
timeout_ms: 10000
|
||||
max_concurrency: 6
|
||||
targets:
|
||||
- {
|
||||
key: "smoke:mastra",
|
||||
url: "https://showcase-mastra-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:agno",
|
||||
url: "https://showcase-agno-production.up.railway.app/smoke",
|
||||
}
|
||||
```
|
||||
|
||||
**Dynamic discovery** — probes that enumerate targets from an external source (Railway API, pnpm workspace, etc.). Used by `image_drift` and `version_drift`:
|
||||
|
||||
```yaml
|
||||
kind: image_drift
|
||||
id: image-drift
|
||||
schedule: "*/15 * * * *"
|
||||
timeout_ms: 30000
|
||||
max_concurrency: 4
|
||||
discovery:
|
||||
source: railway-services
|
||||
filter:
|
||||
namePrefix: "showcase-"
|
||||
key_template: "image_drift:${name}"
|
||||
```
|
||||
|
||||
**Single target** — report-style probes whose driver fans out internally across many entities but emits exactly one synthetic ProbeResult. Used by `pin_drift`, `redirect_decommission`, `aimock_wiring`:
|
||||
|
||||
```yaml
|
||||
kind: pin_drift
|
||||
id: pin-drift-weekly
|
||||
schedule: "0 10 * * 1"
|
||||
target:
|
||||
key: "pin_drift:overall"
|
||||
```
|
||||
|
||||
### Kind → dimension mapping
|
||||
|
||||
Every `kind` resolves to a driver registered in `src/probes/drivers/index.ts`. The driver owns the emitted ProbeResult's `key` prefix, which must match a declared `Dimension` in `src/types/index.ts` (closed enum) so the rule-YAML side can narrow cleanly.
|
||||
|
||||
| YAML `kind` | Driver file | Emitted key prefix(es) | Shape |
|
||||
| ----------------------- | ---------------------------------- | -------------------------------------- | --------- |
|
||||
| `smoke` | `drivers/smoke.ts` | `smoke:<slug>` **and** `health:<slug>` | static |
|
||||
| `e2e_smoke` (deferred) | `drivers/e2e-smoke.ts` | `e2e_smoke:<suite>` | static |
|
||||
| `image_drift` | `drivers/image-drift.ts` | `image_drift:<service>` | discovery |
|
||||
| `version_drift` | `drivers/version-drift.ts` | `version_drift:<pkg>` | discovery |
|
||||
| `pin_drift` | `drivers/pin-drift.ts` | `pin_drift:overall` | single |
|
||||
| `redirect_decommission` | `drivers/redirect-decommission.ts` | `redirect_decommission:overall` | single |
|
||||
| `aimock_wiring` | `drivers/aimock-wiring.ts` | `aimock_wiring:global` | single |
|
||||
|
||||
The `smoke` driver is the only one that emits **two** keys per target invocation: the primary `smoke:<slug>` ProbeResult is the driver's return value (written by the invoker), and the paired `health:<slug>` ProbeResult is side-emitted through `ctx.writer.write()` before returning. One YAML static target = two writer ticks per cycle. See the JSDoc on `smokeDriver` for why the paired emission is a writer side-channel rather than an array return.
|
||||
|
||||
### Discovery sources
|
||||
|
||||
Registered in `src/probes/discovery/index.ts`. Closed enum — a typo in `discovery.source` fails the load with `probe-loader: <file>: discovery.source 'X' is not registered (registered: …)`.
|
||||
|
||||
| Source | Used by | Reads |
|
||||
| ------------------ | --------------- | -------------------------------------------------------------------------------------------------------------------- |
|
||||
| `railway-services` | `image_drift` | Railway GraphQL `project.services` via `RAILWAY_TOKEN` + `RAILWAY_PROJECT_ID`. Filter by `namePrefix` / `nameRegex`. |
|
||||
| `pnpm-packages` | `version_drift` | `pnpm-workspace.yaml` + per-package manifests via `fs`. Filter by `pathPrefix` / `nameGlob`. |
|
||||
|
||||
A new source is added by implementing the `DiscoverySource` interface (`src/probes/types.ts`), writing ≥95% unit coverage against a fake backend, and registering it in the orchestrator's discovery registry at boot alongside the existing entries.
|
||||
|
||||
### Fan-out semantics
|
||||
|
||||
One probe tick produces N driver invocations (N = target count, resolved at tick time for discovery configs). Each invocation is bounded independently by `timeout_ms`; concurrency across a single tick is capped at `max_concurrency` (default 4, min 1, max 32 — raise it to overlap independent targets, lower it to serialize). Each invocation writes ≥1 `status.changed` event, and each event independently passes through the alert engine — a multi-target probe with 17 services produces 17 rule evaluations per tick, not one.
|
||||
|
||||
`max_concurrency` is a per-tick worker pool. A tick that overruns its own schedule (e.g. 17 services × 30s timeout > 15 min cron window on `max_concurrency=1`) is skipped by Croner's overlap protection rather than queued.
|
||||
|
||||
### Hot reload
|
||||
|
||||
`chokidar` watches `config/probes/`. Any add / change / unlink re-runs the loader and calls `diffProbeSchedules` — removed configs are `scheduler.unregister`'d (drains in-flight handlers first); added / changed configs are re-registered (ID uses a `probe:` prefix so it never collides with rule-cron `<ruleId>:cron:<idx>` or internal IDs). A load failure emits `probes.reload.failed` on the bus without dropping the running schedule, mirroring rule-loader semantics. SIGHUP forces the same re-read path.
|
||||
|
||||
## 1.4 Slack webhook alias convention
|
||||
|
||||
A rule declares `webhook: <alias>`. The Slack target resolves it by uppercasing + dash-to-underscore, then reading `SLACK_WEBHOOK_<ALIAS>` from the env.
|
||||
|
||||
- Rule: `webhook: oss_alerts` → env: `SLACK_WEBHOOK_OSS_ALERTS`
|
||||
- Rule: `webhook: eng-alerts` → env: `SLACK_WEBHOOK_ENG_ALERTS`
|
||||
|
||||
First resolution per alias per process emits a `slack-webhook.alias-resolved` info log so operators can spot a mismatch. An invalid alias shape (non `[a-z0-9_-]+`) logs `slack-webhook.invalid-alias-shape` and the delivery throws (no silent drop).
|
||||
|
||||
## 1.5 Shared-secret rotation
|
||||
|
||||
See `showcase/ops/docs/rotation-drill.md` for the full runbook. Summary: stage `SHARED_SECRET_PREV` = current, set `SHARED_SECRET` = new, rotate GitHub Actions secret `SHOWCASE_OPS_SHARED_SECRET`, drop `PREV` after one full CI cycle confirms the new key works.
|
||||
|
||||
---
|
||||
|
||||
# Part 2 — Build it, run it, extend it
|
||||
|
||||
This section is for anyone touching `showcase/ops/src/` or the Dockerfile.
|
||||
|
||||
## 2.1 Architecture
|
||||
|
||||
```
|
||||
┌────────────────────┐ signed webhook ┌──────────────────────────┐
|
||||
│ GitHub Actions │──────────────────▶│ /webhooks/deploy │
|
||||
│ (showcase_deploy) │ │ HMAC verify + schema │
|
||||
└────────────────────┘ └────────────┬─────────────┘
|
||||
│ DeployResultEvent
|
||||
▼
|
||||
┌────────────────────────────────────────────────────────────────────┐
|
||||
│ Event bus (TypedEventBus) — in-process pub/sub │
|
||||
└────────────────────────────────────────────────────────────────────┘
|
||||
▲ ▲ │
|
||||
│ │ ▼
|
||||
┌────────┴────────┐ ┌─────────┴──────────┐ ┌────────────────────┐
|
||||
│ Probes (cron) │──▶│ Status writer │ │ Alert engine │
|
||||
│ smoke, health │ │ PB status + history│──▶│ transition → │
|
||||
│ image-drift │ │ keyed mutex │ │ guards/suppress/ │
|
||||
│ aimock-wiring │ │ writer.failed evts │ │ rate-limit → │
|
||||
│ pin/version │ └─────────────────────┘ │ render → │
|
||||
│ redirect-decom │ │ sendToTargets │
|
||||
└─────────────────┘ └────────┬──────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────┐
|
||||
│ Slack target │
|
||||
│ (retry + HMAC │
|
||||
│ alias env) │
|
||||
└───────────────┘
|
||||
|
||||
Side channels: metrics (Prometheus), /health, logger, S3 backup cron
|
||||
Storage: PocketBase (status, status_history, alert_state)
|
||||
```
|
||||
|
||||
Core invariants:
|
||||
|
||||
- **Single-writer per status key.** `status-writer` takes a keyed mutex before reading prior state and persisting; concurrent ticks for the same key serialize. Writer never emits `status.changed` unless the PB write succeeded — no phantom transitions.
|
||||
- **Fail-closed dispatch, fail-open observation.** Suppress DSL eval error → suppress (don't spam Slack). Prior-state PB read error → fall open (still fire the alert so operators see the probe).
|
||||
- **Dedupe holds on partial failure.** Multi-target rule with one failing webhook does not advance dedupe for that target — it'll retry next tick.
|
||||
- **Bootstrap window.** First 15 minutes post-boot suppress bare `first` reds/degraded (cold-start noise). Transition-bearing triggers (`green_to_red`, `set_drifted`, etc.) fire normally.
|
||||
|
||||
## 2.2 Code layout
|
||||
|
||||
```
|
||||
src/
|
||||
├── orchestrator.ts # boot(): wire all components, own lifecycle
|
||||
├── cli.ts # (not present — orchestrator is the entrypoint)
|
||||
├── logger.ts # structured JSON logger, SIGHUP-reloadable level
|
||||
├── types/index.ts # Dimension enum, State, Transition, Severity
|
||||
├── http/
|
||||
│ ├── server.ts # Hono server, /health /metrics /webhooks
|
||||
│ ├── hmac.ts # canonical payload + timing-safe verify
|
||||
│ ├── metrics.ts # typed counter registry
|
||||
│ └── webhooks/deploy.ts # signed-deploy ingest + dedupe LRU
|
||||
├── events/
|
||||
│ ├── event-bus.ts # TypedEventBus + BusEvents union
|
||||
│ └── transition-detector.ts # 16-cell state-machine table
|
||||
├── probes/
|
||||
│ ├── types.ts # ProbeDriver / DiscoverySource / registry interfaces
|
||||
│ ├── deploy-result.ts # webhook deploy-event → ProbeResult mapper
|
||||
│ ├── smoke.ts # legacy smoke probe (deriveHealthUrl + SMOKE_SLACK_SAFE_FIELDS)
|
||||
│ ├── pin-drift.ts # pinDriftProbe state-machine authority
|
||||
│ ├── aimock-wiring.ts # aimockWiringProbe (used by driver + legacy cron resolver)
|
||||
│ ├── redirect-decommission.ts # legacy probe + REDIRECT_DECOMMISSION_SLACK_SAFE_FIELDS
|
||||
│ ├── drivers/ # YAML-driven ProbeDriver implementations (one per kind)
|
||||
│ ├── discovery/ # DiscoverySource implementations (railway-services, pnpm-packages)
|
||||
│ └── loader/ # probe-loader + probe-invoker + ProbeConfigSchema
|
||||
├── rules/
|
||||
│ ├── schema.ts # Zod schema + TriggerEnum + DimensionEnum
|
||||
│ └── rule-loader.ts # compile + chokidar watcher + bus emission
|
||||
├── render/
|
||||
│ ├── renderer.ts # two-phase Mustache + sentinel fence
|
||||
│ ├── filters.ts # FILTER_NAMES tuple + implementations
|
||||
│ └── filter-regex.ts # shared filter-path regex
|
||||
├── alerts/
|
||||
│ ├── dsl.ts # parseDuration, evalSuppress
|
||||
│ └── alert-engine.ts # dispatch, buildContext, resolveTriggers
|
||||
├── writers/
|
||||
│ └── status-writer.ts # keyed-mutex PB writer, errorInfo classifier
|
||||
├── targets/
|
||||
│ └── slack-webhook.ts # retry + Retry-After + alias env resolution
|
||||
├── storage/
|
||||
│ ├── pb-client.ts # retry-budget HTTP wrapper
|
||||
│ ├── alert-state-store.ts # dedupe state with TOCTOU retry
|
||||
│ └── s3-backup.ts # optional nightly PB backup
|
||||
└── scheduler/
|
||||
└── scheduler.ts # cron registry, drain, overlap-skip
|
||||
```
|
||||
|
||||
`docs/rotation-drill.md` — secret rotation runbook (§1.5).
|
||||
|
||||
`config/alerts/` — alert rule YAMLs (§1.3).
|
||||
|
||||
## 2.3 Local dev
|
||||
|
||||
```bash
|
||||
cd showcase/ops
|
||||
pnpm install --filter @copilotkit/showcase-ops
|
||||
pnpm dev # tsx watch src/orchestrator.ts
|
||||
|
||||
# Or just run the built artifact:
|
||||
pnpm build && pnpm start
|
||||
```
|
||||
|
||||
Needs a running PocketBase. For local iteration, either `pnpm --filter showcase-pocketbase dev` in `showcase/pocketbase/` or point `POCKETBASE_URL` at any PB 0.22 instance with the expected collections (see `showcase/pocketbase/pb_migrations/`).
|
||||
|
||||
## 2.4 Tests
|
||||
|
||||
```bash
|
||||
pnpm test # 675 unit tests, <10s
|
||||
pnpm test:watch
|
||||
pnpm test:coverage
|
||||
pnpm test:integration # config wired but test/integration/ empty today
|
||||
pnpm test:e2e # same — test/e2e/ empty
|
||||
pnpm typecheck # tsc --noEmit
|
||||
```
|
||||
|
||||
Golden-file tests (renderer, filters): regenerate with `pnpm test:update-goldens`.
|
||||
|
||||
All LLM-adjacent targets (none in this service today, but see `aimock`) should use `npx aimock` for deterministic replay — never hand-rolled vi.mock response stubs.
|
||||
|
||||
## 2.5 Build + deploy
|
||||
|
||||
Production runs a single image on Railway, pulled from `ghcr.io/copilotkit/showcase-ops:latest`.
|
||||
|
||||
```bash
|
||||
# 1) Build + push (amd64 is required — Railway runs x86 hosts)
|
||||
docker buildx build --platform linux/amd64 --push \
|
||||
-f showcase/ops/Dockerfile \
|
||||
-t ghcr.io/copilotkit/showcase-ops:latest .
|
||||
|
||||
# 2) Trigger a Railway redeploy pinned to the new digest.
|
||||
# serviceInstanceDeployV2 forces a fresh snapshot — serviceInstanceRedeploy
|
||||
# replays the prior manifest and can re-pull a stale digest.
|
||||
RW_TOKEN=$(jq -r .user.token ~/.railway/config.json)
|
||||
curl -s -X POST https://backboard.railway.com/graphql/v2 \
|
||||
-H "Authorization: Bearer $RW_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query":"mutation { serviceInstanceDeployV2(serviceId:\"3a14bfed-0537-4d71-897b-7c593dca161d\", environmentId:\"b14919f4-6417-429f-848d-c6ae2201e04f\") }"}'
|
||||
|
||||
# 3) Verify
|
||||
curl -s https://showcase-ops-production.up.railway.app/health
|
||||
# {"status":"ok","pb":"ok","loop":"ok","rules":8,"schedulerJobs":3}
|
||||
```
|
||||
|
||||
Railway service/environment IDs above are for the `showcase` project's production environment.
|
||||
|
||||
There is no CI workflow that auto-builds showcase-ops. Deploys are manual until a build job lands.
|
||||
|
||||
## 2.6 Adding things
|
||||
|
||||
**A new probe.** Pick or extend a `kind` in `src/probes/drivers/`. Implement `ProbeDriver<Input, Signal>` with ≥95% test coverage. Register the driver in `orchestrator.ts` (`probeRegistry.register(...)`). Drop a `config/probes/<name>.yml` — one of `targets` / `discovery` / `target` (see §1.3a). Add the dimension to `DIMENSIONS` in `src/types/index.ts` if new. If any `signal.*` field is safe to triple-brace in a template, export it as `<NAME>_SLACK_SAFE_FIELDS` and register it in the renderer's `slackSafeFields` map (orchestrator boot). Reviewer checklist: unit tests cover success + each error branch + timeout; the discovery source (if any) has its own tests with a fake backend at ≥95% coverage; YAML validates against `ProbeConfigSchema` at load (`pnpm typecheck` + `pnpm test` cover both).
|
||||
|
||||
**A new alert rule.** Drop a `.yml` under `config/alerts/`. Reload via SIGHUP or edit-in-place (chokidar watches). Load-time validator rejects unknown filters, unsafe triple-brace, unknown trigger names, and malformed durations — fix the load error, the service never ships a broken rule.
|
||||
|
||||
**A new trigger name.** Add it to `StringTriggerEnum` in `src/rules/schema.ts` AND make sure `deriveSignalFlags` (or a transition rule) emits it. There's a runtime invariant test (`alert-engine.test.ts`) that asserts the enum and `emptyTriggerFlags()` stay in sync.
|
||||
|
||||
**A new target.** Implement the `Target` interface (`send(rendered, config)`). Register the `kind` in `orchestrator.ts` alongside `slack_webhook`. Dedupe logic is per-target in `sendToTargets` — a failing target does not advance dedupe, so retries land on the next tick.
|
||||
|
||||
**A new filter.** Add to `FILTER_NAMES` in `src/render/filters.ts` and implement. Rule loader imports `FILTER_NAMES` directly so the known-filter Set can't drift from the union.
|
||||
|
||||
**A new dimension.** Extend `DIMENSIONS` in `src/types/index.ts`. Downstream Zod validation and rule loader narrow automatically. Update any `deriveDimension` call-sites that hard-case on specific dimension strings.
|
||||
|
||||
## 2.7 Known quirks
|
||||
|
||||
- **PocketBase 0.22 auth** — superuser auth uses `/api/admins` (pre-0.23 endpoint); a warn-once log calls this out at boot. Upgrade path: drop the legacy fallback once deployed PB is 0.23+.
|
||||
- **No integration tests** — `test/integration/` and `test/e2e/` dirs exist but are empty. Unit coverage is dense (675 tests across 37 files) but no test hits a live PB or posts a real Slack webhook end-to-end.
|
||||
- **No auto-build workflow** — push to main does not produce a new `ghcr.io/copilotkit/showcase-ops:latest`; deploys are manual via §2.5.
|
||||
- **`/metrics` is unauthenticated** — intentional, gated by Railway private networking. If the service ever moves to a public mesh, add a scraper token.
|
||||
- **Bootstrap window is 15m and not env-overridable** — shift requires a code change to `AlertEngineDeps.bootstrapWindowMs`.
|
||||
|
||||
## 2.8 Related
|
||||
|
||||
- `.github/workflows/showcase_deploy.yml` — sender side of the `/webhooks/deploy` handshake. `notify-ops` step signs and POSTs.
|
||||
- `showcase/pocketbase/` — PB image + migrations. Deployed as `showcase-pocketbase` Railway service.
|
||||
- `showcase/aimock/` — fixture-based LLM mock. The aimock-wiring probe checks that every showcase package routes through it.
|
||||
- `showcase/ops/docs/rotation-drill.md` — secret rotation.
|
||||
|
||||
## 2.9 Legacy cron workflows — where their logic lives now
|
||||
|
||||
The four legacy GitHub Actions cron workflows (`showcase_smoke-monitor`, `showcase_drift-detection`, `showcase_drift-report`, `showcase_redirect-report`) are replaced by in-process probes driven by showcase-ops. Each legacy workflow lane maps to one YAML probe config + one driver in `src/probes/drivers/`:
|
||||
|
||||
| Legacy workflow | New probe YAML | Driver | Discovery |
|
||||
| ------------------------------------------ | ----------------------------------------- | ----------------------- | ------------------ |
|
||||
| `showcase_smoke-monitor.yml` (smoke) | `config/probes/smoke.yml` | `smoke` | — (static list) |
|
||||
| `showcase_smoke-monitor.yml` (image drift) | `config/probes/image-drift.yml` | `image_drift` | `railway-services` |
|
||||
| `showcase_drift-detection.yml` (L1-3) | _deferred_ | `e2e_smoke` (deferred) | — |
|
||||
| `showcase_drift-detection.yml` (L4 daily) | _deferred_ | `e2e_smoke` (deferred) | — |
|
||||
| `showcase_drift-detection.yml` (version) | `config/probes/version-drift.yml` | `version_drift` | `pnpm-packages` |
|
||||
| `showcase_drift-report.yml` (pin) | `config/probes/pin-drift.yml` | `pin_drift` | — |
|
||||
| `showcase_redirect-report.yml` | `config/probes/redirect-decommission.yml` | `redirect_decommission` | — |
|
||||
|
||||
**Deferred — auto-rebuild**: the auto-rebuild action from `showcase_smoke-monitor.yml` (automatically rebuild+redeploy on image drift) is NOT wired in this PR. Image drift still alerts via `image-drift.yml`; operators must manually redeploy off that Slack post. A follow-up PR adds a `railway-redeploy` action kind to alert-engine's target registry so the rule itself can close the loop.
|
||||
|
||||
**Deferred — e2e-smoke**: the `e2e_smoke` driver ships in this PR but its Playwright runner is not yet wired, so the `config/probes/e2e-smoke.yml` + `config/probes/e2e-smoke-daily.yml` YAMLs are intentionally absent from `config/probes/`. The `showcase_drift-detection.yml` (L1-3) and (L4 daily) lanes therefore remain on the legacy GitHub Actions cron until a follow-up PR lands the runner and re-adds the two YAMLs.
|
||||
@@ -0,0 +1,13 @@
|
||||
defaults:
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
severity: warn
|
||||
conditions:
|
||||
rate_limit:
|
||||
window: 15m
|
||||
# NOTE: `renderer: mustache` was previously declared here. The schema
|
||||
# accepts the field but rule-loader silently drops it — every rule renders
|
||||
# via the one built-in mustache path. Removed to avoid suggesting the
|
||||
# default is configurable. If/when a second renderer lands, re-add both
|
||||
# schema wiring in rule-loader AND the default here in the same change.
|
||||
@@ -0,0 +1,51 @@
|
||||
id: aimock-wiring-drift
|
||||
name: "aimock-universal invariant drift (spec §6.4)"
|
||||
owner: "@oss"
|
||||
severity: error
|
||||
|
||||
signal:
|
||||
dimension: aimock_wiring
|
||||
|
||||
# HF13-E5: set_errored wiring. Mirrors the image-drift.yml fix (R10 HF-D2)
|
||||
# — a pure-errored aimock-wiring tick emits state:"red" with an empty
|
||||
# `unwired` set and non-empty `errored` set (or `probeErrored=true` when
|
||||
# the probe itself couldn't run, per HF13-C1). Without `set_errored`
|
||||
# declared, neither `set_drifted` nor `red_to_green` matches and the alert
|
||||
# silently collapses — operators see nothing when e.g. AIMOCK_BASE_URL is
|
||||
# malformed or every service env-var read fails.
|
||||
#
|
||||
# `deriveSignalFlags` (alert-engine.ts) emits `set_errored: true` when
|
||||
# EITHER `signal.errored` is a non-empty array OR `signal.probeErrored`
|
||||
# is true — see HF13-C1 landing in src/alerts/alert-engine.ts.
|
||||
triggers:
|
||||
- set_drifted
|
||||
- set_errored
|
||||
- red_to_green
|
||||
# cron_only heartbeat on the same cadence as the aimock-wiring probe
|
||||
# (see config/probes/aimock-wiring.yml). Pairing the probe schedule with
|
||||
# a rule-level cron trigger means a long green-run still produces an
|
||||
# engine tick — the engine's synthesized `resolvedState=green` outcome
|
||||
# is available for any template that wants to render a periodic "still
|
||||
# healthy" report. Keep the cron string identical to the probe's
|
||||
# schedule; a mismatch would desynchronise the two signals and produce
|
||||
# interleaved firings.
|
||||
- cron_only: { schedule: "0 */6 * * *" }
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
template:
|
||||
text: |
|
||||
{{#trigger.set_drifted}}:warning: *aimock wiring drift — {{signal.unwiredCount}} {{signal.unwiredNoun}} bypassing showcase-aimock:*
|
||||
{{#signal.unwired}}• `{{.}}`
|
||||
{{/signal.unwired}}
|
||||
Fix: set `OPENAI_BASE_URL` (or `ANTHROPIC_BASE_URL` for claude-sdk) on each service to the aimock production URL.
|
||||
<{{{env.dashboardUrl}}}|Dashboard>{{#event.runUrl}} | <{{{event.runUrl}}}|Run>{{/event.runUrl}}{{/trigger.set_drifted}}
|
||||
{{#trigger.set_errored}}:rotating_light: *aimock wiring probe errored — {{signal.erroredCount}} service(s) unreadable:*
|
||||
{{#signal.erroredPreview}}• `{{.}}`
|
||||
{{/signal.erroredPreview}}
|
||||
{{#signal.probeErrorDesc}}Probe error: `{{.}}`
|
||||
{{/signal.probeErrorDesc}}Investigate aimock probe config / network before trusting drift state.
|
||||
<{{{env.dashboardUrl}}}|Dashboard>{{#event.runUrl}} | <{{{event.runUrl}}}|Run>{{/event.runUrl}}{{/trigger.set_errored}}
|
||||
{{#trigger.red_to_green}}:white_check_mark: *aimock wiring recovered* — all services route through showcase-aimock again. <{{{env.dashboardUrl}}}|Dashboard>{{#event.runUrl}} | <{{{event.runUrl}}}|Run>{{/event.runUrl}}{{/trigger.red_to_green}}
|
||||
@@ -0,0 +1,35 @@
|
||||
id: deploy-result
|
||||
name: "Showcase deploy result"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: deploy
|
||||
filter:
|
||||
key: overall
|
||||
|
||||
# HF13-E1: `gate_skipped` fires when the showcase_deploy.yml
|
||||
# lockfile/detect-changes gate blocks the build matrix before any service
|
||||
# deploys. The probe resolves that payload to state="green" / failedCount=0,
|
||||
# so no state-machine transition matches. `deriveSignalFlags` in
|
||||
# alert-engine.ts lifts `signal.gateSkipped === true` into a derived
|
||||
# trigger flag so this rule observes the event and renders the gate-skipped
|
||||
# template branch below.
|
||||
triggers:
|
||||
- green_to_red
|
||||
- red_to_green
|
||||
- cancelled_midmatrix
|
||||
- cancelled_prebuild
|
||||
- gate_skipped
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
template:
|
||||
text: |
|
||||
{{#trigger.green_to_red}}:x: *Showcase deploy*: {{#signal.partial}}{{signal.failedCount}}/{{signal.totalCount}} service(s) failed ({{ signal.failedList | truncateCsv 200 }}) — {{ signal.succeededList | truncateCsv 200 }} ok{{/signal.partial}}{{^signal.partial}}FAILED — {{signal.totalCount}} service(s) targeted ({{ signal.servicesList | truncateCsv 200 }}){{/signal.partial}}{{/trigger.green_to_red}}
|
||||
{{#trigger.red_to_green}}:white_check_mark: *Showcase deploy*: recovered (was down since {{signal.firstFailureAt}}){{/trigger.red_to_green}}
|
||||
{{#trigger.cancelled_midmatrix}}:information_source: *Showcase deploy*: cancelled mid-matrix — newer run (or manual retrigger) continuing; inspect if issues persist{{/trigger.cancelled_midmatrix}}
|
||||
{{#trigger.cancelled_prebuild}}:information_source: *Showcase deploy*: cancelled before any build started — likely a concurrent run superseded or manually aborted{{/trigger.cancelled_prebuild}}
|
||||
{{#trigger.gate_skipped}}:no_entry: *Showcase deploy*: build matrix gated off — no services deployed (see run for gate reason){{/trigger.gate_skipped}}
|
||||
| <{{{event.runUrl}}}|View run>
|
||||
@@ -0,0 +1,24 @@
|
||||
id: e2e-smoke-failure
|
||||
name: "Showcase E2E smoke suite failed"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: e2e_smoke
|
||||
|
||||
triggers:
|
||||
- green_to_red
|
||||
- sustained_red
|
||||
- red_to_green
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
template:
|
||||
text: |
|
||||
{{#trigger.isRedTick}}:x: *Showcase E2E suite failed*
|
||||
<{{{event.runUrl}}}|View run> · <{{{event.jobUrl}}}|View job>
|
||||
```
|
||||
{{ signal.failureSummary | stripAnsi | truncateUtf8 200 }}
|
||||
```{{/trigger.isRedTick}}
|
||||
{{#trigger.red_to_green}}:white_check_mark: *Showcase E2E suite recovered* | <{{{event.runUrl}}}|run>{{/trigger.red_to_green}}
|
||||
@@ -0,0 +1,46 @@
|
||||
id: image-drift
|
||||
name: "GHCR image drift vs showcase/ or examples/integrations/ HEAD"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: image_drift
|
||||
|
||||
triggers:
|
||||
# set_changed catches stale-set transitions (services newly out of date).
|
||||
# set_errored catches the pure-error case: a service flips stale→errored
|
||||
# without changing the stale set, so set_changed stays false but the
|
||||
# template's {{signal.errored.length}} would silently render 0 without
|
||||
# firing. deriveSignalFlags emits set_errored when signal.errored is
|
||||
# non-empty; StringTriggerEnum accepts it (rules/schema.ts R5 C4).
|
||||
- set_changed
|
||||
- set_errored
|
||||
|
||||
conditions:
|
||||
guards:
|
||||
- minDeployAgeMin: 20
|
||||
|
||||
actions:
|
||||
- kind: rebuild
|
||||
target: railway_redeploy
|
||||
forEach: "{{signal.staleServices}}"
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
# Template only references fields the ImageDriftSignal probe actually
|
||||
# emits. The `rebuildFailures` branch (present in an earlier revision)
|
||||
# was dead code — the probe never sets that field, so the {{#...}}
|
||||
# section never rendered. Removed to prevent another reviewer from
|
||||
# trusting dead UI.
|
||||
#
|
||||
# Split text reflects F6.6: triggeredCount historically summed stale +
|
||||
# errored, but `forEach` only redeploys stale. Render both counts
|
||||
# distinctly so the Slack message matches what actions took place.
|
||||
# Coordination with cluster 4: probe should keep emitting
|
||||
# `staleCount` and `erroredCount` (or the template needs updating to
|
||||
# derive them from the arrays). Template below falls back to array
|
||||
# lengths via mustache `.length` for robustness.
|
||||
template:
|
||||
text: |
|
||||
:package: *Image drift detected* — {{signal.staleServices.length}} {{signal.rebuildNoun}} triggered, {{signal.errored.length}} errored (<{{{event.runUrl}}}|run>)
|
||||
@@ -0,0 +1,30 @@
|
||||
id: pin-drift-weekly
|
||||
name: "Weekly pin-drift report (FAIL count vs baseline)"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: pin_drift
|
||||
|
||||
triggers:
|
||||
- cron_only:
|
||||
schedule: "0 10 * * 1"
|
||||
|
||||
conditions:
|
||||
rate_limit: null
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
# Render `[first run — no baseline yet]` on the initial weekly run
|
||||
# instead of leaking the raw enum value `[no_baseline]` into Slack.
|
||||
# Probe emits `signal.noBaseline = true` when `signal.setStatus ==
|
||||
# "no_baseline"`; template branches on that flag.
|
||||
template:
|
||||
text: |
|
||||
{{#signal.noBaseline}}:chart_with_downwards_trend: *Showcase pin-drift (weekly)*: FAIL={{signal.actualCount}} [first run — no baseline yet] | <{{{event.runUrl}}}|Run>{{/signal.noBaseline}}
|
||||
{{^signal.noBaseline}}:chart_with_downwards_trend: *Showcase pin-drift (weekly)*: FAIL={{signal.actualCount}} (baseline {{signal.baselineCount}}) [{{signal.setStatus}}] | <{{{event.runUrl}}}|Run>{{/signal.noBaseline}}
|
||||
|
||||
on_error:
|
||||
template:
|
||||
text: ":x: *Showcase drift report*: job failed | <{{{event.runUrl}}}|Run>"
|
||||
@@ -0,0 +1,33 @@
|
||||
id: redirect-decommission-monthly
|
||||
name: "Monthly SEO redirect decommission candidates"
|
||||
owner: "@growth"
|
||||
|
||||
signal:
|
||||
dimension: redirect_decommission
|
||||
|
||||
triggers:
|
||||
- cron_only:
|
||||
schedule: "0 9 1 * *"
|
||||
|
||||
# NOTE: the alert-engine suppress DSL does NOT support dot-access — only
|
||||
# flat identifiers. The probe emits `signal.hasCandidates` (boolean) and
|
||||
# `signal.probeErrored` (boolean, surfaced when the audit itself failed);
|
||||
# both are aliased into flat identifiers by alert-engine's suppress-var
|
||||
# construction AND declared in rule-loader's SUPPRESS_VALIDATION_VARS.
|
||||
#
|
||||
# HF13-E2: the original `hasCandidates != true` expression suppressed both
|
||||
# "probe ran, found nothing" (correct silence) AND "probe failed, returned
|
||||
# hasCandidates=false as a default" (silent audit failure). Widen to also
|
||||
# require `probeErrored != true` so a failed audit fires instead of being
|
||||
# suppressed as "no candidates".
|
||||
conditions:
|
||||
suppress:
|
||||
when: hasCandidates != true && probeErrored != true
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
template:
|
||||
text: |
|
||||
{{#signal.probeErrored}}:warning: *Redirect-decommission audit failed*: {{signal.probeErrorDesc}}{{/signal.probeErrored}}{{^signal.probeErrored}}{{{ signal.body }}}{{/signal.probeErrored}}
|
||||
@@ -0,0 +1,42 @@
|
||||
id: smoke-red-tick
|
||||
name: "Starter smoke probe failing"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: smoke
|
||||
filter:
|
||||
kind: starter
|
||||
|
||||
triggers:
|
||||
- green_to_red
|
||||
- sustained_red
|
||||
- red_to_green
|
||||
|
||||
conditions:
|
||||
escalations:
|
||||
- whenFailCount: 4
|
||||
mention: "!channel"
|
||||
severity: critical
|
||||
|
||||
guards:
|
||||
- minDeployAgeMin: 20
|
||||
|
||||
rate_limit:
|
||||
perKey: "{{signal.slug}}:{{triggerName}}"
|
||||
window: 15m
|
||||
|
||||
suppress:
|
||||
when: 'trigger == "sustained_red" && lastAlertAgeMin < 15'
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
template:
|
||||
text: |
|
||||
{{#trigger.green_to_red}}:red_circle: *{{signal.slug}}* — down, error: {{signal.errorDesc}} (<{{{signal.links.smoke}}}|smoke> · <{{{signal.links.health}}}|health>){{/trigger.green_to_red}}
|
||||
{{#trigger.sustained_red}}:red_circle: *{{signal.slug}}* — attempt: {{signal.failCount}}, error: {{signal.errorDesc}} (<{{{signal.links.smoke}}}|smoke> · <{{{signal.links.health}}}|health>){{/trigger.sustained_red}}
|
||||
{{#trigger.red_to_green}}:white_check_mark: *{{signal.slug}}* recovered (was down since {{signal.firstFailureAt}}){{/trigger.red_to_green}}
|
||||
{{#escalated}}<!channel> :rotating_light: *{{signal.slug}}* has been failing for 1 hour (since {{signal.firstFailureAt}}){{/escalated}}
|
||||
|
||||
<{{{env.dashboardUrl}}}|Showcase> · <{{{env.dashboardUrl}}}/runs/{{{event.runId}}}|Run>
|
||||
@@ -0,0 +1,30 @@
|
||||
id: version-drift-weekly
|
||||
name: "Weekly version-drift report (node/python)"
|
||||
owner: "@oss"
|
||||
|
||||
signal:
|
||||
dimension: version_drift
|
||||
|
||||
triggers:
|
||||
- cron_only:
|
||||
schedule: "0 9 * * 1"
|
||||
|
||||
conditions:
|
||||
rate_limit: null
|
||||
|
||||
targets:
|
||||
- kind: slack_webhook
|
||||
webhook: oss_alerts
|
||||
|
||||
# Probe always returns state="green" (the weekly cadence is informational
|
||||
# — drift is surfaced via signal.driftType flags, not a red dimension),
|
||||
# which means the alert-engine never dispatches to `on_error`. Error
|
||||
# rendering therefore lives inside the main template under
|
||||
# `{{#signal.driftType.probeErrored}}`. The previous top-level `on_error`
|
||||
# block was dead and removed.
|
||||
template:
|
||||
text: |
|
||||
{{#signal.driftType.probeErrored}}:x: *Showcase version-drift (weekly)*: probe errored — {{signal.npmProbeErrorDesc}}{{#signal.pythonProbeErrorDesc}} | {{signal.pythonProbeErrorDesc}}{{/signal.pythonProbeErrorDesc}} | <{{{event.runUrl}}}|Run>{{/signal.driftType.probeErrored}}
|
||||
{{#signal.driftType.stable}}:white_check_mark: *Showcase version-drift (weekly)*: no drift | <{{{event.runUrl}}}|Run>{{/signal.driftType.stable}}
|
||||
{{#signal.driftType.npmDrift}}:package: *Showcase version-drift (weekly)*: npm drift detected — {{signal.npmSummary}} | <{{{event.runUrl}}}|Run>{{/signal.driftType.npmDrift}}
|
||||
{{#signal.driftType.pythonDrift}}:snake: *Showcase version-drift (weekly)*: python drift detected — {{signal.pythonSummary}} | <{{{event.runUrl}}}|Run>{{/signal.driftType.pythonDrift}}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Probe: aimock-wiring
|
||||
#
|
||||
# Verifies every LLM-calling showcase service has its OpenAI/Anthropic base
|
||||
# URL pointed at showcase-aimock. Single-target probe — the driver fans out
|
||||
# internally to every Railway service (minus the exclusion list), which is
|
||||
# why `target` is a single synthetic entry rather than `targets: [...]`.
|
||||
#
|
||||
# Schedule mirrors the pre-existing cron cadence from orchestrator.ts
|
||||
# (every 6 hours at the top of the hour). The accompanying alert rule
|
||||
# `config/alerts/aimock-wiring-drift.yml` adds a `cron_only` trigger on
|
||||
# the same schedule so a long green-run still produces a heartbeat tick
|
||||
# the alert-engine can reason about.
|
||||
kind: aimock_wiring
|
||||
id: aimock-wiring
|
||||
schedule: "0 */6 * * *"
|
||||
target:
|
||||
key: "aimock_wiring:global"
|
||||
@@ -0,0 +1,24 @@
|
||||
# Probe: image-drift
|
||||
#
|
||||
# Per-service GHCR manifest drift check. Discovery source `railway-services`
|
||||
# enumerates every Railway service in the orchestrator's project, filters
|
||||
# to `showcase-*` entries, and hands each one to the `image_drift` driver.
|
||||
# The driver fetches the expected GHCR digest (tag = `latest` by default)
|
||||
# and compares against the digest embedded in the Railway service's
|
||||
# deployed `imageRef`.
|
||||
#
|
||||
# Schedule mirrors the legacy CI bash loop cadence (every 15 minutes).
|
||||
# timeout_ms is generous enough for a single GHCR round-trip on a cold
|
||||
# TCP stack but short enough that 100 stuck calls can't stack up across
|
||||
# ticks. max_concurrency=4 matches the legacy bash pool size; 17
|
||||
# showcase services × ~200ms per GHCR call ≈ 850ms wall time per tick.
|
||||
kind: image_drift
|
||||
id: image-drift
|
||||
schedule: "*/15 * * * *"
|
||||
timeout_ms: 30000
|
||||
max_concurrency: 4
|
||||
discovery:
|
||||
source: railway-services
|
||||
filter:
|
||||
namePrefix: "showcase-"
|
||||
key_template: "image_drift:${name}"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Probe: pin-drift
|
||||
#
|
||||
# Weekly ratchet tick against `showcase/scripts/fail-baseline.json`.
|
||||
# Runs `validate-pins.ts` to collect the current `[FAIL] ...` set,
|
||||
# hashes it (sort -u | sha256), and compares count + hash against the
|
||||
# committed baseline. Emits setStatus: stable / regressed / improved /
|
||||
# no_baseline so the existing pin-drift alert template can route off
|
||||
# the same signal shape.
|
||||
#
|
||||
# Schedule: Monday 10:00 UTC — mirrors the weekly cron in
|
||||
# `.github/workflows/showcase_validate.yml`'s pin-drift ratchet so the
|
||||
# in-cluster probe tick lines up with the external CI ratchet. Single-
|
||||
# target probe (the driver internally walks every showcase package via
|
||||
# validate-pins.ts' built-in fan-out), so `target` is one synthetic
|
||||
# entry rather than a targets list.
|
||||
kind: pin_drift
|
||||
id: pin-drift-weekly
|
||||
schedule: "0 10 * * 1"
|
||||
target:
|
||||
key: "pin_drift:overall"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Probe: redirect-decommission
|
||||
#
|
||||
# Monthly SEO redirect-decommission audit: queries PostHog for seo_redirect
|
||||
# hit counts over the last 30 days, cross-references against the full
|
||||
# redirect catalogue (showcase/shell/src/lib/seo-redirects.ts) and reports
|
||||
# zero-hit decommission candidates. Schedule mirrors the existing alert
|
||||
# rule (`config/alerts/redirect-decommission-monthly.yml`) so the probe
|
||||
# tick and the rule trigger land on the same minute.
|
||||
#
|
||||
# Single-target probe — the driver itself handles the PostHog fetch and
|
||||
# fans out internally across the 300+ redirect IDs, so `target` is a
|
||||
# single synthetic entry rather than `targets: [...]`. The accompanying
|
||||
# rule suppresses on `hasCandidates != true && probeErrored != true`, so a
|
||||
# clean month emits no Slack post and a failed audit fires the dedicated
|
||||
# "audit failed" branch instead of being silently swallowed.
|
||||
kind: redirect_decommission
|
||||
id: redirect-decommission-monthly
|
||||
schedule: "0 9 1 * *"
|
||||
target:
|
||||
key: "redirect_decommission:overall"
|
||||
@@ -0,0 +1,100 @@
|
||||
# Probe: smoke
|
||||
#
|
||||
# Per-service GET /smoke health check across every showcase starter on
|
||||
# Railway. Static-targets shape — one entry per known service slug —
|
||||
# because the smoke domain is small, bounded, and operator-authored:
|
||||
# discovery would add moving parts (Railway auth, service-list drift,
|
||||
# filter predicates) for no upside when the slug list changes roughly
|
||||
# never. When a new showcase lands, add a line here.
|
||||
#
|
||||
# Each driver invocation issues TWO HTTP GETs: `url` (the /smoke
|
||||
# endpoint) and the derived /health endpoint. The smoke driver emits
|
||||
# the primary `smoke:<slug>` ProbeResult as its return value and
|
||||
# side-emits the paired `health:<slug>` ProbeResult through
|
||||
# ctx.writer, so a single YAML target produces two writer ticks.
|
||||
#
|
||||
# Schedule mirrors the pre-YAML cron cadence (every 15 minutes) — the
|
||||
# accompanying alert rule in config/alerts/ covers green→red /
|
||||
# red→green transitions and a cron-only heartbeat tick so silent
|
||||
# green runs still produce observable liveness.
|
||||
#
|
||||
# timeout_ms (10s) bounds the WHOLE driver invocation at the invoker
|
||||
# level; the driver re-applies the same bound inside each HTTP call
|
||||
# so one hung endpoint can't steal budget from its paired probe.
|
||||
# max_concurrency (6) caps simultaneous per-tick driver invocations —
|
||||
# at 17 services the tick takes ≤ 3 pool-cycles and stays well under
|
||||
# any Railway edge rate limit.
|
||||
kind: smoke
|
||||
id: smoke
|
||||
schedule: "*/15 * * * *"
|
||||
timeout_ms: 10000
|
||||
max_concurrency: 6
|
||||
targets:
|
||||
- {
|
||||
key: "smoke:ag2",
|
||||
url: "https://showcase-ag2-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:agno",
|
||||
url: "https://showcase-agno-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:claude-sdk-python",
|
||||
url: "https://showcase-claude-sdk-python-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:claude-sdk-typescript",
|
||||
url: "https://showcase-claude-sdk-typescript-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:crewai-crews",
|
||||
url: "https://showcase-crewai-crews-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:google-adk",
|
||||
url: "https://showcase-google-adk-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:langgraph-fastapi",
|
||||
url: "https://showcase-langgraph-fastapi-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:langgraph-python",
|
||||
url: "https://showcase-langgraph-python-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:langgraph-typescript",
|
||||
url: "https://showcase-langgraph-typescript-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:langroid",
|
||||
url: "https://showcase-langroid-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:llamaindex",
|
||||
url: "https://showcase-llamaindex-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:mastra",
|
||||
url: "https://showcase-mastra-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:ms-agent-dotnet",
|
||||
url: "https://showcase-ms-agent-dotnet-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:ms-agent-python",
|
||||
url: "https://showcase-ms-agent-python-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:pydantic-ai",
|
||||
url: "https://showcase-pydantic-ai-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:spring-ai",
|
||||
url: "https://showcase-spring-ai-production.up.railway.app/smoke",
|
||||
}
|
||||
- {
|
||||
key: "smoke:strands",
|
||||
url: "https://showcase-strands-production.up.railway.app/smoke",
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# Probe: version-drift (per-package)
|
||||
#
|
||||
# Discovery-driven fan-out: the `pnpm-packages` source enumerates every
|
||||
# workspace package, and the driver checks each against npmjs/pypi for
|
||||
# upstream drift. One ProbeResult per package is written — the status
|
||||
# writer's per-key tracking means individual packages green/red
|
||||
# independently and the weekly alert rule aggregates across them.
|
||||
#
|
||||
# Schedule: Mondays at 09:00 UTC (same cadence as the legacy aggregate
|
||||
# weekly probe in orchestrator.ts). If the registry queue is large
|
||||
# enough that the tick overruns, the next tick is simply skipped —
|
||||
# Croner's overlap protection handles that.
|
||||
#
|
||||
# timeout_ms / max_concurrency: 10s per package is generous for the
|
||||
# small JSON responses (<2KB) npmjs and pypi return; 5 concurrent
|
||||
# is well below either registry's anonymous rate limit but high
|
||||
# enough that 50 packages finish inside a few seconds rather than
|
||||
# a minute of serial waiting.
|
||||
#
|
||||
# pathPrefix filter: scope to `packages/` so examples/ and showcase/
|
||||
# don't get probed every tick. Those paths are tested independently;
|
||||
# mixing them into the weekly drift feed would flood the alert
|
||||
# channel with false-positives for pinned-on-purpose demo deps.
|
||||
kind: version_drift
|
||||
id: version-drift-weekly
|
||||
schedule: "0 9 * * 1"
|
||||
timeout_ms: 10000
|
||||
max_concurrency: 5
|
||||
discovery:
|
||||
source: pnpm-packages
|
||||
filter:
|
||||
pathPrefix: "packages/"
|
||||
key_template: "version_drift:${name}"
|
||||
@@ -0,0 +1,146 @@
|
||||
# SHARED_SECRET rotation drill
|
||||
|
||||
This runbook walks through how to swap the shared password that
|
||||
`showcase-ops` uses to check that deploy-result webhooks are really
|
||||
coming from our own `showcase_deploy.yml` workflow (spec §4.5).
|
||||
|
||||
## Glossary (plain English)
|
||||
|
||||
- **Shared secret**: a random string known to both the sender (GitHub
|
||||
Actions) and the receiver (showcase-ops). The sender uses it to sign
|
||||
each webhook; the receiver uses it to check the signature.
|
||||
- **Signer**: the side that uses the secret to sign outgoing webhooks —
|
||||
in our case, the GitHub Actions workflow.
|
||||
- **Verifier**: the side that uses the secret to check signatures on
|
||||
incoming webhooks — in our case, the showcase-ops service.
|
||||
- **Overlap window**: the short period during a rotation when both the
|
||||
old and new secrets are valid, so an in-flight request signed with
|
||||
the old secret still gets accepted.
|
||||
- **Rotation**: retiring the old secret and putting a new one in place
|
||||
without dropping any webhook deliveries.
|
||||
|
||||
The service accepts either `SHARED_SECRET` **or** `SHARED_SECRET_PREV`
|
||||
as a valid signing key (see `orchestrator.ts` — both are loaded into
|
||||
the `webhookSecrets` array). This is what makes a clean, no-downtime
|
||||
rotation possible.
|
||||
|
||||
## Invariants
|
||||
|
||||
- There MUST be exactly **two** valid secrets accepted at any point
|
||||
during the drill: the one GitHub Actions is currently signing with,
|
||||
plus the previous one the service still accepts.
|
||||
- The service MUST remain able to check both old and new signatures
|
||||
during the overlap window.
|
||||
- The overlap window is ≥ 10 minutes — long enough for any in-flight
|
||||
GitHub Actions job to finish with the old secret.
|
||||
|
||||
## Procedure
|
||||
|
||||
**1. Generate the new secret.**
|
||||
|
||||
```sh
|
||||
python -c 'import secrets; print(secrets.token_urlsafe(48))'
|
||||
```
|
||||
|
||||
**2. Stage it as the NEW value on Railway.**
|
||||
|
||||
Set `SHARED_SECRET_NEW` on the showcase-ops service — a temporary
|
||||
holding slot. Do NOT yet promote it to `SHARED_SECRET`.
|
||||
|
||||
```sh
|
||||
railway variables --service showcase-ops --set SHARED_SECRET_NEW="<new>"
|
||||
```
|
||||
|
||||
**3. Roll the verifier forward (step A).**
|
||||
|
||||
First, assert the staged NEW value is actually present — skipping this
|
||||
check turns a fat-fingered step 2 into a silent outage (the verifier
|
||||
would promote an empty string as the new signer).
|
||||
|
||||
```sh
|
||||
STAGED=$(railway variables --service showcase-ops --json | jq -r '.SHARED_SECRET_NEW // empty')
|
||||
if [ -z "$STAGED" ]; then
|
||||
echo "FATAL: SHARED_SECRET_NEW is empty on showcase-ops; re-run step 2 first" >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Then on showcase-ops:
|
||||
|
||||
- Move the existing `SHARED_SECRET` → `SHARED_SECRET_PREV`
|
||||
- Move `SHARED_SECRET_NEW` → `SHARED_SECRET`
|
||||
- Unset `SHARED_SECRET_NEW`
|
||||
|
||||
```sh
|
||||
CURRENT=$(railway variables --service showcase-ops --json | jq -r .SHARED_SECRET)
|
||||
railway variables --service showcase-ops --set SHARED_SECRET_PREV="$CURRENT"
|
||||
railway variables --service showcase-ops --set SHARED_SECRET="$STAGED"
|
||||
# Remove the staging slot. Recent Railway CLI uses `--unset`; older
|
||||
# versions used `--remove`. Detect CLI capability upfront rather than
|
||||
# relying on `A || B` — a transient auth/network failure on `--unset`
|
||||
# would incorrectly fall through to `--remove`, which on a modern CLI
|
||||
# is itself an unknown-flag error and could mask the real cause.
|
||||
if railway variables --help 2>&1 | grep -q -- '--unset'; then
|
||||
UNSET_FLAG=--unset
|
||||
elif railway variables --help 2>&1 | grep -q -- '--remove'; then
|
||||
UNSET_FLAG=--remove
|
||||
else
|
||||
echo "railway CLI supports neither --unset nor --remove for variables; upgrade CLI" >&2
|
||||
exit 1
|
||||
fi
|
||||
railway variables --service showcase-ops "$UNSET_FLAG" SHARED_SECRET_NEW
|
||||
```
|
||||
|
||||
Railway will redeploy. Wait for `/health` to return 200. The verifier
|
||||
now accepts both old + new signatures.
|
||||
|
||||
**4. Roll the signer forward (step B).**
|
||||
|
||||
Update the GH Actions secret `SHOWCASE_OPS_SHARED_SECRET` in the repo
|
||||
to the new value. From the CI side, this is a single write:
|
||||
|
||||
```sh
|
||||
gh secret set SHOWCASE_OPS_SHARED_SECRET --repo CopilotKit/CopilotKit --body "<new>"
|
||||
```
|
||||
|
||||
Trigger a test deploy (e.g. re-run `showcase_deploy.yml` against a
|
||||
scratch branch) and confirm the `webhook.deploy.accepted` log appears
|
||||
on showcase-ops. If it does, the signer is now using the new key.
|
||||
|
||||
**5. Close the overlap (step C).**
|
||||
|
||||
After ≥ 10 minutes — confirmed by zero `webhook.deploy.reject
|
||||
{reason=bad-signature}` logs in the interim — remove the previous key:
|
||||
|
||||
```sh
|
||||
railway variables --service showcase-ops --unset SHARED_SECRET_PREV \
|
||||
|| railway variables --service showcase-ops --remove SHARED_SECRET_PREV
|
||||
```
|
||||
|
||||
The service redeploys and from this point forward only the new
|
||||
`SHARED_SECRET` is accepted.
|
||||
|
||||
## Verification
|
||||
|
||||
- `/health` returns 200 throughout the drill.
|
||||
- At no point does `webhook.deploy.reject {reason=bad-signature}`
|
||||
appear in the logs (except intentionally during a negative test).
|
||||
- After step 5, `grep SHARED_SECRET_PREV` returns no match in the
|
||||
service env.
|
||||
|
||||
## Rollback
|
||||
|
||||
If step 4 surfaces signer issues, revert `SHOWCASE_OPS_SHARED_SECRET` in
|
||||
GH Actions to the old value. The verifier on showcase-ops still accepts
|
||||
the old key (`SHARED_SECRET_PREV`), so rolling back the signer requires
|
||||
no service change.
|
||||
|
||||
If step 3 surfaces verifier issues (the `SHARED_SECRET_PREV` slot is
|
||||
kept specifically to give us an undo path without having to regenerate
|
||||
the secret from scratch), set `SHARED_SECRET` back to the old value and
|
||||
unset `SHARED_SECRET_PREV`.
|
||||
|
||||
## Cadence
|
||||
|
||||
Rotate every 90 days OR immediately on suspicion of compromise. Mark
|
||||
the next rotation date in the team calendar when step 5 completes.
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "@copilotkit/showcase-ops",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:integration": "vitest run --config vitest.integration.config.ts",
|
||||
"test:e2e": "vitest run --config vitest.e2e.config.ts",
|
||||
"test:update-goldens": "UPDATE_GOLDENS=1 vitest run",
|
||||
"dev": "tsx watch src/orchestrator.ts",
|
||||
"start": "node dist/orchestrator.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.710.0",
|
||||
"@hono/node-server": "^1.14.0",
|
||||
"chokidar": "^4.0.3",
|
||||
"croner": "^9.0.0",
|
||||
"hono": "^4.6.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"mustache": "^4.2.0",
|
||||
"playwright": "^1.59.1",
|
||||
"ulid": "^2.3.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/mustache": "^4.2.5",
|
||||
"@types/node": "^22.0.0",
|
||||
"@vitest/coverage-v8": "^3.2.4",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.7.0",
|
||||
"vitest": "^3.2.4"
|
||||
}
|
||||
}
|
||||
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Load test simulating 50 keys × 5-minute cadence against a local or
|
||||
* deployed showcase-ops (spec §9 Phase 5).
|
||||
*
|
||||
* For each iteration (default: 3), the script fires a burst of 50
|
||||
* requests to each probed endpoint, records per-request latency, and
|
||||
* prints a summary table of p50/p95/p99 per endpoint. Exit code 0 on
|
||||
* success; non-zero when any percentile breaches a configurable
|
||||
* threshold (see LOAD_TEST_MAX_MS, default 5000).
|
||||
*
|
||||
* Usage:
|
||||
* tsx scripts/load-test.ts --url https://showcase-ops.railway.app
|
||||
*
|
||||
* Env overrides:
|
||||
* LOAD_TEST_URL (alias for --url)
|
||||
* LOAD_TEST_KEYS number of simulated keys per burst (default 50)
|
||||
* LOAD_TEST_ITERATIONS number of bursts (default 3)
|
||||
* LOAD_TEST_MAX_MS fail if p99 exceeds this (default 5000)
|
||||
*/
|
||||
|
||||
interface EndpointSpec {
|
||||
label: string;
|
||||
path: string;
|
||||
method?: "GET" | "POST";
|
||||
body?: () => string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
const ENDPOINTS: EndpointSpec[] = [
|
||||
{ label: "GET /health", path: "/health" },
|
||||
{ label: "GET /metrics", path: "/metrics" },
|
||||
];
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const urlFlagIdx = args.indexOf("--url");
|
||||
const url =
|
||||
(urlFlagIdx !== -1 ? args[urlFlagIdx + 1] : undefined) ??
|
||||
process.env.LOAD_TEST_URL ??
|
||||
"http://localhost:8080";
|
||||
const keys = Number(process.env.LOAD_TEST_KEYS ?? "50");
|
||||
const iterations = Number(process.env.LOAD_TEST_ITERATIONS ?? "3");
|
||||
const maxMs = Number(process.env.LOAD_TEST_MAX_MS ?? "5000");
|
||||
|
||||
/**
|
||||
* Nearest-rank percentile. Note `p=1.0` returns the last element (max), which
|
||||
* is expected behavior for small n: with the default 50 requests/iteration,
|
||||
* p99 effectively degenerates to the max. If that ambiguity matters, pass a
|
||||
* larger LOAD_TEST_KEYS to get a stable p99.
|
||||
*/
|
||||
function percentile(sorted: number[], p: number): number {
|
||||
if (sorted.length === 0) return 0;
|
||||
const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
|
||||
return sorted[idx]!;
|
||||
}
|
||||
|
||||
async function measure(spec: EndpointSpec): Promise<number> {
|
||||
const start = Date.now();
|
||||
const res = await fetch(`${url}${spec.path}`, {
|
||||
method: spec.method ?? "GET",
|
||||
body: spec.body?.(),
|
||||
headers: spec.headers,
|
||||
});
|
||||
// Drain response so the measurement includes body transfer.
|
||||
await res.text();
|
||||
if (!res.ok && res.status !== 404) {
|
||||
throw new Error(`${spec.label} → HTTP ${res.status}`);
|
||||
}
|
||||
// 404 on /metrics is not fatal (some deploys disable the endpoint) but it
|
||||
// IS operationally visible — warn so operators notice if they expected
|
||||
// metrics to be enabled.
|
||||
if (res.status === 404) {
|
||||
console.warn(
|
||||
`WARN: ${spec.label} returned 404 — endpoint disabled on this deploy?`,
|
||||
);
|
||||
}
|
||||
return Date.now() - start;
|
||||
}
|
||||
|
||||
async function runBurst(spec: EndpointSpec, count: number): Promise<number[]> {
|
||||
const tasks: Promise<number>[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
tasks.push(measure(spec));
|
||||
}
|
||||
return Promise.all(tasks);
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log(
|
||||
`load-test against ${url}: ${iterations} iterations × ${keys} keys`,
|
||||
);
|
||||
const perEndpoint = new Map<string, number[]>();
|
||||
for (const ep of ENDPOINTS) perEndpoint.set(ep.label, []);
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
for (const ep of ENDPOINTS) {
|
||||
const timings = await runBurst(ep, keys);
|
||||
perEndpoint.get(ep.label)!.push(...timings);
|
||||
console.log(
|
||||
` iter ${i + 1}/${iterations} ${ep.label}: ${timings.length} requests, min=${Math.min(...timings)}ms max=${Math.max(...timings)}ms`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let failed = false;
|
||||
console.log("\nper-endpoint latency percentiles (ms):");
|
||||
console.log("endpoint p50 p95 p99 n");
|
||||
console.log("-------------------------------- ------ ------ ------ -----");
|
||||
for (const [label, timings] of perEndpoint) {
|
||||
const sorted = [...timings].sort((a, b) => a - b);
|
||||
const p50 = percentile(sorted, 0.5);
|
||||
const p95 = percentile(sorted, 0.95);
|
||||
const p99 = percentile(sorted, 0.99);
|
||||
console.log(
|
||||
`${label.padEnd(32)} ${String(p50).padStart(6)} ${String(p95).padStart(6)} ${String(p99).padStart(6)} ${String(sorted.length).padStart(5)}`,
|
||||
);
|
||||
if (p99 > maxMs) {
|
||||
console.error(`FAIL: ${label} p99=${p99}ms exceeds threshold ${maxMs}ms`);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (failed) process.exit(1);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("load-test crashed:", err);
|
||||
process.exit(2);
|
||||
});
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env bash
|
||||
# Guards the notify-ops per-job partition jq expression against
|
||||
# service/starter-name collisions.
|
||||
#
|
||||
# The showcase_deploy.yml `notify-ops` step maps each service name in
|
||||
# $SERVICES to the matrix build job that ran for it, then partitions into
|
||||
# FAILED / SUCCEEDED. Matrix job names render as
|
||||
# "build (<dispatch_name>, <context>, <image>, ...)". The previous
|
||||
# `contains($svc)` matcher produced false positives: `agno` matched both
|
||||
# `build (agno, ...)` AND `build (starter-agno, ...)`. The fixed matcher
|
||||
# uses a token-bounded prefix `startswith("build (" + $svc + ",")`.
|
||||
#
|
||||
# This script replays the bug scenario: a matrix that built BOTH `agno`
|
||||
# (failed) and `starter-agno` (succeeded). Old matcher would mis-attribute
|
||||
# `starter-agno`'s success to `agno` and/or count `agno`'s failure against
|
||||
# `starter-agno`. New matcher must:
|
||||
# - FAILED[agno] true SUCCEEDED[agno] false
|
||||
# - FAILED[starter-agno] false SUCCEEDED[starter-agno] true
|
||||
#
|
||||
# Usage: bash showcase/ops/scripts/test-notify-ops-jq.sh
|
||||
# Exits non-zero on regression.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Fixture matching the real GitHub Actions jobs API shape: a single build
|
||||
# job per matrix leg, named `build (<all object fields, comma-separated>)`.
|
||||
BUILD_JOBS='[
|
||||
{"name":"build (agno, showcase/packages/agno, showcase-agno, 32cab80b, 15, false, , , /api/health)","conclusion":"failure"},
|
||||
{"name":"build (starter-agno, showcase/starters/agno, showcase-starter-agno, baf9f0db, 15, false, , , /api/health)","conclusion":"success"},
|
||||
{"name":"build (ag2, showcase/packages/ag2, showcase-ag2, 4a37481b, 15, false, , , /api/health)","conclusion":"success"},
|
||||
{"name":"build (starter-ag2, showcase/starters/ag2, showcase-starter-ag2, 0d7ce4ea, 15, false, , , /api/health)","conclusion":"failure"},
|
||||
{"name":"build (mastra, showcase/packages/mastra, showcase-mastra, d7979eb7, 15, false, , , /api/health)","conclusion":"success"},
|
||||
{"name":"build (starter-mastra, showcase/starters/mastra, showcase-starter-mastra, 315270a7, 15, false, , , /api/health)","conclusion":"success"}
|
||||
]'
|
||||
|
||||
SERVICES='["agno","starter-agno","ag2","starter-ag2","mastra","starter-mastra"]'
|
||||
|
||||
# Fixed matcher — mirrors the jq inside showcase_deploy.yml.
|
||||
FAILED=$(echo "$SERVICES" | jq -c --argjson jobs "$BUILD_JOBS" '
|
||||
[
|
||||
.[] as $svc
|
||||
| $jobs[]
|
||||
| select((.name // "") as $n | ($n | startswith("build (" + $svc + ",")) or $n == ("build (" + $svc + ")"))
|
||||
| select(.conclusion == "failure")
|
||||
| $svc
|
||||
] | unique
|
||||
')
|
||||
|
||||
SUCCEEDED=$(echo "$SERVICES" | jq -c --argjson jobs "$BUILD_JOBS" '
|
||||
[
|
||||
.[] as $svc
|
||||
| $jobs[]
|
||||
| select((.name // "") as $n | ($n | startswith("build (" + $svc + ",")) or $n == ("build (" + $svc + ")"))
|
||||
| select(.conclusion == "success")
|
||||
| $svc
|
||||
] | unique
|
||||
')
|
||||
|
||||
EXPECTED_FAILED='["agno","starter-ag2"]'
|
||||
EXPECTED_SUCCEEDED='["ag2","mastra","starter-agno","starter-mastra"]'
|
||||
|
||||
fail=0
|
||||
if [ "$FAILED" != "$EXPECTED_FAILED" ]; then
|
||||
echo "FAIL: FAILED mismatch"
|
||||
echo " expected: $EXPECTED_FAILED"
|
||||
echo " got: $FAILED"
|
||||
fail=1
|
||||
fi
|
||||
if [ "$SUCCEEDED" != "$EXPECTED_SUCCEEDED" ]; then
|
||||
echo "FAIL: SUCCEEDED mismatch"
|
||||
echo " expected: $EXPECTED_SUCCEEDED"
|
||||
echo " got: $SUCCEEDED"
|
||||
fail=1
|
||||
fi
|
||||
|
||||
if [ $fail -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanity: prove the OLD `contains()` matcher would have regressed this case
|
||||
# — we want the NEW matcher to differ from the old one on the collision
|
||||
# fixture, otherwise this test is vacuous.
|
||||
OLD_FAILED=$(echo "$SERVICES" | jq -c --argjson jobs "$BUILD_JOBS" '
|
||||
[
|
||||
.[] as $svc
|
||||
| $jobs[]
|
||||
| select((.name // "") | contains($svc))
|
||||
| select(.conclusion == "failure")
|
||||
| $svc
|
||||
] | unique
|
||||
')
|
||||
if [ "$OLD_FAILED" = "$FAILED" ]; then
|
||||
echo "FAIL: test is vacuous — old matcher produced the same FAILED as new one"
|
||||
echo " both: $OLD_FAILED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: notify-ops jq partition is collision-free"
|
||||
echo " FAILED: $FAILED"
|
||||
echo " SUCCEEDED: $SUCCEEDED"
|
||||
echo " (old contains() would have: FAILED=$OLD_FAILED)"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Shared DSL primitives used by both alert-engine (runtime dispatch) and
|
||||
* rule-loader (load-time validation). Living in its own module so that the
|
||||
* loader doesn't have to import from alert-engine, which would otherwise
|
||||
* create a type↔value cycle (alert-engine imports `CompiledRule` from
|
||||
* rule-loader; rule-loader imports `evalSuppress` from alert-engine).
|
||||
*
|
||||
* The cycle worked in practice — `CompiledRule` is a `type` import and
|
||||
* gets erased — but kept tripping up reviewers and toolchains that scan
|
||||
* for cycles (esbuild's dep graph, some ESLint rules). Splitting these
|
||||
* into a leaf module eliminates the concern without changing behavior.
|
||||
*/
|
||||
|
||||
const UNIT_MS: Record<string, number> = {
|
||||
s: 1_000,
|
||||
m: 60_000,
|
||||
h: 3_600_000,
|
||||
d: 86_400_000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a duration spec into milliseconds.
|
||||
*
|
||||
* Accepts either a number (already in ms) or a string like `"15m"` /
|
||||
* `"1h"` / `"30s"`. Rejects zero and negative values at parse time — a
|
||||
* zero-window rate-limit would either suppress every alert forever
|
||||
* (elapsed < windowMs always true) or be meaningless depending on the
|
||||
* caller, and both outcomes are bugs we'd rather surface loudly.
|
||||
*/
|
||||
export function parseDuration(spec: string | number): number {
|
||||
if (typeof spec === "number") {
|
||||
if (!Number.isFinite(spec) || spec <= 0) {
|
||||
throw new Error(`invalid duration: ${spec} (must be > 0)`);
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
const m = spec.match(/^(\d+)([smhd])$/);
|
||||
if (!m) throw new Error(`invalid duration: ${spec}`);
|
||||
const [, num, unit] = m;
|
||||
const ms = Number(num) * UNIT_MS[unit!]!;
|
||||
if (ms <= 0) {
|
||||
throw new Error(`invalid duration: ${spec} (must be > 0)`);
|
||||
}
|
||||
return ms;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal expression evaluator for YAML `conditions.suppress.when`.
|
||||
* Supports: identifiers, string literals ("..." or '...'), number literals,
|
||||
* boolean/null, binary ops (==, !=, <=, >=, <, >), logical (&&, ||), unary !,
|
||||
* and parenthesized sub-expressions.
|
||||
*
|
||||
* Rejects any other syntax — in particular no function calls, member access,
|
||||
* indexing, or object/array literals — so YAML-authored suppression rules
|
||||
* cannot reach arbitrary JS.
|
||||
*/
|
||||
export function evalSuppress(
|
||||
expr: string,
|
||||
vars: Record<string, unknown>,
|
||||
): boolean {
|
||||
try {
|
||||
const tokens = tokenizeSuppress(expr);
|
||||
const parser = new SuppressParser(tokens, vars);
|
||||
const value = parser.parseOr();
|
||||
parser.expectEnd();
|
||||
return Boolean(value);
|
||||
} catch (err) {
|
||||
throw new Error(`invalid suppress expression: ${expr} (${String(err)})`);
|
||||
}
|
||||
}
|
||||
|
||||
type Tok =
|
||||
| { t: "ident"; v: string }
|
||||
| { t: "str"; v: string }
|
||||
| { t: "num"; v: number }
|
||||
| { t: "bool"; v: boolean }
|
||||
| { t: "null" }
|
||||
| {
|
||||
t: "op";
|
||||
v: "==" | "!=" | "<=" | ">=" | "<" | ">" | "&&" | "||" | "!" | "(" | ")";
|
||||
};
|
||||
|
||||
function tokenizeSuppress(src: string): Tok[] {
|
||||
const out: Tok[] = [];
|
||||
let i = 0;
|
||||
while (i < src.length) {
|
||||
const c = src[i]!;
|
||||
if (c === " " || c === "\t" || c === "\n" || c === "\r") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '"' || c === "'") {
|
||||
const quote = c;
|
||||
let j = i + 1;
|
||||
let value = "";
|
||||
while (j < src.length && src[j] !== quote) {
|
||||
if (src[j] === "\\" && j + 1 < src.length) {
|
||||
value += src[j + 1];
|
||||
j += 2;
|
||||
} else {
|
||||
value += src[j];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
if (src[j] !== quote) throw new Error(`unterminated string at ${i}`);
|
||||
out.push({ t: "str", v: value });
|
||||
i = j + 1;
|
||||
continue;
|
||||
}
|
||||
if (c >= "0" && c <= "9") {
|
||||
let j = i;
|
||||
while (j < src.length && /[0-9.]/.test(src[j]!)) j++;
|
||||
const n = Number(src.slice(i, j));
|
||||
if (!Number.isFinite(n)) throw new Error(`bad number at ${i}`);
|
||||
out.push({ t: "num", v: n });
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
if (/[A-Za-z_]/.test(c)) {
|
||||
let j = i;
|
||||
while (j < src.length && /[A-Za-z0-9_]/.test(src[j]!)) j++;
|
||||
const word = src.slice(i, j);
|
||||
if (word === "true") out.push({ t: "bool", v: true });
|
||||
else if (word === "false") out.push({ t: "bool", v: false });
|
||||
else if (word === "null") out.push({ t: "null" });
|
||||
else out.push({ t: "ident", v: word });
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
const two = src.slice(i, i + 2);
|
||||
if (
|
||||
two === "==" ||
|
||||
two === "!=" ||
|
||||
two === "<=" ||
|
||||
two === ">=" ||
|
||||
two === "&&" ||
|
||||
two === "||"
|
||||
) {
|
||||
out.push({ t: "op", v: two });
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === "<" || c === ">" || c === "!" || c === "(" || c === ")") {
|
||||
out.push({
|
||||
t: "op",
|
||||
v: c as "<" | ">" | "!" | "(" | ")",
|
||||
});
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unexpected character ${JSON.stringify(c)} at ${i}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
class SuppressParser {
|
||||
private pos = 0;
|
||||
constructor(
|
||||
private readonly tokens: Tok[],
|
||||
private readonly vars: Record<string, unknown>,
|
||||
) {}
|
||||
|
||||
private peek(): Tok | undefined {
|
||||
return this.tokens[this.pos];
|
||||
}
|
||||
|
||||
private consume(): Tok {
|
||||
const t = this.tokens[this.pos++];
|
||||
if (!t) throw new Error("unexpected end of expression");
|
||||
return t;
|
||||
}
|
||||
|
||||
expectEnd(): void {
|
||||
if (this.pos !== this.tokens.length)
|
||||
throw new Error(`unexpected token at pos ${this.pos}`);
|
||||
}
|
||||
|
||||
parseOr(): unknown {
|
||||
let left = this.parseAnd();
|
||||
while (this.matchOp("||")) {
|
||||
const right = this.parseAnd();
|
||||
left = Boolean(left) || Boolean(right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
parseAnd(): unknown {
|
||||
let left = this.parseEq();
|
||||
while (this.matchOp("&&")) {
|
||||
const right = this.parseEq();
|
||||
left = Boolean(left) && Boolean(right);
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
parseEq(): unknown {
|
||||
let left = this.parseRel();
|
||||
while (true) {
|
||||
if (this.matchOp("==")) {
|
||||
const r = this.parseRel();
|
||||
left = left === r;
|
||||
} else if (this.matchOp("!=")) {
|
||||
const r = this.parseRel();
|
||||
left = left !== r;
|
||||
} else break;
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
parseRel(): unknown {
|
||||
let left = this.parseUnary();
|
||||
while (true) {
|
||||
if (this.matchOp("<=")) left = Number(left) <= Number(this.parseUnary());
|
||||
else if (this.matchOp(">="))
|
||||
left = Number(left) >= Number(this.parseUnary());
|
||||
else if (this.matchOp("<"))
|
||||
left = Number(left) < Number(this.parseUnary());
|
||||
else if (this.matchOp(">"))
|
||||
left = Number(left) > Number(this.parseUnary());
|
||||
else break;
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
parseUnary(): unknown {
|
||||
if (this.matchOp("!")) return !this.parseUnary();
|
||||
return this.parsePrimary();
|
||||
}
|
||||
|
||||
parsePrimary(): unknown {
|
||||
const t = this.consume();
|
||||
if (t.t === "num") return t.v;
|
||||
if (t.t === "str") return t.v;
|
||||
if (t.t === "bool") return t.v;
|
||||
if (t.t === "null") return null;
|
||||
if (t.t === "ident") {
|
||||
// `Object.hasOwn` (not the `in` operator) so identifiers like
|
||||
// `toString`, `hasOwnProperty`, `constructor`, `__proto__` do NOT
|
||||
// resolve against Object.prototype. Pre-fix a rule typo like
|
||||
// `when: "toString"` walked the prototype chain, returned a
|
||||
// function reference (truthy), and silently suppressed every alert
|
||||
// on every tick.
|
||||
if (!Object.hasOwn(this.vars, t.v))
|
||||
throw new Error(`unknown identifier: ${t.v}`);
|
||||
return this.vars[t.v];
|
||||
}
|
||||
if (t.t === "op" && t.v === "(") {
|
||||
const val = this.parseOr();
|
||||
const close = this.consume();
|
||||
if (close.t !== "op" || close.v !== ")")
|
||||
throw new Error("missing closing paren");
|
||||
return val;
|
||||
}
|
||||
throw new Error(`unexpected token ${JSON.stringify(t)}`);
|
||||
}
|
||||
|
||||
private matchOp(op: string): boolean {
|
||||
const p = this.peek();
|
||||
if (p && p.t === "op" && p.v === op) {
|
||||
this.pos++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { createEventBus } from "./event-bus.js";
|
||||
|
||||
describe("event-bus", () => {
|
||||
it("emits and receives typed events", () => {
|
||||
const bus = createEventBus();
|
||||
const received: string[] = [];
|
||||
bus.on("rules.reloaded", (p) => {
|
||||
received.push(`count=${p.count}`);
|
||||
});
|
||||
bus.emit("rules.reloaded", { count: 3 });
|
||||
bus.emit("rules.reloaded", { count: 5 });
|
||||
expect(received).toEqual(["count=3", "count=5"]);
|
||||
});
|
||||
|
||||
it("unsubscribe removes listener", () => {
|
||||
const bus = createEventBus();
|
||||
let hits = 0;
|
||||
const unsub = bus.on("rules.reloaded", () => {
|
||||
hits += 1;
|
||||
});
|
||||
bus.emit("rules.reloaded", { count: 1 });
|
||||
unsub();
|
||||
bus.emit("rules.reloaded", { count: 1 });
|
||||
expect(hits).toBe(1);
|
||||
});
|
||||
|
||||
it("isolates subscriber errors: a throwing handler does not prevent later handlers from running", () => {
|
||||
const bus = createEventBus();
|
||||
let hitB = 0;
|
||||
// Register the throwing subscriber FIRST so there's something downstream
|
||||
// that depends on the throw being swallowed. Without error isolation,
|
||||
// Node's EventEmitter re-throws and halts dispatch on the current emit.
|
||||
bus.on("rules.reloaded", () => {
|
||||
throw new Error("boom");
|
||||
});
|
||||
bus.on("rules.reloaded", () => {
|
||||
hitB += 1;
|
||||
});
|
||||
// emit must not throw — the bus wraps each handler in try/catch.
|
||||
expect(() => bus.emit("rules.reloaded", { count: 1 })).not.toThrow();
|
||||
// The downstream subscriber must still have run.
|
||||
expect(hitB).toBe(1);
|
||||
});
|
||||
|
||||
it("unsubscribe returned from on() removes the wrapped listener (not the raw handler)", () => {
|
||||
// Regression: previously `off()` tried to remove the caller's handler
|
||||
// reference, but `on()` registers a wrapper closure for error isolation.
|
||||
// The returned unsubscribe function is the canonical way to detach.
|
||||
const bus = createEventBus();
|
||||
let hits = 0;
|
||||
const unsub = bus.on("rules.reloaded", () => {
|
||||
hits += 1;
|
||||
});
|
||||
bus.emit("rules.reloaded", { count: 1 });
|
||||
unsub();
|
||||
bus.emit("rules.reloaded", { count: 2 });
|
||||
bus.emit("rules.reloaded", { count: 3 });
|
||||
expect(hits).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import crypto from "node:crypto";
|
||||
import type { ProbeResult, WriteOutcome } from "../types/index.js";
|
||||
import { logger } from "../logger.js";
|
||||
|
||||
export interface DeployResultEvent {
|
||||
runId: string;
|
||||
runUrl?: string;
|
||||
services: string[];
|
||||
failed: string[];
|
||||
succeeded: string[];
|
||||
cancelled: boolean;
|
||||
/**
|
||||
* True when the showcase deploy workflow reached the report job but the
|
||||
* build matrix never ran (e.g. the lockfile gate failed). Senders use this
|
||||
* to disambiguate a gated-skip from an all-services failure. Optional so
|
||||
* older senders that pre-date the field still decode cleanly.
|
||||
*/
|
||||
gateSkipped?: boolean;
|
||||
/**
|
||||
* Free-form discriminator co-emitted with `gateSkipped: true`:
|
||||
* `lockfile-failed`, `lockfile-cancelled`, `verify-image-refs-failed`,
|
||||
* `verify-image-refs-cancelled`, `detect-changes-<result>`. Empty string
|
||||
* normalised to undefined at the webhook boundary so downstream checks
|
||||
* only need to guard one shape.
|
||||
*/
|
||||
gateReason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classification of a writer failure's underlying cause. Lets the alert
|
||||
* engine route transient errors (auth blip, rate limit) separately from
|
||||
* structural errors (schema mismatch, bad credentials) — the former is
|
||||
* noise, the latter is an actionable ops signal.
|
||||
*
|
||||
* - `pb_auth_error` — 401/403 from PocketBase; creds bad or token revoked.
|
||||
* - `pb_schema_error` — 400 validation / missing column; schema drift.
|
||||
* - `pb_permission` — 403 rule-level reject that isn't auth.
|
||||
* - `pb_rate_limited` — 429 after exhausting retries; transient.
|
||||
* - `pb_server_error` — 5xx; transient unless sustained.
|
||||
* - `network_error` — fetch threw (ECONN, AbortError, DNS).
|
||||
* - `unknown` — couldn't classify.
|
||||
*/
|
||||
export type WriterFailureReason =
|
||||
| "pb_auth_error"
|
||||
| "pb_schema_error"
|
||||
| "pb_permission"
|
||||
| "pb_rate_limited"
|
||||
| "pb_server_error"
|
||||
| "network_error"
|
||||
| "unknown";
|
||||
|
||||
export interface WriterFailedEvent {
|
||||
/** Probe/deploy key the writer was processing (e.g. "smoke:mastra"). */
|
||||
key: string;
|
||||
/** Phase of the write that failed — useful for /health triage. */
|
||||
phase: "status_upsert" | "history_create";
|
||||
/**
|
||||
* Serialized error context. Uses a structured representation (message +
|
||||
* status + validation payload) rather than bare `String(err)` so PB's
|
||||
* `{ data: { field: { code, message } } }` shapes stay legible after
|
||||
* emission. See status-writer.errorInfo() for the extraction logic.
|
||||
*/
|
||||
err: string;
|
||||
/**
|
||||
* Classification of the failure's underlying cause. Alert routing can
|
||||
* distinguish transient-vs-structural failures without string-matching
|
||||
* the err field. Optional (undefined before B6 landed / producers that
|
||||
* don't classify).
|
||||
*/
|
||||
reason?: WriterFailureReason;
|
||||
/** HTTP status if the failure was a PB response-code error. */
|
||||
status?: number;
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Payload for `rules.reload.failed` — produced by rule-loader.watch when
|
||||
* one or more files fail to parse or compile during a hot-reload. Declared
|
||||
* here so subscribers are type-safe; rule-loader itself only holds a
|
||||
* structural `RuleLoadErrorEmitter` interface to avoid coupling to the bus.
|
||||
*/
|
||||
export interface RulesReloadFailedEvent {
|
||||
errors: { file: string; error: string }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Payload for `probes.reload.failed` — produced by probe-loader.watch when
|
||||
* one or more YAML files fail to parse, validate, or resolve (unknown
|
||||
* driver kind / unregistered discovery source) during a hot-reload.
|
||||
* Mirrors `RulesReloadFailedEvent` so subscribers that aggregate loader
|
||||
* errors (e.g. a shared `/health` panel) can use the same shape for both
|
||||
* DSLs.
|
||||
*/
|
||||
export interface ProbesReloadFailedEvent {
|
||||
errors: { file: string; error: string }[];
|
||||
}
|
||||
|
||||
export interface BusEvents {
|
||||
"status.changed": { outcome: WriteOutcome; result: ProbeResult<unknown> };
|
||||
"deploy.result": DeployResultEvent;
|
||||
"rule.scheduled": {
|
||||
ruleId: string;
|
||||
scheduledAt: string;
|
||||
result?: ProbeResult<unknown>;
|
||||
};
|
||||
"rules.reloaded": { count: number };
|
||||
/** Emitted when a hot-reload fails to parse/compile one or more rule files. */
|
||||
"rules.reload.failed": RulesReloadFailedEvent;
|
||||
/**
|
||||
* Emitted by probe-loader after a successful reload — symmetric with
|
||||
* `rules.reloaded`. Consumers (e.g. the orchestrator's scheduler diff
|
||||
* loop) use this to know the probe set changed without having to
|
||||
* subscribe to the file watcher directly.
|
||||
*/
|
||||
"probes.reloaded": { count: number };
|
||||
/** Emitted when a hot-reload fails to parse/validate one or more probe YAML files. */
|
||||
"probes.reload.failed": ProbesReloadFailedEvent;
|
||||
/**
|
||||
* Emitted when status-writer fails mid-flight (PB upsert or history
|
||||
* create throws). Orchestrator listens to surface degraded state on
|
||||
* /health. Listener side owned by F1 agent.
|
||||
*/
|
||||
"writer.failed": WriterFailedEvent;
|
||||
/**
|
||||
* Emitted when the S3 backup job fails. Produced by s3-backup.ts via
|
||||
* its `onFailure` injection when the orchestrator wires it to this
|
||||
* bus. The alert engine can fire a rule off this event so backup
|
||||
* failures are first-class signals rather than silent log entries.
|
||||
*/
|
||||
"internal.backup.failed": { err: string };
|
||||
/**
|
||||
* Emitted when S3 backup INITIALIZATION fails at boot — e.g.
|
||||
* `createDefaultS3Uploader` throws because `@aws-sdk/client-s3` is not
|
||||
* installed, a bad AWS_REGION is set, or the credential provider chain
|
||||
* throws. Pre-fix, this path only logged at error level and the service
|
||||
* booted green while backups silently never ran. The bus emit gives
|
||||
* operators a first-class observable surface (alert rule or dashboard
|
||||
* subscription) alongside the existing error log.
|
||||
*/
|
||||
"internal.backup.init-failed": { err: string; bucket: string };
|
||||
/**
|
||||
* Emitted by the alert engine when a rule's `suppress.when` expression
|
||||
* throws at evaluation time (R24 bucket-a#7). Fail-closed semantics:
|
||||
* the triggering alert is suppressed on eval error to avoid spamming
|
||||
* during a DSL-eval regression. Operators subscribe to surface the
|
||||
* failure on a dedicated channel — the error log line alone is easy
|
||||
* to miss. `expression` is the raw DSL text from the rule for triage.
|
||||
*/
|
||||
"suppress.eval-failed": {
|
||||
ruleId: string;
|
||||
expression: string;
|
||||
error: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TypedEventBus {
|
||||
emit<K extends keyof BusEvents>(event: K, payload: BusEvents[K]): void;
|
||||
/**
|
||||
* Subscribe to `event`. The returned function unsubscribes this exact
|
||||
* handler. There is no separate `off(event, handler)` method — handlers
|
||||
* are stored as wrapper closures internally (for error isolation) so the
|
||||
* handler reference the caller holds is not the one Node's EventEmitter
|
||||
* knows about. Using the returned unsubscribe closure guarantees the
|
||||
* correct wrapper is removed.
|
||||
*/
|
||||
on<K extends keyof BusEvents>(
|
||||
event: K,
|
||||
handler: (payload: BusEvents[K]) => void,
|
||||
): () => void;
|
||||
removeAll(): void;
|
||||
}
|
||||
|
||||
// MAX_LISTENERS bumped higher than the default 10 to absorb hot-reload churn
|
||||
// (rule-loader watch() reattaches on every file change; under a rapid edit
|
||||
// loop we can briefly exceed a lower cap). If this ever fires a
|
||||
// MaxListenersExceededWarning in prod, check for leaked subscriptions from
|
||||
// repeated boot/stop cycles before bumping further.
|
||||
const MAX_LISTENERS = 200;
|
||||
|
||||
export function createEventBus(): TypedEventBus {
|
||||
const emitter = new EventEmitter();
|
||||
emitter.setMaxListeners(MAX_LISTENERS);
|
||||
// Per-subscriber failure counters, keyed by `${event}|${subscriberId}`, so
|
||||
// a constantly-failing handler becomes visible at error level (and, once
|
||||
// the metrics cluster wires a counter, a Prometheus series). Using a
|
||||
// short random id keeps the key stable for the lifetime of a subscription
|
||||
// without imposing a caller-supplied id contract.
|
||||
return {
|
||||
emit(event, payload) {
|
||||
emitter.emit(String(event), payload);
|
||||
},
|
||||
on(event, handler) {
|
||||
// Wrap the handler so a throw in one subscriber never prevents later
|
||||
// subscribers from running. Node's EventEmitter re-throws listener
|
||||
// errors by default and halts further dispatch on that emit.
|
||||
const subscriberId = crypto.randomBytes(4).toString("hex");
|
||||
let failureCount = 0;
|
||||
const wrapper = (p: unknown) => {
|
||||
try {
|
||||
handler(p as never);
|
||||
} catch (err) {
|
||||
failureCount += 1;
|
||||
// errorId lets operators cross-reference the log line with the
|
||||
// subscriber metric (once wired) and any downstream capture.
|
||||
const errorId = crypto.randomBytes(6).toString("hex");
|
||||
logger.error("event-bus: subscriber threw, continuing dispatch", {
|
||||
event: String(event),
|
||||
subscriberId,
|
||||
errorId,
|
||||
failureCount,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
stack: err instanceof Error ? err.stack : undefined,
|
||||
});
|
||||
}
|
||||
};
|
||||
emitter.on(String(event), wrapper);
|
||||
return () => emitter.off(String(event), wrapper);
|
||||
},
|
||||
removeAll() {
|
||||
// Also warn if removeAll is called while callers still hold unsubs:
|
||||
// those unsubs become no-ops (the wrappers they'd have detached are
|
||||
// already gone), but the caller may expect their handler to still
|
||||
// be reachable. Surfacing this prevents subtle "why isn't my
|
||||
// subscription firing" debugging trips. Kept at debug level — this
|
||||
// is intentional behavior on shutdown, just worth noting.
|
||||
const count = emitter
|
||||
.eventNames()
|
||||
.reduce((n, name) => n + emitter.listenerCount(name), 0);
|
||||
if (count > 0) {
|
||||
logger.debug("event-bus: removeAll detaching active listeners", {
|
||||
count,
|
||||
});
|
||||
}
|
||||
emitter.removeAllListeners();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { detectTransition } from "./transition-detector.js";
|
||||
import type { ProbeState, State, Transition } from "../types/index.js";
|
||||
|
||||
const PREVS: (State | null)[] = [null, "green", "red", "degraded"];
|
||||
const NEXTS: ProbeState[] = ["green", "red", "degraded", "error"];
|
||||
|
||||
const TABLE: Record<string, Transition> = {
|
||||
"null|green": "first",
|
||||
"null|red": "first",
|
||||
"null|degraded": "first",
|
||||
"null|error": "error",
|
||||
"green|green": "sustained_green",
|
||||
"green|red": "green_to_red",
|
||||
"green|degraded": "green_to_red",
|
||||
"green|error": "error",
|
||||
"red|green": "red_to_green",
|
||||
"red|red": "sustained_red",
|
||||
"red|degraded": "sustained_red",
|
||||
"red|error": "error",
|
||||
"degraded|green": "red_to_green",
|
||||
"degraded|red": "sustained_red",
|
||||
"degraded|degraded": "sustained_red",
|
||||
"degraded|error": "error",
|
||||
};
|
||||
|
||||
describe("transition-detector", () => {
|
||||
for (const prev of PREVS) {
|
||||
for (const next of NEXTS) {
|
||||
const key = `${prev ?? "null"}|${next}`;
|
||||
it(`${key} -> ${TABLE[key]}`, () => {
|
||||
expect(detectTransition(prev, next)).toBe(TABLE[key]);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { ProbeState, State, Transition } from "../types/index.js";
|
||||
|
||||
/**
|
||||
* Pure state-machine transition detector. No side effects.
|
||||
*
|
||||
* prev: the last known *world state* (3-valued: green/red/degraded) or null.
|
||||
* next: the probe result's state (4-valued: adds "error").
|
||||
*
|
||||
* Transition table (row = prev, col = next):
|
||||
* | | green | red | degraded | error |
|
||||
* |----------|------------------|------------------|------------------|---------|
|
||||
* | null | first | first | first | error |
|
||||
* | green | sustained_green | green_to_red | green_to_red | error |
|
||||
* | red | red_to_green | sustained_red | sustained_red | error |
|
||||
* | degraded | red_to_green | sustained_red | sustained_red | error |
|
||||
*
|
||||
* Design decisions:
|
||||
*
|
||||
* - `error` dominates `prev`: once a probe reports error, the transition is
|
||||
* always `error` regardless of prior world-state. This keeps the onError
|
||||
* dispatch path orthogonal to the normal green/red machine. Alert-engine
|
||||
* applies its own bootstrap gate on onError so a prev=null → error
|
||||
* transition is still suppressed during the bootstrap window.
|
||||
*
|
||||
* - `degraded` collapses into `red` for transition-naming (spec §2 — "red
|
||||
* dominates"). No `green_to_degraded` / `degraded_to_green` etc. — any
|
||||
* cross-family move is `green_to_red` / `red_to_green`, and within-family
|
||||
* shifts surface as `sustained_red`. The 3-valued world-state is retained
|
||||
* so UI cells can render amber distinctly; trigger names stay 2-valued.
|
||||
*
|
||||
* - There is deliberately no `first_observation` transition distinct from
|
||||
* `first`. Every first-ever record maps to `first`, and alert-engine's
|
||||
* bootstrap-window gate plus the explicit `isFreshRed` check in
|
||||
* handleStatusChanged already dedupe fresh-boot noise. Introducing a
|
||||
* separate kind would force every rule to enumerate both (or confuse
|
||||
* authors who omit one). See types/index.ts `Transition` for the
|
||||
* authoritative closed set.
|
||||
*/
|
||||
export function detectTransition(
|
||||
prev: State | null,
|
||||
next: ProbeState,
|
||||
): Transition {
|
||||
if (next === "error") return "error";
|
||||
if (prev === null) return "first";
|
||||
const prevRed = prev === "red" || prev === "degraded";
|
||||
const nextRed = next === "red" || next === "degraded";
|
||||
if (!prevRed && nextRed) return "green_to_red";
|
||||
if (prevRed && !nextRed) return "red_to_green";
|
||||
if (prevRed && nextRed) return "sustained_red";
|
||||
return "sustained_green";
|
||||
}
|
||||
@@ -0,0 +1,470 @@
|
||||
import crypto from "node:crypto";
|
||||
import { describe, it, expect, afterEach, vi } from "vitest";
|
||||
import { canonicalPayload, computeSignature, verifyHmac } from "./hmac.js";
|
||||
|
||||
const NOW = 1_700_000_000;
|
||||
const nowSec = (): number => NOW;
|
||||
|
||||
function sign(
|
||||
secret: string,
|
||||
method: string,
|
||||
path: string,
|
||||
ts: number,
|
||||
body: string,
|
||||
): string {
|
||||
return `sha256=${computeSignature(secret, canonicalPayload(method, path, String(ts), body))}`;
|
||||
}
|
||||
|
||||
describe("canonicalPayload", () => {
|
||||
it("formats METHOD|path|ts|sha256(body) with uppercase method", () => {
|
||||
const c = canonicalPayload("post", "/webhooks/deploy", "123", "hello");
|
||||
expect(c.startsWith("POST|/webhooks/deploy|123|")).toBe(true);
|
||||
// sha256("hello")
|
||||
expect(
|
||||
c.endsWith(
|
||||
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeSignature", () => {
|
||||
it("produces deterministic hex of length 64 for sha256", () => {
|
||||
const sig = computeSignature("secret", "POST|/x|1|abc");
|
||||
expect(sig).toMatch(/^[0-9a-f]{64}$/);
|
||||
// Deterministic: same inputs → same output.
|
||||
expect(computeSignature("secret", "POST|/x|1|abc")).toBe(sig);
|
||||
});
|
||||
|
||||
it("differs across secrets (rotate primary vs secondary)", () => {
|
||||
const canonical = canonicalPayload("POST", "/x", "1", "body");
|
||||
const primary = computeSignature("primary-key", canonical);
|
||||
const rotate = computeSignature("rotate-key", canonical);
|
||||
expect(primary).not.toBe(rotate);
|
||||
expect(primary).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(rotate).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
it("differs when any canonical field changes", () => {
|
||||
const a = computeSignature("k", canonicalPayload("POST", "/x", "1", "b"));
|
||||
const b = computeSignature("k", canonicalPayload("POST", "/x", "2", "b"));
|
||||
const c = computeSignature("k", canonicalPayload("POST", "/y", "1", "b"));
|
||||
const d = computeSignature("k", canonicalPayload("GET", "/x", "1", "b"));
|
||||
const e = computeSignature("k", canonicalPayload("POST", "/x", "1", "B"));
|
||||
expect(new Set([a, b, c, d, e]).size).toBe(5);
|
||||
});
|
||||
|
||||
it("handles empty body and empty path (hex, fixed length)", () => {
|
||||
const sig = computeSignature("k", canonicalPayload("POST", "", "0", ""));
|
||||
expect(sig).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("verifyHmac", () => {
|
||||
const secret = "primary-key";
|
||||
const body = '{"ok":true}';
|
||||
const path = "/webhooks/deploy";
|
||||
const method = "POST";
|
||||
const sig = sign(secret, method, path, NOW, body);
|
||||
|
||||
it("accepts a valid signature within skew", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts signatures with no sha256= prefix", () => {
|
||||
const raw = sig.replace(/^sha256=/, "");
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: raw,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects a stale timestamp", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW - 1000),
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("stale");
|
||||
});
|
||||
|
||||
it("rejects a future timestamp beyond skew", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW + 1000),
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("stale");
|
||||
});
|
||||
|
||||
it("rejects a non-integer (float) timestamp with invalid-timestamp", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: "1700000000.5",
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("invalid-timestamp");
|
||||
});
|
||||
|
||||
it("rejects a non-numeric timestamp with invalid-timestamp", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: "not-a-number",
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("invalid-timestamp");
|
||||
});
|
||||
|
||||
it("rejects a wrong signature", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "sha256=deadbeef",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("bad-signature");
|
||||
});
|
||||
|
||||
it("rejects with missing-timestamp when only timestamp is absent", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: "",
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("missing-timestamp");
|
||||
});
|
||||
|
||||
it("rejects with missing-signature when only signature is absent", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("missing-signature");
|
||||
});
|
||||
|
||||
it("rejects with missing-headers when both are absent (legacy code retained)", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: "",
|
||||
body,
|
||||
signatureHeader: "",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("missing-headers");
|
||||
});
|
||||
|
||||
it("accepts a bare hex signature without the sha256= prefix (lenient by design)", () => {
|
||||
const raw = sig.replace(/^sha256=/, "");
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: raw,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("signals timing-safe compare shape check: mismatched-length hex → bad-signature, not invalid-format", () => {
|
||||
// Half-length valid hex. The signature-format regex accepts any
|
||||
// even-length hex string; the timingSafeEqual shape-check inside
|
||||
// the loop returns false (length mismatch) and we fall through
|
||||
// with bad-signature rather than surfacing a compare error.
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "sha256=abcdef",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("bad-signature");
|
||||
});
|
||||
|
||||
it("accepts the secondary key during rotation", () => {
|
||||
const oldSecret = "old-key";
|
||||
const newSecret = "new-key";
|
||||
const oldSig = sign(oldSecret, method, path, NOW, body);
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: oldSig,
|
||||
secrets: [newSecret, oldSecret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects when neither rotation key matches", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: sign("third-key", method, path, NOW, body),
|
||||
secrets: ["k1", "k2"],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("bad-signature");
|
||||
});
|
||||
|
||||
it("rejects malformed hex in signature with invalid-signature-format", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "sha256=zzzz",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("invalid-signature-format");
|
||||
});
|
||||
|
||||
it("rejects odd-length hex in signature with invalid-signature-format", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "sha256=abc",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("invalid-signature-format");
|
||||
});
|
||||
|
||||
it("trims whitespace from the signature header before verifying", () => {
|
||||
// Simulate a sender that accidentally smuggled whitespace into the
|
||||
// header (jq `$(...)` trailing newline is a classic offender).
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: ` ${sig}\n`,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("trims whitespace from the timestamp header", () => {
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: ` ${NOW}\n`,
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
});
|
||||
expect(r.ok).toBe(true);
|
||||
});
|
||||
|
||||
describe("compare-error classification (F3.2)", () => {
|
||||
// Regression: previously both length-mismatch (expected, noisy) and
|
||||
// genuinely unexpected crypto-layer errors (rare, page-worthy) were
|
||||
// logged at debug, making it impossible to alert on real breakage
|
||||
// without being drowned by happy-path rejection noise. We now split
|
||||
// them so operators can page on `HMAC_COMPARE_UNEXPECTED_ERROR`.
|
||||
|
||||
function captureLogger(): {
|
||||
logger: {
|
||||
debug: (msg: string, meta?: unknown) => void;
|
||||
info: (msg: string, meta?: unknown) => void;
|
||||
warn: (msg: string, meta?: unknown) => void;
|
||||
error: (msg: string, meta?: unknown) => void;
|
||||
};
|
||||
debugCalls: Array<{ msg: string; meta?: unknown }>;
|
||||
warnCalls: Array<{ msg: string; meta?: unknown }>;
|
||||
} {
|
||||
const debugCalls: Array<{ msg: string; meta?: unknown }> = [];
|
||||
const warnCalls: Array<{ msg: string; meta?: unknown }> = [];
|
||||
return {
|
||||
logger: {
|
||||
debug: (msg, meta) => {
|
||||
debugCalls.push({ msg, meta });
|
||||
},
|
||||
info: () => {},
|
||||
warn: (msg, meta) => {
|
||||
warnCalls.push({ msg, meta });
|
||||
},
|
||||
error: () => {},
|
||||
},
|
||||
debugCalls,
|
||||
warnCalls,
|
||||
};
|
||||
}
|
||||
|
||||
it("logs length-mismatch at debug (not warn) — no pager spam on malformed input", () => {
|
||||
// Force a length-mismatch path by passing a validly-shaped hex
|
||||
// signature (even-length, all hex) that's shorter than the
|
||||
// computed expected length. The inner timingSafeEqual call is
|
||||
// guarded by `providedHex.length === expected.length`, so it
|
||||
// returns false without throwing — the catch branch doesn't fire
|
||||
// at all in that path, which is correct.
|
||||
//
|
||||
// To actually exercise the catch branch with a length mismatch,
|
||||
// we use a provided signature whose even-length shape passes the
|
||||
// regex but decodes to a different length than expected. The
|
||||
// length guard short-circuits for sig lengths != expected, so we
|
||||
// must construct a scenario where Buffer.from triggers throwing
|
||||
// behavior. In practice the primary path for length-mismatch is
|
||||
// a direct call to timingSafeEqual with mismatched Buffers — we
|
||||
// simulate that by forcing the comparison via secrets rotation.
|
||||
//
|
||||
// Simplest deterministic test: short even-hex → length guard
|
||||
// short-circuits, returns bad-signature, no catch fires. Verify
|
||||
// warnCalls is empty (we never surfaced HMAC_COMPARE_UNEXPECTED_ERROR).
|
||||
const { logger: cap, warnCalls } = captureLogger();
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: "sha256=abcdef",
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
logger: cap,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toBe("bad-signature");
|
||||
// Malformed/short hex must NOT surface HMAC_COMPARE_UNEXPECTED_ERROR.
|
||||
expect(
|
||||
warnCalls.some(
|
||||
(c) =>
|
||||
typeof c.meta === "object" &&
|
||||
c.meta !== null &&
|
||||
"errorId" in c.meta &&
|
||||
(c.meta as { errorId?: string }).errorId ===
|
||||
"HMAC_COMPARE_UNEXPECTED_ERROR",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
// Guard: if a crypto spy throws mid-test and we forget to restore,
|
||||
// downstream tests would see the injected error. `restoreAllMocks`
|
||||
// on the spy set up by `vi.spyOn` below auto-reverts.
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("logs unexpected crypto errors at warn with stable errorId", () => {
|
||||
// Simulate a crypto-layer failure. Our regex + even-length check
|
||||
// filters malformed hex before reaching timingSafeEqual, so the
|
||||
// only way to reach the catch-with-non-length-mismatch branch
|
||||
// from public API is via an unexpected runtime error (OOM,
|
||||
// platform quirk). We force that via `vi.spyOn` with auto-restore
|
||||
// — safer than manually reassigning a global and relying on
|
||||
// try/finally to clean up (a thrown assertion inside the try
|
||||
// would leak the stub to every subsequent test).
|
||||
const syntheticError = new Error("simulated crypto failure (OOM)");
|
||||
// Capture the spy so we can assert it actually intercepted the call.
|
||||
// `hmac.ts` uses `import crypto from "node:crypto"` and invokes
|
||||
// `crypto.timingSafeEqual(...)` via property access on the default
|
||||
// namespace — both this test and the module see the same namespace
|
||||
// object, so `vi.spyOn` rebinds the property the module reads at
|
||||
// call time. If the module ever switches to a destructured
|
||||
// `import { timingSafeEqual }`, the spy would silently miss; the
|
||||
// `toHaveBeenCalled` assertion below is the tripwire for that.
|
||||
const spy = vi
|
||||
.spyOn(crypto, "timingSafeEqual")
|
||||
.mockImplementationOnce(() => {
|
||||
throw syntheticError;
|
||||
});
|
||||
const { logger: cap, warnCalls } = captureLogger();
|
||||
const r = verifyHmac({
|
||||
method,
|
||||
path,
|
||||
timestamp: String(NOW),
|
||||
body,
|
||||
signatureHeader: sig,
|
||||
secrets: [secret],
|
||||
nowSec,
|
||||
logger: cap,
|
||||
});
|
||||
expect(r.ok).toBe(false);
|
||||
// After the synthetic crypto failure, the loop falls through to
|
||||
// bad-signature (no secret matched).
|
||||
expect(r.reason).toBe("bad-signature");
|
||||
// Tripwire: prove the spy actually intercepted. A green test with
|
||||
// zero spy invocations would mean we never exercised the catch
|
||||
// branch (e.g. the module switched to a destructured import and
|
||||
// kept the original binding).
|
||||
expect(spy).toHaveBeenCalled();
|
||||
const unexpectedCall = warnCalls.find(
|
||||
(c) =>
|
||||
typeof c.meta === "object" &&
|
||||
c.meta !== null &&
|
||||
"errorId" in c.meta &&
|
||||
(c.meta as { errorId?: string }).errorId ===
|
||||
"HMAC_COMPARE_UNEXPECTED_ERROR",
|
||||
);
|
||||
expect(unexpectedCall).toBeDefined();
|
||||
expect(unexpectedCall!.msg).toBe("hmac.verify.compare-error");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,156 @@
|
||||
import crypto from "node:crypto";
|
||||
import type { Logger } from "../types/index.js";
|
||||
|
||||
export interface HmacVerifyInput {
|
||||
method: string;
|
||||
path: string;
|
||||
timestamp: string;
|
||||
body: string;
|
||||
signatureHeader: string;
|
||||
secrets: string[];
|
||||
/** Allowed clock skew in seconds (default 300). */
|
||||
maxSkewSec?: number;
|
||||
/** Current epoch seconds; override for tests. */
|
||||
nowSec?: () => number;
|
||||
/** Optional logger for diagnosing unexpected crypto/decode failures. */
|
||||
logger?: Logger;
|
||||
}
|
||||
|
||||
export interface HmacVerifyResult {
|
||||
ok: boolean;
|
||||
reason?:
|
||||
| "stale"
|
||||
| "bad-signature"
|
||||
| "missing-headers"
|
||||
| "missing-timestamp"
|
||||
| "missing-signature"
|
||||
| "invalid-timestamp"
|
||||
| "invalid-signature-format";
|
||||
}
|
||||
|
||||
/**
|
||||
* NOTE: the canonical payload `path` MUST be a stable string agreed by the
|
||||
* signer AND the verifier. The canonical value is the ROUTE CONSTANT that
|
||||
* the handler is mounted at (e.g. `"/webhooks/deploy"`) — NOT `c.req.path`.
|
||||
*
|
||||
* Rationale: `c.req.path` varies with proxy configuration. A reverse proxy
|
||||
* that strips a path prefix, normalizes trailing slashes, or mounts this
|
||||
* service at a different base would surface a different observed path to
|
||||
* the handler than the one the signer hashed — producing silent
|
||||
* signature-verification failures that are painful to diagnose.
|
||||
*
|
||||
* Canonical contract:
|
||||
* - Signer (`.github/workflows/showcase_deploy.yml`): uses the ROUTE
|
||||
* CONSTANT `"/webhooks/deploy"` as the `path` component.
|
||||
* - Verifier (`src/http/webhooks/deploy.ts`): uses the same route
|
||||
* constant (`deps.webhookPath ?? route`), NOT `c.req.path`.
|
||||
*
|
||||
* If the route is renamed, BOTH the workflow signer AND the handler's
|
||||
* route constant MUST move in lockstep. The override via
|
||||
* `deps.webhookPath` exists precisely for the proxy case — callers mount
|
||||
* this service behind a rewrite and declare the path the signer used.
|
||||
*/
|
||||
export function canonicalPayload(
|
||||
method: string,
|
||||
path: string,
|
||||
timestamp: string,
|
||||
body: string,
|
||||
): string {
|
||||
const bodyHash = crypto.createHash("sha256").update(body).digest("hex");
|
||||
return `${method.toUpperCase()}|${path}|${timestamp}|${bodyHash}`;
|
||||
}
|
||||
|
||||
export function computeSignature(secret: string, canonical: string): string {
|
||||
return crypto.createHmac("sha256", secret).update(canonical).digest("hex");
|
||||
}
|
||||
|
||||
export function verifyHmac(input: HmacVerifyInput): HmacVerifyResult {
|
||||
// Trim the signature header defensively: some workflow senders / jq
|
||||
// invocations accidentally prefix or suffix whitespace (e.g. a trailing
|
||||
// newline from `$(…)` substitution). Without trimming we'd fall through
|
||||
// to `bad-signature` with no diagnostic, which is painful to debug.
|
||||
const signatureHeader = input.signatureHeader?.trim() ?? "";
|
||||
const timestamp = input.timestamp?.trim() ?? "";
|
||||
// Split the reason codes for missing timestamp vs missing signature
|
||||
// so operators can diagnose a signer that forgot one header without
|
||||
// reverse-engineering "missing-headers". The legacy `missing-headers`
|
||||
// reason is retained for the both-missing case (rare; usually means
|
||||
// a middleware stripped both) so existing dashboards keep working.
|
||||
if (!timestamp && !signatureHeader) {
|
||||
return { ok: false, reason: "missing-headers" };
|
||||
}
|
||||
if (!timestamp) {
|
||||
return { ok: false, reason: "missing-timestamp" };
|
||||
}
|
||||
if (!signatureHeader) {
|
||||
return { ok: false, reason: "missing-signature" };
|
||||
}
|
||||
// Require an integer — floats like "1700000000.5" are almost certainly
|
||||
// a bug on the signer side, and accepting them invites drift around the
|
||||
// skew boundary.
|
||||
const ts = Number(timestamp);
|
||||
if (!Number.isInteger(ts)) {
|
||||
return { ok: false, reason: "invalid-timestamp" };
|
||||
}
|
||||
const nowSec = input.nowSec?.() ?? Math.floor(Date.now() / 1000);
|
||||
const skew = input.maxSkewSec ?? 300;
|
||||
if (Math.abs(nowSec - ts) > skew) return { ok: false, reason: "stale" };
|
||||
// Lenient by design: accept both `sha256=<hex>` and a bare `<hex>`
|
||||
// string. Sender lint drift / manual curl testing has historically
|
||||
// flip-flopped on the prefix, and the canonical message already pins
|
||||
// the algorithm to sha256 — there's no security benefit to requiring
|
||||
// the prefix. If that changes (e.g. multi-algo support), remove this
|
||||
// `replace` and require the prefix explicitly.
|
||||
const providedHex = signatureHeader.replace(/^sha256=/, "");
|
||||
// Validate the hex shape before reaching for timingSafeEqual — an
|
||||
// upstream sender producing non-hex garbage (or base64 by mistake)
|
||||
// deserves a distinct reason code so operators can diagnose quickly
|
||||
// instead of chasing "bad-signature" for a malformed header.
|
||||
if (!/^[0-9a-f]+$/i.test(providedHex) || providedHex.length % 2 !== 0) {
|
||||
return { ok: false, reason: "invalid-signature-format" };
|
||||
}
|
||||
const canonical = canonicalPayload(
|
||||
input.method,
|
||||
input.path,
|
||||
timestamp,
|
||||
input.body,
|
||||
);
|
||||
for (const secret of input.secrets) {
|
||||
if (!secret) continue;
|
||||
const expected = computeSignature(secret, canonical);
|
||||
try {
|
||||
if (
|
||||
providedHex.length === expected.length &&
|
||||
crypto.timingSafeEqual(
|
||||
Buffer.from(providedHex, "hex"),
|
||||
Buffer.from(expected, "hex"),
|
||||
)
|
||||
) {
|
||||
return { ok: true };
|
||||
}
|
||||
} catch (err) {
|
||||
// Split error classification: benign length-mismatches (expected on
|
||||
// malformed hex) stay at debug to avoid log spam, while genuinely
|
||||
// unexpected crypto-layer failures (OOM, platform quirks, bad
|
||||
// inputs that slipped past the hex-shape regex) surface at warn
|
||||
// with a stable errorId so operators can alert on them and
|
||||
// distinguish real breakage from the noisy happy-path rejection.
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const isLengthMismatch =
|
||||
/input buffers must have the same byte length/i.test(message) ||
|
||||
/Input buffers must have the same length/i.test(message);
|
||||
if (isLengthMismatch) {
|
||||
input.logger?.debug("hmac.verify.compare-error", {
|
||||
reason: "length-mismatch",
|
||||
err: message,
|
||||
});
|
||||
} else {
|
||||
input.logger?.warn("hmac.verify.compare-error", {
|
||||
errorId: "HMAC_COMPARE_UNEXPECTED_ERROR",
|
||||
err: message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: false, reason: "bad-signature" };
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Tests for the Prometheus-format `/metrics` endpoint (spec §9 Phase 5).
|
||||
*/
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { createMetricsRegistry, renderPrometheus } from "./metrics.js";
|
||||
|
||||
describe("metrics registry", () => {
|
||||
it("exposes the five baseline counters as TYPE-annotated Prometheus text", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.inc("probe_runs", { dimension: "smoke" });
|
||||
reg.inc("probe_runs", { dimension: "smoke" });
|
||||
reg.inc("probe_runs", { dimension: "health" });
|
||||
reg.inc("alert_matches", { rule: "smoke-red-tick" });
|
||||
reg.inc("alert_sends", { target: "slack_webhook" });
|
||||
reg.inc("rule_reloads");
|
||||
reg.inc("hmac_failures");
|
||||
|
||||
const text = renderPrometheus(reg);
|
||||
|
||||
expect(text).toContain("# TYPE showcase_ops_probe_runs counter");
|
||||
expect(text).toContain('showcase_ops_probe_runs{dimension="smoke"} 2');
|
||||
expect(text).toContain('showcase_ops_probe_runs{dimension="health"} 1');
|
||||
expect(text).toContain("# TYPE showcase_ops_alert_matches counter");
|
||||
expect(text).toContain(
|
||||
'showcase_ops_alert_matches{rule="smoke-red-tick"} 1',
|
||||
);
|
||||
expect(text).toContain("# TYPE showcase_ops_alert_sends counter");
|
||||
expect(text).toContain("# TYPE showcase_ops_rule_reloads counter");
|
||||
// Anchor to line-start/end so `rule_reloads 1` isn't satisfied by e.g.
|
||||
// `rule_reloads 10`. Same rationale for hmac_failures below.
|
||||
expect(text).toMatch(/^showcase_ops_rule_reloads\s+1$/m);
|
||||
expect(text).toContain("# TYPE showcase_ops_hmac_failures counter");
|
||||
expect(text).toMatch(/^showcase_ops_hmac_failures\s+1$/m);
|
||||
});
|
||||
|
||||
it("emits HELP lines with a description for every metric", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.observe("probe_duration_ms", 10);
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_probe_runs .+/m);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_alert_matches .+/m);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_alert_sends .+/m);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_rule_reloads .+/m);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_hmac_failures .+/m);
|
||||
expect(text).toMatch(/^# HELP showcase_ops_probe_duration_ms .+/m);
|
||||
});
|
||||
|
||||
it("records histogram observations for probe latency", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.observe("probe_duration_ms", 42, { dimension: "smoke" });
|
||||
reg.observe("probe_duration_ms", 150, { dimension: "smoke" });
|
||||
reg.observe("probe_duration_ms", 2500, { dimension: "smoke" });
|
||||
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toContain("# TYPE showcase_ops_probe_duration_ms histogram");
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_bucket\{dimension="smoke",le="100"\}\s+1/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_bucket\{dimension="smoke",le="1000"\}\s+2/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_bucket\{dimension="smoke",le="\+Inf"\}\s+3/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_count\{dimension="smoke"\}\s+3/,
|
||||
);
|
||||
});
|
||||
|
||||
it("escapes label values with backslashes and quotes", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.inc("probe_runs", { dimension: 'a"b\\c' });
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toContain('showcase_ops_probe_runs{dimension="a\\"b\\\\c"}');
|
||||
});
|
||||
|
||||
it("escapes \\r and \\n in label values", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.inc("probe_runs", { dimension: "line1\r\nline2" });
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toContain(
|
||||
'showcase_ops_probe_runs{dimension="line1\\r\\nline2"}',
|
||||
);
|
||||
});
|
||||
|
||||
it("emits a zero-valued empty histogram with full bucket schema before any observation", () => {
|
||||
// Regression: prior to this we emitted `_sum`/`_count` with no labels
|
||||
// for empty histograms, then labelled series for populated ones —
|
||||
// Prometheus scrapers relying on consistent dimensionality would
|
||||
// silently drop one form. The empty shape must include every
|
||||
// configured upper bound + `+Inf` and a zero sum/count with no
|
||||
// labels.
|
||||
const reg = createMetricsRegistry();
|
||||
const text = renderPrometheus(reg);
|
||||
for (const bound of ["10", "50", "100", "500", "1000", "5000", "+Inf"]) {
|
||||
expect(text).toContain(
|
||||
`showcase_ops_probe_duration_ms_bucket{le="${bound}"} 0`,
|
||||
);
|
||||
}
|
||||
expect(text).toMatch(/^showcase_ops_probe_duration_ms_sum 0$/m);
|
||||
expect(text).toMatch(/^showcase_ops_probe_duration_ms_count 0$/m);
|
||||
});
|
||||
|
||||
it("merges `le` into existing labels for populated histograms (alphabetical sort — locked)", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.observe("probe_duration_ms", 5, { dimension: "smoke", key: "a" });
|
||||
const text = renderPrometheus(reg);
|
||||
// Alphabetical: dimension, key, le. This ordering is cosmetic — the
|
||||
// Prometheus parser is order-insensitive — but locked in a test so
|
||||
// dashboard templates / recording rules relying on this ordering
|
||||
// don't silently break on a sort refactor.
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_bucket\{dimension="smoke",key="a",le="10"\}\s+1/,
|
||||
);
|
||||
});
|
||||
|
||||
// HF-A5: `internal_backup_failures_total` is a first-class counter —
|
||||
// distinct series from probe_runs so backup failures don't pollute the
|
||||
// probe-run dashboards. Must register in COUNTER_NAMES (typecheck
|
||||
// enforces caller correctness) and must render in the Prometheus output.
|
||||
it("exposes internal_backup_failures_total as a dedicated counter", () => {
|
||||
const reg = createMetricsRegistry();
|
||||
reg.inc("internal_backup_failures_total");
|
||||
reg.inc("internal_backup_failures_total");
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toContain(
|
||||
"# TYPE showcase_ops_internal_backup_failures_total counter",
|
||||
);
|
||||
expect(text).toMatch(/^showcase_ops_internal_backup_failures_total\s+2$/m);
|
||||
// Must not have leaked into probe_runs.
|
||||
expect(text).toMatch(/^showcase_ops_probe_runs 0$/m);
|
||||
});
|
||||
|
||||
it("emits mixed label sets consistently across empty and populated series", () => {
|
||||
// Two histograms in one registry: one with observations, the other
|
||||
// empty (N/A today but guards against a future second histogram).
|
||||
// The populated one carries its labels; the empty-series shape must
|
||||
// not contaminate it (no stray unlabelled _sum/_count rows).
|
||||
const reg = createMetricsRegistry();
|
||||
reg.observe("probe_duration_ms", 50, { dimension: "smoke" });
|
||||
const text = renderPrometheus(reg);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_probe_duration_ms_sum\{dimension="smoke"\}\s+50/,
|
||||
);
|
||||
expect(text).not.toMatch(/^showcase_ops_probe_duration_ms_sum 0$/m);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* Minimal Prometheus-format metrics registry (spec §9 Phase 5).
|
||||
*
|
||||
* Zero external dependencies — we don't need prom-client's full feature
|
||||
* set; this service exposes a handful of counters and one histogram.
|
||||
* Keeping it in-house means no extra surface area to keep patched.
|
||||
*
|
||||
* Counters: probe_runs, alert_matches, alert_sends, rule_reloads,
|
||||
* webhook_rejections, hmac_failures (deprecated alias)
|
||||
* Histogram: probe_duration_ms (buckets: 10, 50, 100, 500, 1000, 5000 ms)
|
||||
*
|
||||
* All metrics carry `showcase_ops_` prefix on export.
|
||||
*
|
||||
* `webhook_rejections{reason=...}` replaces the earlier `hmac_failures`
|
||||
* counter. Every webhook rejection (HMAC and payload validation) now
|
||||
* increments the unified counter so dashboards can't miss a category.
|
||||
*
|
||||
* DEPRECATION: `hmac_failures` remains incrementable as an alias — any
|
||||
* `webhook_rejections` increment whose reason falls into `HMAC_REASONS`
|
||||
* also bumps `hmac_failures` so existing Grafana panels don't go dark.
|
||||
* New callers MUST use `webhook_rejections`. Panels that sum BOTH
|
||||
* counters will double-count HMAC rejections — switch to
|
||||
* `webhook_rejections` only.
|
||||
*
|
||||
* Sunset: remove the alias (and `hmac_failures` from COUNTER_NAMES /
|
||||
* COUNTER_HELP) once no production dashboard queries it. Track removal
|
||||
* intent in the showcase-ops deprecation backlog rather than leaving
|
||||
* it as a dangling "grace period" forever.
|
||||
*/
|
||||
|
||||
// HMAC-verification reason codes. Kept in sync with `HmacVerifyResult`
|
||||
// in ./hmac.ts — when we add/split/remove reasons there, mirror them
|
||||
// here so the `hmac_failures` deprecated alias still covers every HMAC-
|
||||
// category rejection. Non-HMAC reasons (invalid-json, invalid-payload,
|
||||
// unknown) must NOT appear here.
|
||||
const HMAC_REASONS = new Set([
|
||||
"stale",
|
||||
"bad-signature",
|
||||
"missing-headers",
|
||||
"missing-timestamp",
|
||||
"missing-signature",
|
||||
"invalid-timestamp",
|
||||
"invalid-signature-format",
|
||||
]);
|
||||
|
||||
const COUNTER_NAMES = [
|
||||
"probe_runs",
|
||||
"alert_matches",
|
||||
"alert_sends",
|
||||
"rule_reloads",
|
||||
"webhook_rejections",
|
||||
"hmac_failures",
|
||||
// HF-A5: distinct counter for `internal.backup.failed` bus events. Pre-fix
|
||||
// the orchestrator folded backup failures into `probe_runs` under a fake
|
||||
// `dimension=internal_backup` label, which polluted the probe-run
|
||||
// dashboards (probes that never actually ran inflated the probe_runs
|
||||
// series) and made alert rules keying on `probe_runs{dimension=~...}`
|
||||
// see phantom signal. Backup failures are a first-class, low-volume
|
||||
// counter on their own series.
|
||||
"internal_backup_failures_total",
|
||||
] as const;
|
||||
type CounterName = (typeof COUNTER_NAMES)[number];
|
||||
|
||||
const HISTOGRAM_NAMES = ["probe_duration_ms"] as const;
|
||||
type HistogramName = (typeof HISTOGRAM_NAMES)[number];
|
||||
|
||||
const HISTOGRAM_BUCKETS: Record<HistogramName, number[]> = {
|
||||
probe_duration_ms: [10, 50, 100, 500, 1000, 5000],
|
||||
};
|
||||
|
||||
const COUNTER_HELP: Record<CounterName, string> = {
|
||||
probe_runs: "Total probe executions grouped by dimension/key.",
|
||||
alert_matches: "Total rule evaluations that matched, grouped by rule id.",
|
||||
alert_sends: "Total alert deliveries grouped by target kind.",
|
||||
rule_reloads: "Total times the rule loader has reloaded from disk.",
|
||||
webhook_rejections:
|
||||
"Total webhook request rejections grouped by reason (HMAC verify + payload validation).",
|
||||
hmac_failures:
|
||||
"DEPRECATED. Alias of webhook_rejections filtered to HMAC-verification reasons. Prefer webhook_rejections{reason=...}.",
|
||||
internal_backup_failures_total:
|
||||
"Total internal.backup.failed bus emissions (S3 backup producer/uploader failures).",
|
||||
};
|
||||
|
||||
const HISTOGRAM_HELP: Record<HistogramName, string> = {
|
||||
probe_duration_ms: "Probe handler duration in milliseconds.",
|
||||
};
|
||||
|
||||
type Labels = Record<string, string>;
|
||||
|
||||
function labelKey(labels: Labels | undefined): string {
|
||||
if (!labels) return "";
|
||||
const keys = Object.keys(labels).sort();
|
||||
return keys.map((k) => `${k}=${labels[k]}`).join(",");
|
||||
}
|
||||
|
||||
function formatLabels(labels: Labels | undefined): string {
|
||||
if (!labels || Object.keys(labels).length === 0) return "";
|
||||
const pairs = Object.keys(labels)
|
||||
.sort()
|
||||
.map((k) => `${k}="${escapeLabelValue(labels[k]!)}"`);
|
||||
return `{${pairs.join(",")}}`;
|
||||
}
|
||||
|
||||
function formatLabelsWithLe(labels: Labels | undefined, le: string): string {
|
||||
const merged: Labels = { ...(labels ?? {}), le };
|
||||
return formatLabels(merged);
|
||||
}
|
||||
|
||||
function escapeLabelValue(v: string): string {
|
||||
// Order matters: escape backslashes first so the subsequent replacements
|
||||
// don't re-escape their own inserted backslashes. `\r` is handled in
|
||||
// addition to `\n` because Windows-origin label values (file paths,
|
||||
// commit messages) otherwise produce malformed exposition output.
|
||||
return v
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/"/g, '\\"')
|
||||
.replace(/\n/g, "\\n")
|
||||
.replace(/\r/g, "\\r");
|
||||
}
|
||||
|
||||
interface CounterSeries {
|
||||
labels: Labels | undefined;
|
||||
value: number;
|
||||
}
|
||||
|
||||
interface HistogramSeries {
|
||||
labels: Labels | undefined;
|
||||
count: number;
|
||||
sum: number;
|
||||
buckets: Map<number, number>; // upper-bound -> cumulative count
|
||||
}
|
||||
|
||||
export interface MetricsRegistry {
|
||||
inc(name: CounterName, labels?: Labels): void;
|
||||
observe(name: HistogramName, value: number, labels?: Labels): void;
|
||||
/** For testing — read the raw counter series map. */
|
||||
_counters(): Map<CounterName, Map<string, CounterSeries>>;
|
||||
_histograms(): Map<HistogramName, Map<string, HistogramSeries>>;
|
||||
}
|
||||
|
||||
export function createMetricsRegistry(): MetricsRegistry {
|
||||
const counters = new Map<CounterName, Map<string, CounterSeries>>();
|
||||
for (const n of COUNTER_NAMES) counters.set(n, new Map());
|
||||
|
||||
const histograms = new Map<HistogramName, Map<string, HistogramSeries>>();
|
||||
for (const n of HISTOGRAM_NAMES) histograms.set(n, new Map());
|
||||
|
||||
function incOne(name: CounterName, labels: Labels | undefined): void {
|
||||
const bucket = counters.get(name)!;
|
||||
const k = labelKey(labels);
|
||||
const existing = bucket.get(k);
|
||||
if (existing) {
|
||||
existing.value += 1;
|
||||
} else {
|
||||
bucket.set(k, { labels, value: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
inc(name, labels) {
|
||||
incOne(name, labels);
|
||||
// Deprecated alias mirror: a `webhook_rejections` increment whose
|
||||
// reason falls into the HMAC-verification category also bumps
|
||||
// `hmac_failures` so existing dashboards don't go dark during the
|
||||
// deprecation window. The mirror is skipped for non-HMAC reasons
|
||||
// (invalid-json, invalid-payload, ...) to preserve the alias's
|
||||
// historical meaning.
|
||||
if (name === "webhook_rejections") {
|
||||
const reason = labels?.reason;
|
||||
if (reason && HMAC_REASONS.has(reason)) {
|
||||
incOne("hmac_failures", labels);
|
||||
}
|
||||
}
|
||||
},
|
||||
observe(name, value, labels) {
|
||||
const bucket = histograms.get(name)!;
|
||||
const k = labelKey(labels);
|
||||
let series = bucket.get(k);
|
||||
if (!series) {
|
||||
const buckets = new Map<number, number>();
|
||||
for (const b of HISTOGRAM_BUCKETS[name]) buckets.set(b, 0);
|
||||
series = { labels, count: 0, sum: 0, buckets };
|
||||
bucket.set(k, series);
|
||||
}
|
||||
series.count += 1;
|
||||
series.sum += value;
|
||||
for (const b of HISTOGRAM_BUCKETS[name]) {
|
||||
if (value <= b) series.buckets.set(b, series.buckets.get(b)! + 1);
|
||||
}
|
||||
},
|
||||
_counters() {
|
||||
return counters;
|
||||
},
|
||||
_histograms() {
|
||||
return histograms;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function renderPrometheus(reg: MetricsRegistry): string {
|
||||
const lines: string[] = [];
|
||||
for (const name of COUNTER_NAMES) {
|
||||
const bucket = reg._counters().get(name)!;
|
||||
lines.push(`# HELP showcase_ops_${name} ${COUNTER_HELP[name]}`);
|
||||
lines.push(`# TYPE showcase_ops_${name} counter`);
|
||||
if (bucket.size === 0) {
|
||||
lines.push(`showcase_ops_${name} 0`);
|
||||
continue;
|
||||
}
|
||||
for (const series of bucket.values()) {
|
||||
lines.push(
|
||||
`showcase_ops_${name}${formatLabels(series.labels)} ${series.value}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const name of HISTOGRAM_NAMES) {
|
||||
const bucket = reg._histograms().get(name)!;
|
||||
lines.push(`# HELP showcase_ops_${name} ${HISTOGRAM_HELP[name]}`);
|
||||
lines.push(`# TYPE showcase_ops_${name} histogram`);
|
||||
if (bucket.size === 0) {
|
||||
// Consistency with counters: always emit a zero-count series so a
|
||||
// TYPE/HELP line is never orphaned. Includes every configured bucket
|
||||
// plus the mandatory `+Inf` so scrapers see the full schema.
|
||||
for (const b of HISTOGRAM_BUCKETS[name]) {
|
||||
lines.push(
|
||||
`showcase_ops_${name}_bucket${formatLabelsWithLe(undefined, String(b))} 0`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`showcase_ops_${name}_bucket${formatLabelsWithLe(undefined, "+Inf")} 0`,
|
||||
);
|
||||
lines.push(`showcase_ops_${name}_sum 0`);
|
||||
lines.push(`showcase_ops_${name}_count 0`);
|
||||
continue;
|
||||
}
|
||||
for (const series of bucket.values()) {
|
||||
for (const b of HISTOGRAM_BUCKETS[name]) {
|
||||
lines.push(
|
||||
`showcase_ops_${name}_bucket${formatLabelsWithLe(series.labels, String(b))} ${series.buckets.get(b)!}`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`showcase_ops_${name}_bucket${formatLabelsWithLe(series.labels, "+Inf")} ${series.count}`,
|
||||
);
|
||||
lines.push(
|
||||
`showcase_ops_${name}_sum${formatLabels(series.labels)} ${series.sum}`,
|
||||
);
|
||||
lines.push(
|
||||
`showcase_ops_${name}_count${formatLabels(series.labels)} ${series.count}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n") + "\n";
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { buildServer } from "./server.js";
|
||||
import { logger } from "../logger.js";
|
||||
import { createMetricsRegistry } from "./metrics.js";
|
||||
import type { PbClient } from "../storage/pb-client.js";
|
||||
|
||||
function fakePb(healthy: boolean): PbClient {
|
||||
return {
|
||||
getOne: async () => null,
|
||||
getFirst: async () => null,
|
||||
list: async () => ({
|
||||
page: 1,
|
||||
perPage: 0,
|
||||
totalPages: 0,
|
||||
totalItems: 0,
|
||||
items: [],
|
||||
}),
|
||||
create: async () => ({}) as never,
|
||||
update: async () => ({}) as never,
|
||||
upsertByField: async () => ({}) as never,
|
||||
delete: async () => {},
|
||||
deleteByFilter: async () => 0,
|
||||
health: async () => healthy,
|
||||
createBackup: async () => {},
|
||||
downloadBackup: async () => new Uint8Array(),
|
||||
deleteBackup: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
describe("http/server", () => {
|
||||
it("GET /health returns 200 when pb up, loop alive, rules>0", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
status: string;
|
||||
pb: string;
|
||||
rules: number;
|
||||
};
|
||||
expect(body.status).toBe("ok");
|
||||
expect(body.pb).toBe("ok");
|
||||
expect(body.rules).toBe(1);
|
||||
});
|
||||
|
||||
it("GET /health returns 503 with loop:no-jobs when scheduler has zero entries", async () => {
|
||||
// Regression: if rule-loader crashes or loads zero rules, the HTTP
|
||||
// server still reports healthy because loopAlive/schedulerStarted
|
||||
// don't care about job count. Require schedulerJobCount > 0 so
|
||||
// this pathological state surfaces in /health.
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
schedulerStarted: () => true,
|
||||
schedulerJobCount: () => 0,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
const body = (await res.json()) as {
|
||||
loop: string;
|
||||
status: string;
|
||||
schedulerJobs: number;
|
||||
};
|
||||
expect(body.loop).toBe("no-jobs");
|
||||
expect(body.schedulerJobs).toBe(0);
|
||||
expect(body.status).toBe("degraded");
|
||||
});
|
||||
|
||||
it("GET /health returns 503 with loop:stopped when schedulerIsStopped is true even if alive was never flipped", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true, // legacy flag, still true
|
||||
schedulerStarted: () => true,
|
||||
schedulerIsStopped: () => true, // but scheduler.stop() completed
|
||||
schedulerJobCount: () => 0,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
const body = (await res.json()) as { loop: string };
|
||||
expect(body.loop).toBe("stopped");
|
||||
});
|
||||
|
||||
it("GET /health returns 200 when all scheduler signals are healthy", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 3,
|
||||
loopAlive: () => true,
|
||||
schedulerStarted: () => true,
|
||||
schedulerIsStopped: () => false,
|
||||
schedulerJobCount: () => 5,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
loop: string;
|
||||
schedulerJobs: number;
|
||||
};
|
||||
expect(body.loop).toBe("ok");
|
||||
expect(body.schedulerJobs).toBe(5);
|
||||
});
|
||||
|
||||
it("GET /health returns 503 when pb down", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(false),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
});
|
||||
|
||||
it("GET /health returns 503 when no rules loaded", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 0,
|
||||
loopAlive: () => true,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
});
|
||||
|
||||
it("GET /health returns 503 when loop not alive", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => false,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
});
|
||||
|
||||
it("GET /health reports loop:starting (503) when scheduler has not started", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
schedulerStarted: () => false,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
const body = (await res.json()) as { loop: string; status: string };
|
||||
expect(body.loop).toBe("starting");
|
||||
expect(body.status).toBe("degraded");
|
||||
});
|
||||
|
||||
it("GET /health reports loop:ok when scheduler has started and is alive", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
schedulerStarted: () => true,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { loop: string };
|
||||
expect(body.loop).toBe("ok");
|
||||
});
|
||||
|
||||
it("GET /health reports loop:stopped (503) when loop explicitly stopped even if started", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => false,
|
||||
schedulerStarted: () => true,
|
||||
});
|
||||
const res = await app.request("/health");
|
||||
expect(res.status).toBe(503);
|
||||
const body = (await res.json()) as { loop: string };
|
||||
expect(body.loop).toBe("stopped");
|
||||
});
|
||||
|
||||
it("GET /metrics exposes Prometheus-format counters when metrics is provided", async () => {
|
||||
const metrics = createMetricsRegistry();
|
||||
metrics.inc("probe_runs", { dimension: "smoke" });
|
||||
metrics.inc("hmac_failures");
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
metrics,
|
||||
});
|
||||
const res = await app.request("/metrics");
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("content-type")).toContain("text/plain");
|
||||
const body = await res.text();
|
||||
expect(body).toContain('showcase_ops_probe_runs{dimension="smoke"} 1');
|
||||
expect(body).toContain("showcase_ops_hmac_failures 1");
|
||||
});
|
||||
|
||||
it("GET /metrics returns 404 when metrics registry is absent", async () => {
|
||||
const app = buildServer({
|
||||
pb: fakePb(true),
|
||||
logger,
|
||||
ruleCount: () => 1,
|
||||
loopAlive: () => true,
|
||||
});
|
||||
const res = await app.request("/metrics");
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { Hono } from "hono";
|
||||
import type { PbClient } from "../storage/pb-client.js";
|
||||
import type { Logger } from "../types/index.js";
|
||||
import type { TypedEventBus } from "../events/event-bus.js";
|
||||
import { registerDeployWebhook } from "./webhooks/deploy.js";
|
||||
import { renderPrometheus, type MetricsRegistry } from "./metrics.js";
|
||||
|
||||
export interface ServerDeps {
|
||||
pb: PbClient;
|
||||
logger: Logger;
|
||||
ruleCount: () => number;
|
||||
/**
|
||||
* Historically exposed as `loop: ok|stopped` on /health, but the flag
|
||||
* only reflected whether `orchestrator.stop()` had been called — it
|
||||
* never reflected actual scheduler/probe-loop liveness. Kept as an
|
||||
* optional knob for backwards compatibility; when absent the /health
|
||||
* response omits the `loop` field rather than lying about it.
|
||||
*/
|
||||
loopAlive?: () => boolean;
|
||||
/**
|
||||
* Callback returning `true` once the scheduler has been started and is
|
||||
* actively running. When supplied, /health's `loop` field reflects
|
||||
* `schedulerStarted && loopAlive` instead of the weaker `loopAlive`
|
||||
* alone — this prevents the endpoint reporting `loop: ok` during the
|
||||
* narrow boot window between server-listen and scheduler-start where a
|
||||
* crashed scheduler otherwise stays invisible.
|
||||
*/
|
||||
schedulerStarted?: () => boolean;
|
||||
/**
|
||||
* Number of entries currently registered with the scheduler. When
|
||||
* supplied, /health treats zero as a hard 503 — a running HTTP server
|
||||
* with no cron jobs means the rule loader silently crashed (or loaded
|
||||
* zero rules) and no probes will tick. Without this callback the
|
||||
* endpoint still reports 200 in that pathological state.
|
||||
*/
|
||||
schedulerJobCount?: () => number;
|
||||
/**
|
||||
* `true` once `scheduler.stop()` has completed. When supplied, /health
|
||||
* returns 503 with `loop: "stopped"` rather than relying on the weaker
|
||||
* `loopAlive` signal alone, which closes the post-shutdown window
|
||||
* where /health can otherwise report healthy for a few seconds after
|
||||
* stop() is called.
|
||||
*/
|
||||
schedulerIsStopped?: () => boolean;
|
||||
/** Event bus for webhook emissions. Optional so older callers (tests) don't break. */
|
||||
bus?: TypedEventBus;
|
||||
/** HMAC secrets for signed webhooks. If unset, webhook routes are not registered. */
|
||||
webhookSecrets?: string[];
|
||||
/** Metrics registry. When provided, `/metrics` returns Prometheus text. */
|
||||
metrics?: MetricsRegistry;
|
||||
}
|
||||
|
||||
export function buildServer(deps: ServerDeps): Hono {
|
||||
const app = new Hono();
|
||||
|
||||
if (deps.bus && deps.webhookSecrets && deps.webhookSecrets.length > 0) {
|
||||
registerDeployWebhook(app, {
|
||||
bus: deps.bus,
|
||||
logger: deps.logger,
|
||||
secrets: deps.webhookSecrets,
|
||||
metrics: deps.metrics,
|
||||
});
|
||||
}
|
||||
|
||||
if (deps.metrics) {
|
||||
const registry = deps.metrics;
|
||||
// NOTE: `/metrics` is intentionally unauthenticated so in-cluster
|
||||
// Prometheus scrapers can reach it without credential plumbing. If
|
||||
// this service is ever exposed directly to the public internet, this
|
||||
// route leaks internal counters (probe cadence, alert volume, HMAC
|
||||
// failure rate) and must be locked down (e.g. private network ACL,
|
||||
// reverse-proxy basic auth, or token-based auth). Tracked as a
|
||||
// hardening item post-v1 rather than a default; until then, operators
|
||||
// must keep this service behind Railway's private network.
|
||||
app.get("/metrics", (c) => {
|
||||
const body = renderPrometheus(registry);
|
||||
return c.body(body, 200, { "Content-Type": "text/plain; version=0.0.4" });
|
||||
});
|
||||
}
|
||||
|
||||
app.get("/health", async (c) => {
|
||||
const pbOk = await deps.pb.health();
|
||||
const ruleCount = deps.ruleCount();
|
||||
// Loop-alive semantics:
|
||||
// - `schedulerStarted` (optional): true once start() returned.
|
||||
// - `schedulerIsStopped` (optional): true once stop() completed —
|
||||
// takes priority over `loopAlive` so post-shutdown responses are
|
||||
// accurate.
|
||||
// - `schedulerJobCount` (optional): if supplied AND zero, /health
|
||||
// returns 503. An HTTP server up with no cron entries means the
|
||||
// scheduler is ticking nothing — a silent outage we previously
|
||||
// reported as healthy.
|
||||
// - `loopAlive`: legacy flag flipped by orchestrator.stop().
|
||||
// Order: stopped > !started > !alive > jobCount==0 > alive.
|
||||
const alive = deps.loopAlive?.() ?? true;
|
||||
const started = deps.schedulerStarted?.() ?? true;
|
||||
const schedulerStopped = deps.schedulerIsStopped?.() ?? false;
|
||||
const jobCount = deps.schedulerJobCount?.();
|
||||
const jobCountOk = jobCount === undefined ? true : jobCount > 0;
|
||||
const loopOk = !schedulerStopped && started && alive && jobCountOk;
|
||||
const loopLabel = schedulerStopped
|
||||
? "stopped"
|
||||
: !started
|
||||
? "starting"
|
||||
: !alive
|
||||
? "stopped"
|
||||
: !jobCountOk
|
||||
? "no-jobs"
|
||||
: "ok";
|
||||
const ok = pbOk && loopOk && ruleCount > 0;
|
||||
return c.json(
|
||||
{
|
||||
status: ok ? "ok" : "degraded",
|
||||
pb: pbOk ? "ok" : "down",
|
||||
loop: loopLabel,
|
||||
rules: ruleCount,
|
||||
...(jobCount !== undefined ? { schedulerJobs: jobCount } : {}),
|
||||
},
|
||||
ok ? 200 : 503,
|
||||
);
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,771 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import { registerDeployWebhook } from "./deploy.js";
|
||||
import {
|
||||
createEventBus,
|
||||
type DeployResultEvent,
|
||||
} from "../../events/event-bus.js";
|
||||
import { canonicalPayload, computeSignature } from "../hmac.js";
|
||||
import { createMetricsRegistry, renderPrometheus } from "../metrics.js";
|
||||
import { logger } from "../../logger.js";
|
||||
|
||||
const NOW = 1_700_000_000;
|
||||
const PATH = "/webhooks/deploy";
|
||||
const SECRET = "primary";
|
||||
|
||||
function signed(
|
||||
body: string,
|
||||
ts = NOW,
|
||||
): { headers: Record<string, string>; body: string } {
|
||||
const canonical = canonicalPayload("POST", PATH, String(ts), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
return {
|
||||
body,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(ts),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildApp(): {
|
||||
app: Hono;
|
||||
bus: ReturnType<typeof createEventBus>;
|
||||
seen: DeployResultEvent[];
|
||||
} {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const seen: DeployResultEvent[] = [];
|
||||
bus.on("deploy.result", (e) => {
|
||||
seen.push(e);
|
||||
});
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
});
|
||||
return { app, bus, seen };
|
||||
}
|
||||
|
||||
describe("POST /webhooks/deploy", () => {
|
||||
let app: Hono;
|
||||
let seen: DeployResultEvent[];
|
||||
|
||||
beforeEach(() => {
|
||||
const built = buildApp();
|
||||
app = built.app;
|
||||
seen = built.seen;
|
||||
});
|
||||
|
||||
it("accepts a valid signed payload and emits deploy.result", async () => {
|
||||
const payload = JSON.stringify({
|
||||
runId: "42",
|
||||
runUrl: "https://github.com/x/y/actions/runs/42",
|
||||
services: ["a", "b"],
|
||||
failed: [],
|
||||
succeeded: ["a", "b"],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(202);
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0].runId).toBe("42");
|
||||
expect(seen[0].services).toEqual(["a", "b"]);
|
||||
expect(seen[0].succeeded).toEqual(["a", "b"]);
|
||||
expect(seen[0].cancelled).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects a stale timestamp with 401", async () => {
|
||||
const payload = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body } = signed(payload, NOW - 10_000);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(401);
|
||||
expect(seen).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects a wrong signature with 401", async () => {
|
||||
const payload = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": "sha256=deadbeef",
|
||||
},
|
||||
body: payload,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
expect(seen).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("rejects missing signature header with 401", async () => {
|
||||
const payload = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
},
|
||||
body: payload,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("rejects invalid JSON body with 400", async () => {
|
||||
const body = "not-json";
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("rejects payload missing required fields with 400 including zod-flatten detail", async () => {
|
||||
const body = JSON.stringify({ runId: "1" });
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
const parsed = (await res.json()) as {
|
||||
reason: string;
|
||||
errors?: { fieldErrors?: Record<string, string[]> };
|
||||
};
|
||||
expect(parsed.reason).toBe("invalid-payload");
|
||||
// Flatten includes field-level issues so a signer can self-diagnose
|
||||
// without reading the ops service log.
|
||||
expect(parsed.errors).toBeDefined();
|
||||
expect(parsed.errors!.fieldErrors).toBeDefined();
|
||||
});
|
||||
|
||||
it("rejects a javascript: runUrl scheme with 400", async () => {
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
runUrl: "javascript:alert(1)",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("accepts an http runUrl scheme", async () => {
|
||||
const body = JSON.stringify({
|
||||
runId: "2",
|
||||
runUrl: "http://example.com/run/2",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(202);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /webhooks/deploy — webhook_rejections metric wiring", () => {
|
||||
it("increments webhook_rejections on stale timestamp (and mirrors to hmac_failures)", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers } = signed(body, NOW - 10_000);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(401);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="stale"\}\s+1/,
|
||||
);
|
||||
// Deprecated alias still populated for HMAC-category reasons.
|
||||
expect(text).toMatch(/showcase_ops_hmac_failures\{reason="stale"\}\s+1/);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections on bad signature", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": "sha256=deadbeef",
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="bad-signature"\}\s+1/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_hmac_failures\{reason="bad-signature"\}\s+1/,
|
||||
);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections on missing timestamp (split-reason: missing-timestamp)", async () => {
|
||||
// F3.3: timestamp-absent and signature-absent get distinct reasons
|
||||
// so metrics can disambiguate (an ops alert on "signer dropped
|
||||
// timestamp" is very different from "signer dropped signature").
|
||||
// The legacy `missing-headers` code is retained only for the
|
||||
// both-missing case.
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
// no timestamp
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="missing-timestamp"\}\s+1/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_hmac_failures\{reason="missing-timestamp"\}\s+1/,
|
||||
);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections on missing-headers (both absent) with combined reason", async () => {
|
||||
// F3.3: combined `missing-headers` reason preserved for the both-
|
||||
// missing case (legacy, rare; usually a middleware stripped both).
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
// no timestamp, no signature
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="missing-headers"\}\s+1/,
|
||||
);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_hmac_failures\{reason="missing-headers"\}\s+1/,
|
||||
);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections on missing signature (split-reason: missing-signature)", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({
|
||||
runId: "1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
// no signature
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="missing-signature"\}\s+1/,
|
||||
);
|
||||
// Deprecated alias still populated for HMAC-category reasons —
|
||||
// `missing-signature` is in HMAC_REASONS.
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_hmac_failures\{reason="missing-signature"\}\s+1/,
|
||||
);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections with reason=invalid-json on bad body (NOT mirrored to hmac_failures)", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = "not-json";
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="invalid-json"\}\s+1/,
|
||||
);
|
||||
// invalid-json is NOT an HMAC-verify reason — hmac_failures must not be
|
||||
// bumped for this category (it's a body-decode fault, post-verify).
|
||||
expect(text).not.toMatch(
|
||||
/showcase_ops_hmac_failures\{reason="invalid-json"\}/,
|
||||
);
|
||||
});
|
||||
|
||||
it("increments webhook_rejections with reason=invalid-payload when schema rejects", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const metrics = createMetricsRegistry();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
metrics,
|
||||
});
|
||||
const body = JSON.stringify({ runId: "1" }); // missing required fields
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), body);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
const text = renderPrometheus(metrics);
|
||||
expect(text).toMatch(
|
||||
/showcase_ops_webhook_rejections\{reason="invalid-payload"\}\s+1/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /webhooks/deploy — gateSkipped pass-through", () => {
|
||||
it("accepts gateSkipped: true and propagates to the emitted event", async () => {
|
||||
const { app, seen } = buildApp();
|
||||
const payload = JSON.stringify({
|
||||
runId: "gate-1",
|
||||
services: ["a", "b"],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
gateSkipped: true,
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(202);
|
||||
expect(seen).toHaveLength(1);
|
||||
expect(seen[0]!.gateSkipped).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts payload without gateSkipped and emits undefined", async () => {
|
||||
const { app, seen } = buildApp();
|
||||
const payload = JSON.stringify({
|
||||
runId: "gate-0",
|
||||
services: ["a"],
|
||||
failed: [],
|
||||
succeeded: ["a"],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(202);
|
||||
expect(seen[0]!.gateSkipped).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects non-boolean gateSkipped with 400 (strict schema)", async () => {
|
||||
const { app } = buildApp();
|
||||
const payload = JSON.stringify({
|
||||
runId: "gate-bad",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
gateSkipped: "yes",
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /webhooks/deploy — idempotency (composite runId+bodySha)", () => {
|
||||
it("returns 200 (not 202) on duplicate runId+body and does NOT re-emit", async () => {
|
||||
const { app, seen } = buildApp();
|
||||
const payload = JSON.stringify({
|
||||
runId: "dup-1",
|
||||
services: ["a"],
|
||||
failed: [],
|
||||
succeeded: ["a"],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const first = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(first.status).toBe(202);
|
||||
// Identical payload + identical NOW means the signature is identical
|
||||
// too — we're exercising the dedupe path, which runs AFTER signature
|
||||
// verification and keys on runId rather than anything timestamp-
|
||||
// dependent. (Previous comment here claimed "re-sign with a fresh
|
||||
// timestamp"; that was misleading since the test reuses NOW.)
|
||||
const second = signed(payload);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: second.headers,
|
||||
body: second.body,
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const parsed = (await res.json()) as { ok: boolean; duplicate: boolean };
|
||||
expect(parsed.duplicate).toBe(true);
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("same runId with DIFFERENT body re-emits (composite key guards against fork/re-run replay)", async () => {
|
||||
// F3.5: previously the dedupe key was `runId` alone. A workflow
|
||||
// re-run (which preserves runId) with a different service list
|
||||
// would have been silently dropped. The composite
|
||||
// `runId + sha256(body)` key ensures a real payload change
|
||||
// generates a fresh event.
|
||||
const { app, seen } = buildApp();
|
||||
const first = JSON.stringify({
|
||||
runId: "rerun-1",
|
||||
services: ["a"],
|
||||
failed: [],
|
||||
succeeded: ["a"],
|
||||
cancelled: false,
|
||||
});
|
||||
const second = JSON.stringify({
|
||||
runId: "rerun-1",
|
||||
services: ["a", "b"],
|
||||
failed: ["b"],
|
||||
succeeded: ["a"],
|
||||
cancelled: false,
|
||||
});
|
||||
{
|
||||
const { headers, body } = signed(first);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body,
|
||||
});
|
||||
expect(res.status).toBe(202);
|
||||
}
|
||||
{
|
||||
const { headers, body } = signed(second);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body,
|
||||
});
|
||||
// Body differs → composite key differs → must NOT be dedup'd.
|
||||
expect(res.status).toBe(202);
|
||||
}
|
||||
expect(seen).toHaveLength(2);
|
||||
expect(seen[0]!.services).toEqual(["a"]);
|
||||
expect(seen[1]!.services).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("treats runIds independently — different runId always emits", async () => {
|
||||
const { app, seen } = buildApp();
|
||||
for (const runId of ["r1", "r2", "r3"]) {
|
||||
const payload = JSON.stringify({
|
||||
runId,
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(202);
|
||||
}
|
||||
expect(seen.map((e) => e.runId)).toEqual(["r1", "r2", "r3"]);
|
||||
});
|
||||
|
||||
it("honors dedupeSize=0 (disabled) — all posts re-emit", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const seen: DeployResultEvent[] = [];
|
||||
bus.on("deploy.result", (e) => seen.push(e));
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
dedupeSize: 0,
|
||||
});
|
||||
const payload = JSON.stringify({
|
||||
runId: "same",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
const { headers, body } = signed(payload);
|
||||
const res = await app.request(PATH, { method: "POST", headers, body });
|
||||
expect(res.status).toBe(202);
|
||||
}
|
||||
expect(seen).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("touch-on-read: frequently-seen runId survives eviction pressure", async () => {
|
||||
// Regression: the dedupe cache comment claimed "re-seeing refreshes LRU"
|
||||
// but the prior implementation only touched on `record()` (first-seen),
|
||||
// so a hot id would get evicted just like a cold one. We now touch on
|
||||
// read too — exercise it by filling past capacity and confirming the
|
||||
// hot id still deduplicates.
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const seen: DeployResultEvent[] = [];
|
||||
bus.on("deploy.result", (e) => seen.push(e));
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
dedupeSize: 3,
|
||||
});
|
||||
async function post(runId: string): Promise<number> {
|
||||
const body = JSON.stringify({
|
||||
runId,
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
const { headers, body: b } = signed(body);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: b,
|
||||
});
|
||||
return res.status;
|
||||
}
|
||||
// Insert "hot" then fill the cache with cold ids, re-reading "hot"
|
||||
// each round so touch-on-read moves it to the tail.
|
||||
expect(await post("hot")).toBe(202);
|
||||
for (const cold of ["c1", "c2"]) {
|
||||
expect(await post(cold)).toBe(202);
|
||||
// Re-post "hot" as a duplicate — should be dedup'd (200) AND re-
|
||||
// promoted by touch-on-read.
|
||||
expect(await post("hot")).toBe(200);
|
||||
}
|
||||
// Add enough cold ids to overflow cap — hot must still be dedup'd.
|
||||
expect(await post("c3")).toBe(202);
|
||||
expect(await post("c4")).toBe(202);
|
||||
// If hot had been evicted, this would re-emit (202). With touch-on-
|
||||
// read + eviction victim being the least-recent non-hot id, hot
|
||||
// remains and we get 200.
|
||||
expect(await post("hot")).toBe(200);
|
||||
// Hot emitted once total.
|
||||
expect(seen.filter((e) => e.runId === "hot")).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /webhooks/deploy — webhookPath override", () => {
|
||||
it("honors webhookPath override when signer signs a proxy-mounted path", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
const seen: DeployResultEvent[] = [];
|
||||
bus.on("deploy.result", (e) => seen.push(e));
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
webhookPath: "/proxy/webhooks/deploy",
|
||||
});
|
||||
const payload = JSON.stringify({
|
||||
runId: "proxy-1",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
// Sender signs the externally-visible path, not the internal route.
|
||||
const canonical = canonicalPayload(
|
||||
"POST",
|
||||
"/proxy/webhooks/deploy",
|
||||
String(NOW),
|
||||
payload,
|
||||
);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body: payload,
|
||||
});
|
||||
expect(res.status).toBe(202);
|
||||
expect(seen).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rejects when webhookPath override doesn't match the signer", async () => {
|
||||
const app = new Hono();
|
||||
const bus = createEventBus();
|
||||
registerDeployWebhook(app, {
|
||||
bus,
|
||||
logger,
|
||||
secrets: [SECRET],
|
||||
nowSec: () => NOW,
|
||||
webhookPath: "/proxy/webhooks/deploy",
|
||||
});
|
||||
const payload = JSON.stringify({
|
||||
runId: "proxy-bad",
|
||||
services: [],
|
||||
failed: [],
|
||||
succeeded: [],
|
||||
cancelled: false,
|
||||
});
|
||||
// Sender signs the literal route — mismatch with the configured path.
|
||||
const canonical = canonicalPayload("POST", PATH, String(NOW), payload);
|
||||
const sig = computeSignature(SECRET, canonical);
|
||||
const res = await app.request(PATH, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"X-Ops-Timestamp": String(NOW),
|
||||
"X-Ops-Signature": `sha256=${sig}`,
|
||||
},
|
||||
body: payload,
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
import crypto from "node:crypto";
|
||||
import type { Hono } from "hono";
|
||||
import { z } from "zod";
|
||||
import { verifyHmac } from "../hmac.js";
|
||||
import type { MetricsRegistry } from "../metrics.js";
|
||||
import type {
|
||||
TypedEventBus,
|
||||
DeployResultEvent,
|
||||
} from "../../events/event-bus.js";
|
||||
import type { Logger } from "../../types/index.js";
|
||||
|
||||
export interface DeployWebhookDeps {
|
||||
bus: TypedEventBus;
|
||||
logger: Logger;
|
||||
/** Ordered list of HMAC secrets; first is primary, rest enable rotation. */
|
||||
secrets: string[];
|
||||
/** Allowed clock skew in seconds (default 300). */
|
||||
maxSkewSec?: number;
|
||||
/** Override for tests. */
|
||||
nowSec?: () => number;
|
||||
/** Optional metrics registry — when provided, webhook_rejections is incremented for every rejection. */
|
||||
metrics?: MetricsRegistry;
|
||||
/**
|
||||
* Canonical path used when verifying HMAC signatures. Defaults to the
|
||||
* literal route path (`/webhooks/deploy`). Override when the service is
|
||||
* mounted behind a prefix (e.g. Railway proxy) and the sender signs the
|
||||
* externally-visible path rather than whatever Hono sees internally.
|
||||
* Must match what the workflow signer uses.
|
||||
*/
|
||||
webhookPath?: string;
|
||||
/**
|
||||
* Max number of recently-processed runIds remembered for idempotency.
|
||||
* Defaults to 500 (raised from 100 to absorb 17-service × 2-retry ×
|
||||
* burst-day traffic without LRU churn). Set to 0 to disable dedupe
|
||||
* entirely.
|
||||
*/
|
||||
dedupeSize?: number;
|
||||
}
|
||||
|
||||
const deployPayloadSchema = z
|
||||
.object({
|
||||
runId: z.string().min(1),
|
||||
runUrl: z
|
||||
.string()
|
||||
.url()
|
||||
// Reject `javascript:`, `data:`, `file:`, etc. The field ends up
|
||||
// rendered as a link in Slack / dashboard UIs; a signed sender
|
||||
// with a typo (or compromised secret) should not be able to
|
||||
// trick downstream consumers into clicking a script URL.
|
||||
.refine((u) => /^https?:\/\//i.test(u), {
|
||||
message: "runUrl must be http(s)",
|
||||
})
|
||||
.optional(),
|
||||
services: z.array(z.string()),
|
||||
failed: z.array(z.string()),
|
||||
succeeded: z.array(z.string()),
|
||||
cancelled: z.boolean(),
|
||||
// `gateSkipped: true` means the workflow reached the report job but
|
||||
// the build matrix never ran (e.g. lockfile gate failed). Treated as a
|
||||
// distinct signal downstream from an all-services failure.
|
||||
gateSkipped: z.boolean().optional(),
|
||||
// Optional free-form discriminator co-emitted with `gateSkipped: true`
|
||||
// (`lockfile-failed`, `lockfile-cancelled`, `verify-image-refs-failed`,
|
||||
// `verify-image-refs-cancelled`, `detect-changes-<result>`). The
|
||||
// alert template uses this to render a reason-specific message instead
|
||||
// of a generic "gate skipped" line. Empty string accepted so the
|
||||
// workflow can always pass the jq `--arg gateReason` shape without
|
||||
// branching on presence.
|
||||
gateReason: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
/**
|
||||
* Bounded LRU of processed deploy-webhook requests for at-least-once →
|
||||
* exactly-once idempotency. GitHub Actions retries the deploy-result POST
|
||||
* on transient failures (curl retry loop); we must 200 the duplicate
|
||||
* rather than re-emit the event.
|
||||
*
|
||||
* Dedupe key is `runId + ":" + sha256(body)` (composite). runId is the
|
||||
* primary identity — each workflow run has a unique id and a retried
|
||||
* POST carries an identical body, so the natural retry path still
|
||||
* collapses to a single event. The bodySha suffix is defense-in-depth
|
||||
* against two edge cases:
|
||||
* 1. Fork/re-run races that reuse a runId: GitHub Actions re-runs
|
||||
* preserve runId, and a malicious or accidental signer could replay
|
||||
* a runId with a different payload; we want that to re-emit, not
|
||||
* silently dedupe.
|
||||
* 2. Collisions on short numeric runIds if a sender is ever replaced
|
||||
* by a different workflow/infra (future-proofing).
|
||||
*
|
||||
* Bounded to 500 entries by default so the process footprint stays flat
|
||||
* under sustained traffic while comfortably absorbing a day of bursts:
|
||||
* 17 services × 2 retries × daily deploys leaves ample headroom. On
|
||||
* overflow we evict the oldest-seen entry (insertion-order) and log at
|
||||
* warn — if evictions start appearing, raise the cap or back with PB.
|
||||
*
|
||||
* `record` also touches on re-insert so repeatedly-seen ids stay warm
|
||||
* (keeps the LRU semantics described in the class name honest — the
|
||||
* previous implementation only ever inserted on first-seen).
|
||||
*/
|
||||
function createRunIdDedupe(
|
||||
capacity: number,
|
||||
logger: Logger,
|
||||
): {
|
||||
seen: (key: string) => boolean;
|
||||
record: (key: string) => void;
|
||||
size: () => number;
|
||||
} {
|
||||
// Use a Map for insertion-order iteration; re-seeing a runId re-inserts
|
||||
// it to the tail so frequently-seen ids stay warm.
|
||||
const set = new Map<string, true>();
|
||||
let evictionsReported = 0;
|
||||
return {
|
||||
seen(key) {
|
||||
if (capacity <= 0) return false;
|
||||
// Touch on read: promote to tail so frequently-seen keys survive
|
||||
// eviction pressure. The prior comment claimed this behavior but
|
||||
// the implementation only touched on `record()` (first-seen), so
|
||||
// the LRU guarantee was false.
|
||||
if (set.has(key)) {
|
||||
set.delete(key);
|
||||
set.set(key, true);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
},
|
||||
record(key) {
|
||||
if (capacity <= 0) return;
|
||||
if (set.has(key)) {
|
||||
set.delete(key);
|
||||
set.set(key, true);
|
||||
return;
|
||||
}
|
||||
set.set(key, true);
|
||||
while (set.size > capacity) {
|
||||
const oldest = set.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
set.delete(oldest);
|
||||
evictionsReported += 1;
|
||||
// Log every eviction at warn — these should be rare. If they
|
||||
// aren't, operators raise the cap or swap to PB-backed storage
|
||||
// (indexed by runId with a TTL) so evictions can't create
|
||||
// duplicate status.changed emissions across retry windows.
|
||||
logger.warn("webhook.deploy.dedupe-eviction", {
|
||||
evicted: oldest,
|
||||
totalEvictions: evictionsReported,
|
||||
capacity,
|
||||
});
|
||||
}
|
||||
},
|
||||
size() {
|
||||
return set.size;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function registerDeployWebhook(
|
||||
app: Hono,
|
||||
deps: DeployWebhookDeps,
|
||||
): void {
|
||||
const route = "/webhooks/deploy";
|
||||
const signedPath = deps.webhookPath ?? route;
|
||||
const dedupe = createRunIdDedupe(deps.dedupeSize ?? 500, deps.logger);
|
||||
|
||||
app.post(route, async (c) => {
|
||||
const timestamp = c.req.header("x-ops-timestamp") ?? "";
|
||||
const signatureHeader = c.req.header("x-ops-signature") ?? "";
|
||||
const raw = await c.req.text();
|
||||
|
||||
const verify = verifyHmac({
|
||||
method: "POST",
|
||||
// Use the configured canonical path. Defaults to the route constant,
|
||||
// but callers can override when this service is mounted behind a
|
||||
// proxy prefix and the sender signs a different path. The matching
|
||||
// signer in .github/workflows/showcase_deploy.yml must stay in
|
||||
// lockstep — if the route is renamed there, update the default here.
|
||||
path: signedPath,
|
||||
timestamp,
|
||||
body: raw,
|
||||
signatureHeader,
|
||||
secrets: deps.secrets,
|
||||
maxSkewSec: deps.maxSkewSec,
|
||||
nowSec: deps.nowSec,
|
||||
logger: deps.logger,
|
||||
});
|
||||
|
||||
if (!verify.ok) {
|
||||
deps.logger.warn("webhook.deploy.reject", { reason: verify.reason });
|
||||
deps.metrics?.inc("webhook_rejections", {
|
||||
reason: verify.reason ?? "unknown",
|
||||
});
|
||||
return c.json({ ok: false, reason: verify.reason }, 401);
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (err) {
|
||||
// Surface a short body preview so an operator can correlate a
|
||||
// misbehaving sender without exposing a full payload (which may
|
||||
// contain signed-but-malformed content).
|
||||
const preview = raw.length > 200 ? raw.slice(0, 200) + "…" : raw;
|
||||
deps.logger.warn("webhook.deploy.invalid-json", {
|
||||
err: String(err),
|
||||
bytes: raw.length,
|
||||
preview,
|
||||
});
|
||||
deps.metrics?.inc("webhook_rejections", { reason: "invalid-json" });
|
||||
return c.json({ ok: false, reason: "invalid-json" }, 400);
|
||||
}
|
||||
|
||||
const result = deployPayloadSchema.safeParse(parsed);
|
||||
if (!result.success) {
|
||||
const flattened = result.error.flatten();
|
||||
// Server-to-server call: include the zod flatten so operators
|
||||
// grepping the workflow run can see exactly which field failed
|
||||
// validation without reading the ops service log.
|
||||
deps.logger.error("webhook.deploy.invalid-payload", {
|
||||
issues: result.error.issues.map(
|
||||
(i) => i.path.join(".") + ": " + i.message,
|
||||
),
|
||||
flattened,
|
||||
});
|
||||
deps.metrics?.inc("webhook_rejections", { reason: "invalid-payload" });
|
||||
return c.json(
|
||||
{ ok: false, reason: "invalid-payload", errors: flattened },
|
||||
400,
|
||||
);
|
||||
}
|
||||
|
||||
// Idempotency: if we've already accepted this composite key, return
|
||||
// 200 OK without re-emitting. Key = `runId + ":" + sha256(body)` —
|
||||
// the natural retry path (same runId + same body) still collapses to
|
||||
// one event, while a runId replayed with a different payload (fork /
|
||||
// re-run races, manual re-post with tweaked services) correctly
|
||||
// falls through as a fresh event rather than being silently dropped.
|
||||
// The workflow curl-retry loop will replay the same payload on
|
||||
// transient upstream failures; re-emitting would double-count alerts
|
||||
// (especially rate-limited ones). We check AND record inside the
|
||||
// same synchronous block BEFORE emitting so two concurrent POSTs for
|
||||
// the same key can't both slip past `seen()` and produce duplicate
|
||||
// `deploy.result` events — GitHub Actions retries are serial today
|
||||
// but the handler is racy in principle and an infra change could
|
||||
// expose it.
|
||||
const bodySha = crypto.createHash("sha256").update(raw).digest("hex");
|
||||
const dedupeKey = `${result.data.runId}:${bodySha}`;
|
||||
if (dedupe.seen(dedupeKey)) {
|
||||
deps.logger.info("webhook.deploy.duplicate", {
|
||||
runId: result.data.runId,
|
||||
bodySha,
|
||||
});
|
||||
return c.json({ ok: true, duplicate: true }, 200);
|
||||
}
|
||||
// Record BEFORE emit so a concurrent request for the same key lands
|
||||
// on the "seen" branch rather than racing through emit.
|
||||
dedupe.record(dedupeKey);
|
||||
|
||||
// Workflow emits `--arg gateReason "$GATE_REASON"` unconditionally, so
|
||||
// a gate-inactive run sends the empty string. Normalise to undefined
|
||||
// here so downstream probe + template code only has to guard
|
||||
// `gateReason !== undefined`, not `gateReason && gateReason !== ""`.
|
||||
const gateReason =
|
||||
typeof result.data.gateReason === "string" &&
|
||||
result.data.gateReason.length > 0
|
||||
? result.data.gateReason
|
||||
: undefined;
|
||||
const event: DeployResultEvent = {
|
||||
runId: result.data.runId,
|
||||
runUrl: result.data.runUrl,
|
||||
services: result.data.services,
|
||||
failed: result.data.failed,
|
||||
succeeded: result.data.succeeded,
|
||||
cancelled: result.data.cancelled,
|
||||
gateSkipped: result.data.gateSkipped,
|
||||
gateReason,
|
||||
};
|
||||
deps.bus.emit("deploy.result", event);
|
||||
deps.logger.info("webhook.deploy.accepted", {
|
||||
runId: event.runId,
|
||||
services: event.services.length,
|
||||
failed: event.failed.length,
|
||||
cancelled: event.cancelled,
|
||||
gateSkipped: event.gateSkipped ?? false,
|
||||
});
|
||||
return c.json({ ok: true }, 202);
|
||||
});
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user