vercel__eve/.github/workflows/e2e-vercel.yml

name: E2E Tests (Vercel)

on:
  workflow_dispatch:
  pull_request:

concurrency:
  group: e2e-vercel-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # The e2e matrix jobs are required status checks, so they must report on
  # every PR. An `on.pull_request.paths` filter would leave them permanently
  # "expected" on PRs that don't touch e2e-relevant paths, blocking merge.
  # Instead the workflow always triggers, this job decides relevance, and the
  # matrix jobs skip when nothing relevant changed — skipped required checks
  # count as satisfied.
  changes:
    name: changes
    runs-on: ubuntu-latest
    outputs:
      relevant: ${{ steps.filter.outputs.relevant }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false
          fetch-depth: 2

      - name: Check changed paths
        id: filter
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "relevant=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
          # this diff is exactly the PR's changed files.
          if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then
            echo "relevant=true" >> "$GITHUB_OUTPUT"
          else
            echo "relevant=false" >> "$GITHUB_OUTPUT"
          fi

  discover-fixtures:
    name: discover-fixtures
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.discover.outputs.matrix }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Discover eval fixtures
        id: discover
        run: .github/scripts/discover-e2e-fixtures.sh

  e2e:
    name: e2e-vercel (${{ matrix.fixture.name }}, ${{ matrix.model.name }})
    runs-on: ubuntu-latest
    needs: [changes, discover-fixtures]
    # No job-level `if`: the stable aggregate check below requires a successful
    # matrix result. Guarding each step lets an irrelevant diff satisfy it in
    # seconds without running setup or evals.
    strategy:
      fail-fast: false
      matrix:
        fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }}
        model:
          - name: openai-sol
            id: openai/gpt-5.6-sol
          - name: anthropic-opus
            id: anthropic/claude-opus-4.8

    env:
      EVE_E2E_MODEL: ${{ matrix.model.id }}
      EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit
      VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
      VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
      VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
      VERCEL_USE_EXPERIMENTAL_FRAMEWORKS: "1"

    steps:
      - name: Checkout code
        if: needs.changes.outputs.relevant == 'true'
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          persist-credentials: false

      - name: Setup pnpm
        if: needs.changes.outputs.relevant == 'true'
        uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6

      - name: Setup Node.js
        if: needs.changes.outputs.relevant == 'true'
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version-file: .nvmrc
          cache: "pnpm"

      - name: Install dependencies
        if: needs.changes.outputs.relevant == 'true'
        run: pnpm install --frozen-lockfile

      - name: Build eve package
        if: needs.changes.outputs.relevant == 'true'
        # Full build, not build:js: the deployed/runtime bundles must carry the
        # stamped package version (scripts/stamp-version-tokens.mjs) and docs.
        run: pnpm --filter eve run build

      - name: Deploy fixture and run evals
        if: needs.changes.outputs.relevant == 'true'
        env:
          EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json
        run: |
          set -euo pipefail
          mkdir -p "$EVE_EVAL_JUNIT_DIR"
          cd "${{ matrix.fixture.dir }}"

          token_args=()
          if [ -n "${VERCEL_TOKEN:-}" ]; then
            token_args=(--token "$VERCEL_TOKEN")
          fi
          team_args=()
          scope_args=()
          if [ -n "${VERCEL_ORG_ID:-}" ]; then
            team_args=(--team "$VERCEL_ORG_ID")
            scope_args=(--scope "$VERCEL_ORG_ID")
          fi

          pnpm exec vc link --yes --project "$VERCEL_PROJECT_ID" "${team_args[@]}" "${token_args[@]}"
          pnpm exec vc env pull --yes --environment=preview "${token_args[@]}"
          # Sandbox templates key on VERCEL_PROJECT_ID (present at build and
          # runtime). Do not add VERCEL_TEAM_ID: it does not exist at runtime,
          # so the deployed function could not resolve a team-scoped template.
          VERCEL=1 \
            VERCEL_ENV=preview \
            VERCEL_TARGET_ENV=preview \
            VERCEL_PROJECT_ID="$VERCEL_PROJECT_ID" \
            pnpm exec eve build --profile "$EVE_BUILD_PROFILE"
          DEPLOYMENT_URL="$(pnpm exec vc deploy --prebuilt --yes --target=preview \
            --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" "${token_args[@]}" | tail -n 1)"

          npx eve eval --strict --verbose --url "$DEPLOYMENT_URL" \
            --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml"

          # Redeploy-tagged evals rebuild and redeploy the fixture from inside
          # the eval body, repointing a run-scoped alias at each new
          # deployment. They run against the alias (immutable deployment URLs
          # never change what they serve) and are skipped in the main suite
          # above because EVE_E2E_REDEPLOY_ALIAS is unset there.
          if npx eve eval --tag redeploy --list >/dev/null 2>&1; then
            ALIAS="eve-e2e-${{ matrix.fixture.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app"
            # vc alias does not infer the team from the project link the way
            # link/deploy do, so pass the scope explicitly.
            pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}"
            EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \
              npx eve eval --tag redeploy --strict --verbose \
              --url "https://$ALIAS" \
              --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}-redeploy.xml"
          fi

      - name: Log deployable build timing
        if: needs.changes.outputs.relevant == 'true'
        env:
          BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json
          FIXTURE_PATH: ${{ matrix.fixture.dir }}
        run: |
          node ./scripts/build-profile-report.mjs \
            --profile "$BUILD_PROFILE" \
            --app-label "$FIXTURE_PATH" \
            --sandbox-prewarm included \
            --require-phase sandbox.prewarm \
            --output-format build-time

      - name: Upload eval artifacts
        if: (failure()) && needs.changes.outputs.relevant == 'true'
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6
        with:
          name: eval-artifacts-vercel-${{ matrix.fixture.name }}-${{ matrix.model.name }}
          path: |
            .junit
            e2e/fixtures/*/.eve/evals
            apps/fixtures/*/.eve/evals
          include-hidden-files: true
          if-no-files-found: ignore
          retention-days: 7

  e2e-required:
    name: e2e-vercel
    runs-on: ubuntu-latest
    needs: [e2e]
    if: always()
    steps:
      - name: Require successful e2e matrix
        env:
          E2E_RESULT: ${{ needs.e2e.result }}
        run: |
          if [ "$E2E_RESULT" != "success" ]; then
            echo "E2E matrix result: $E2E_RESULT"
            exit 1
          fi