dotnet__skills/.github/workflows/evaluation-run.yml

# Reusable evaluation workflow called by evaluation.yml.
# Contains the build-validator, evaluate, and comment-on-pr jobs.
name: evaluation-run

on:
  workflow_call:
    inputs:
      entries:
        description: 'JSON array of {name, plugin, skills_path} entries to evaluate'
        required: true
        type: string
      checkout-ref:
        description: 'Git ref to checkout for skill content (empty = default)'
        required: false
        type: string
        default: ''
      is-fork:
        description: 'Whether this is a fork PR. When true, the validator is built from the base branch instead of checkout-ref to prevent untrusted code from modifying tooling.'
        required: false
        type: boolean
        default: false
      model:
        description: 'Copilot model to use'
        required: false
        type: string
        default: 'claude-opus-4.6'
      judge-model:
        description: 'Judge model to use'
        required: false
        type: string
        default: 'claude-opus-4.6'
      runs:
        description: 'Number of runs per test'
        required: false
        type: string
        default: '3'
      parallel-skills:
        description: 'Max concurrent skills'
        required: false
        type: string
        default: '5'
      parallel-scenarios:
        description: 'Max concurrent scenarios'
        required: false
        type: string
        default: '5'
      parallel-runs:
        description: 'Max concurrent runs'
        required: false
        type: string
        default: '5'
      verbose:
        description: 'Enable verbose output'
        required: false
        type: boolean
        default: false
      pr-number:
        description: 'PR number for commenting (empty = skip)'
        required: false
        type: string
        default: ''
    secrets:
      COPILOT_GITHUB_TOKEN:
        required: false
      COPILOT_GITHUB_TOKEN_2:
        required: false
      COPILOT_GITHUB_TOKEN_3:
        required: false
      COPILOT_GITHUB_TOKEN_4:
        required: false
      COPILOT_GITHUB_TOKEN_5:
        required: false
      COPILOT_GITHUB_TOKEN_6:
        required: false
      COPILOT_GITHUB_TOKEN_7:
        required: false
      COPILOT_GITHUB_TOKEN_8:
        required: false

jobs:
  build-validator:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      # For same-repo PRs (trusted), build from checkout-ref so skill-validator
      # changes are tested. For fork PRs (untrusted), always build from the
      # base branch to prevent untrusted code from modifying tooling.
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ !inputs.is-fork && inputs.checkout-ref || '' }}
          persist-credentials: false

      - name: Setup .NET SDK
        uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5
        with:
          global-json-file: global.json

      - name: Build skill-validator
        run: dotnet publish eng/skill-validator/src/SkillValidator.csproj

      - name: Create validator archive
        run: tar -czf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release .

      - name: Upload built validator
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: skill-validator-dist.tar.gz
          path: skill-validator-dist.tar.gz
          archive: false
          retention-days: 1

  evaluate:
    needs: build-validator
    if: needs.build-validator.result == 'success'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 180
    name: evaluate (${{ matrix.entry.name }})
    strategy:
      fail-fast: false
      matrix:
        entry: ${{ fromJson(inputs.entries) }}

    steps:
      - name: Checkout skills content
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          ref: ${{ inputs.checkout-ref || '' }}
          persist-credentials: false

      - name: Setup .NET SDK
        uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5
        with:
          global-json-file: global.json

      - name: Download built validator
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8
        with:
          name: skill-validator-dist.tar.gz

      - name: Extract validator
        run: |
          mkdir -p artifacts/publish/SkillValidator/release
          tar -xzf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release

      - name: Select random Copilot token
        id: select-token
        env:
          TOKEN_1: ${{ secrets.COPILOT_GITHUB_TOKEN }}
          TOKEN_2: ${{ secrets.COPILOT_GITHUB_TOKEN_2 }}
          TOKEN_3: ${{ secrets.COPILOT_GITHUB_TOKEN_3 }}
          TOKEN_4: ${{ secrets.COPILOT_GITHUB_TOKEN_4 }}
          TOKEN_5: ${{ secrets.COPILOT_GITHUB_TOKEN_5 }}
          TOKEN_6: ${{ secrets.COPILOT_GITHUB_TOKEN_6 }}
          TOKEN_7: ${{ secrets.COPILOT_GITHUB_TOKEN_7 }}
          TOKEN_8: ${{ secrets.COPILOT_GITHUB_TOKEN_8 }}
        run: |
          # Collect all non-empty token secrets
          TOKENS=()
          NAMES=()
          for i in 1 2 3 4 5 6 7 8; do
            var="TOKEN_$i"
            val="${!var}"
            if [ -n "$val" ]; then
              TOKENS+=("$val")
              if [ "$i" -eq 1 ]; then
                NAMES+=("COPILOT_GITHUB_TOKEN")
              else
                NAMES+=("COPILOT_GITHUB_TOKEN_$i")
              fi
            fi
          done

          if [ ${#TOKENS[@]} -eq 0 ]; then
            echo "::error::No COPILOT_GITHUB_TOKEN secrets are configured"
            exit 1
          fi

          # Assign token deterministically by matrix job index to avoid collisions.
          # Falls back to RANDOM if strategy.job-index is unavailable.
          JOB_INDEX="${{ strategy.job-index }}"
          if [ -n "$JOB_INDEX" ]; then
            IDX=$(( JOB_INDEX % ${#TOKENS[@]} ))
          else
            IDX=$((RANDOM % ${#TOKENS[@]}))
          fi
          echo "Selected ${NAMES[$IDX]} (1 of ${#TOKENS[@]} available tokens, job-index=${JOB_INDEX:-random})"

          # Mask the value so it won't appear in logs, then export
          echo "::add-mask::${TOKENS[$IDX]}"
          echo "token=${TOKENS[$IDX]}" >> $GITHUB_OUTPUT

      - name: Run skill-validator
        env:
          GITHUB_TOKEN: ${{ steps.select-token.outputs.token }}
          RESULTS_PATH: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}
        run: |
          ARGS="--verdict-warn-only --verbose"
          ARGS="$ARGS --results-dir $RESULTS_PATH --reporter console --reporter json --reporter markdown"
          ARGS="$ARGS --model ${{ inputs.model }}"
          ARGS="$ARGS --judge-model ${{ inputs.judge-model }}"
          ARGS="$ARGS --runs ${{ inputs.runs }}"
          ARGS="$ARGS --parallel-skills ${{ inputs.parallel-skills }}"
          ARGS="$ARGS --parallel-scenarios ${{ inputs.parallel-scenarios }}"
          ARGS="$ARGS --parallel-runs ${{ inputs.parallel-runs }}"

          if [ "${{ inputs.verbose }}" = "true" ]; then
            ARGS="$ARGS --verbose"
          fi

          artifacts/publish/SkillValidator/release/skill-validator evaluate $ARGS --tests-dir ./tests/${{ matrix.entry.plugin }} ./${{ matrix.entry.skills_path }}

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: skill-validator-results-${{ matrix.entry.name }}
          path: artifacts/TestResults/skill-validator/${{ matrix.entry.name }}/
          include-hidden-files: true
          retention-days: 30

  comment-on-pr:
    needs: evaluate
    if: always() && needs.evaluate.result != 'cancelled' && inputs.pr-number != ''
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    steps:
      - name: Download all result artifacts
        if: needs.evaluate.result != 'skipped'
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8
        with:
          pattern: skill-validator-results-*
          path: all-results/
          merge-multiple: false
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Download built validator
        if: needs.evaluate.result != 'skipped'
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8
        with:
          name: skill-validator-dist.tar.gz
        continue-on-error: ${{ needs.evaluate.result != 'success' }}

      - name: Extract validator
        if: needs.evaluate.result != 'skipped' && hashFiles('skill-validator-dist.tar.gz') != ''
        run: |
          mkdir -p artifacts/publish/SkillValidator/release
          tar -xzf skill-validator-dist.tar.gz -C artifacts/publish/SkillValidator/release

      - name: Consolidate and post results
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          PR_NUMBER=${{ inputs.pr-number }}
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # If results exist, consolidate them into a summary comment
          JSON_FILES=$(find all-results/ -name results.json 2>/dev/null || true)
          VALIDATOR="artifacts/publish/SkillValidator/release/skill-validator"
          if [ -n "$JSON_FILES" ] && [ -x "$VALIDATOR" ]; then
            "$VALIDATOR" consolidate \
              --output summary-body.md \
              $JSON_FILES

            {
              cat summary-body.md
              echo ""
              echo "[Full results]($RUN_URL)"
            } > consolidated-comment.md

            cat consolidated-comment.md >> $GITHUB_STEP_SUMMARY

            gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
              -X POST -F "body=@consolidated-comment.md"
          else
            # No results — evaluate was skipped or failed before producing artifacts
            if [[ "${{ needs.evaluate.result }}" == "skipped" ]]; then
              BODY="❌ Evaluation did not complete (upstream job failed or was skipped). [View workflow run](${RUN_URL})"
            else
              BODY="❌ Evaluation failed. [View workflow run](${RUN_URL})"
            fi

            gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" -X POST -f body="$BODY"
          fi