mirror of
https://github.com/proffesor-for-testing/agentic-qe.git
synced 2026-09-19 08:45:47 +08:00
ci: remove automatic QCSD telemetry collection
This commit is contained in:
Dragan Spiridonov
@@ -1,232 +0,0 @@
|
||||
name: QCSD Production Telemetry Collection
|
||||
|
||||
on:
|
||||
# After npm publish workflow succeeds
|
||||
workflow_run:
|
||||
workflows: ["Publish to npm"]
|
||||
types: [completed]
|
||||
|
||||
# Weekly health check - Monday 6 AM UTC — disabled: collect-production-telemetry.sh
|
||||
# fails (missing dependencies or config). Re-enable once script is fixed. See #350.
|
||||
# schedule:
|
||||
# - cron: '0 6 * * 1'
|
||||
|
||||
# Manual trigger
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_id:
|
||||
description: 'Release version (e.g., v3.6.9). Leave empty for latest.'
|
||||
required: false
|
||||
type: string
|
||||
lookback_days:
|
||||
description: 'Days of history to analyze'
|
||||
required: false
|
||||
default: '30'
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
collect-telemetry:
|
||||
name: Collect Production Telemetry
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
# Skip if workflow_run triggered by a failed publish
|
||||
if: >
|
||||
github.event_name != 'workflow_run' ||
|
||||
github.event.workflow_run.conclusion == 'success'
|
||||
|
||||
outputs:
|
||||
release_id: ${{ steps.resolve.outputs.release_id }}
|
||||
telemetry_file: ${{ steps.collect.outputs.telemetry_file }}
|
||||
dora_summary: ${{ steps.collect.outputs.dora_summary }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Resolve release ID
|
||||
id: resolve
|
||||
run: |
|
||||
if [ -n "${{ inputs.release_id }}" ]; then
|
||||
RELEASE_ID="${{ inputs.release_id }}"
|
||||
elif [ "${{ github.event_name }}" = "workflow_run" ]; then
|
||||
# Get the release that triggered the publish workflow
|
||||
RELEASE_ID=$(gh release view --json tagName -q '.tagName' 2>/dev/null || echo "unknown")
|
||||
else
|
||||
# Schedule or fallback: use latest release
|
||||
RELEASE_ID=$(gh release view --json tagName -q '.tagName' 2>/dev/null || echo "unknown")
|
||||
fi
|
||||
echo "release_id=${RELEASE_ID}" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved release ID: ${RELEASE_ID}"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Determine trigger type
|
||||
id: trigger
|
||||
run: |
|
||||
case "${{ github.event_name }}" in
|
||||
workflow_run) echo "type=post-deploy" >> "$GITHUB_OUTPUT" ;;
|
||||
schedule) echo "type=scheduled" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "type=manual" >> "$GITHUB_OUTPUT" ;;
|
||||
esac
|
||||
|
||||
- name: Collect telemetry
|
||||
id: collect
|
||||
run: |
|
||||
chmod +x scripts/collect-production-telemetry.sh
|
||||
scripts/collect-production-telemetry.sh \
|
||||
--release-id "${{ steps.resolve.outputs.release_id }}" \
|
||||
--lookback "${{ inputs.lookback_days || '30' }}" \
|
||||
--trigger-type "${{ steps.trigger.outputs.type }}"
|
||||
|
||||
TELEMETRY_FILE="docs/telemetry/production/latest.json"
|
||||
echo "telemetry_file=${TELEMETRY_FILE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Extract summary for issue body
|
||||
if [ -f "$TELEMETRY_FILE" ]; then
|
||||
FREQ=$(jq -r '.dora.deploymentFrequency.value // "N/A"' "$TELEMETRY_FILE")
|
||||
LEAD=$(jq -r '.dora.leadTime.value // "N/A"' "$TELEMETRY_FILE")
|
||||
CFR=$(jq -r '.dora.changeFailureRate.value // "N/A"' "$TELEMETRY_FILE")
|
||||
MTTR=$(jq -r '.dora.mttr.value // "N/A"' "$TELEMETRY_FILE")
|
||||
BUGS=$(jq -r '.issues.openBugs // "N/A"' "$TELEMETRY_FILE")
|
||||
SUMMARY="Freq: ${FREQ}/wk | Lead: ${LEAD}h | CFR: ${CFR}% | MTTR: ${MTTR}h | Bugs: ${BUGS}"
|
||||
echo "dora_summary=${SUMMARY}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Commit telemetry data via PR
|
||||
# Telemetry must not be pushed directly to the protected main branch.
|
||||
# Historically this step ran `git push` against main, which failed 8/10
|
||||
# times because of branch protection. Instead we push to a bot branch
|
||||
# and open a PR; a maintainer reviews/merges it on their own schedule.
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git add docs/telemetry/production/
|
||||
if git diff --cached --quiet; then
|
||||
echo "No telemetry changes to commit"
|
||||
exit 0
|
||||
fi
|
||||
RELEASE_ID="${{ steps.resolve.outputs.release_id }}"
|
||||
BRANCH="telemetry/${RELEASE_ID}-$(date -u +%Y%m%d-%H%M%S)"
|
||||
git checkout -b "$BRANCH"
|
||||
git commit -m "chore(telemetry): production metrics for ${RELEASE_ID}"
|
||||
git push --set-upstream origin "$BRANCH"
|
||||
# Ensure labels exist (idempotent — signal-readiness job recreates these too)
|
||||
gh label create "automated" --color "6C757D" --force 2>/dev/null || true
|
||||
gh label create "qcsd-production-trigger" --color "0E8A16" --force 2>/dev/null || true
|
||||
PR_BODY=$(cat <<PRBODY
|
||||
Automated telemetry collection for release \`${RELEASE_ID}\`.
|
||||
|
||||
This PR adds the DORA snapshot under \`docs/telemetry/production/\`.
|
||||
Opened by the \`qcsd-production-trigger\` workflow — merge at your
|
||||
discretion. The artifact upload step on the parent workflow preserves
|
||||
the raw payload for 90 days regardless of whether this PR lands.
|
||||
PRBODY
|
||||
)
|
||||
gh pr create \
|
||||
--base main \
|
||||
--head "$BRANCH" \
|
||||
--title "chore(telemetry): production metrics for ${RELEASE_ID}" \
|
||||
--body "$PR_BODY" \
|
||||
--label "automated,qcsd-production-trigger"
|
||||
|
||||
- name: Upload telemetry artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: production-telemetry-${{ steps.resolve.outputs.release_id }}
|
||||
path: docs/telemetry/production/
|
||||
retention-days: 90
|
||||
|
||||
- name: Step summary
|
||||
run: |
|
||||
RELEASE_ID="${{ steps.resolve.outputs.release_id }}"
|
||||
TRIGGER="${{ steps.trigger.outputs.type }}"
|
||||
{
|
||||
echo "## QCSD Production Telemetry Collected"
|
||||
echo ""
|
||||
echo "| Field | Value |"
|
||||
echo "|-------|-------|"
|
||||
echo "| Release | \`${RELEASE_ID}\` |"
|
||||
echo "| Trigger | ${TRIGGER} |"
|
||||
echo "| DORA | ${{ steps.collect.outputs.dora_summary }} |"
|
||||
echo ""
|
||||
echo "### Invoke Production Swarm"
|
||||
echo "\`\`\`"
|
||||
echo "/qcsd-production-swarm TELEMETRY_DATA=docs/telemetry/production/latest.json RELEASE_ID=${RELEASE_ID}"
|
||||
echo "\`\`\`"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
signal-readiness:
|
||||
name: Signal Production Swarm Readiness
|
||||
runs-on: ubuntu-latest
|
||||
needs: collect-telemetry
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- name: Create or update trigger issue
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
RELEASE_ID="${{ needs.collect-telemetry.outputs.release_id }}"
|
||||
DORA="${{ needs.collect-telemetry.outputs.dora_summary }}"
|
||||
TRIGGER="${{ github.event_name }}"
|
||||
TITLE="[QCSD] Production Health Check Ready - ${RELEASE_ID}"
|
||||
|
||||
BODY=$(cat <<'ISSUE_EOF'
|
||||
## Production Telemetry Collected
|
||||
|
||||
**Release:** `RELEASE_PLACEHOLDER`
|
||||
**Trigger:** TRIGGER_PLACEHOLDER
|
||||
**DORA:** DORA_PLACEHOLDER
|
||||
|
||||
### Invoke Production Swarm
|
||||
|
||||
Run in Claude Code:
|
||||
```
|
||||
/qcsd-production-swarm TELEMETRY_DATA=docs/telemetry/production/latest.json RELEASE_ID=RELEASE_PLACEHOLDER
|
||||
```
|
||||
|
||||
### What This Does
|
||||
|
||||
The QCSD Production Swarm will:
|
||||
1. Load pre-collected DORA metrics from `docs/telemetry/production/latest.json`
|
||||
2. Detect domain flags from production context
|
||||
3. Spawn 3 core agents (DORA optimizer, defect predictor, root cause analyzer)
|
||||
4. Spawn conditional agents based on flags
|
||||
5. Produce a **HEALTHY / DEGRADED / CRITICAL** verdict
|
||||
6. Feed learnings back to Ideation and Refinement phases
|
||||
|
||||
---
|
||||
*Auto-generated by QCSD Production Telemetry workflow*
|
||||
ISSUE_EOF
|
||||
)
|
||||
|
||||
BODY="${BODY//RELEASE_PLACEHOLDER/$RELEASE_ID}"
|
||||
BODY="${BODY//TRIGGER_PLACEHOLDER/$TRIGGER}"
|
||||
BODY="${BODY//DORA_PLACEHOLDER/$DORA}"
|
||||
|
||||
# Ensure labels exist (gh issue create does not auto-create them)
|
||||
gh label create "qcsd-production-trigger" --color "0E8A16" --force 2>/dev/null || true
|
||||
gh label create "automated" --color "6C757D" --force 2>/dev/null || true
|
||||
|
||||
# Close any existing open trigger issues
|
||||
EXISTING=$(gh issue list --label "qcsd-production-trigger" --state open --json number -q '.[].number' 2>/dev/null || echo "")
|
||||
for ISSUE_NUM in $EXISTING; do
|
||||
gh issue close "$ISSUE_NUM" --comment "Superseded by new telemetry collection for ${RELEASE_ID}"
|
||||
done
|
||||
|
||||
# Create new issue
|
||||
gh issue create \
|
||||
--title "$TITLE" \
|
||||
--body "$BODY" \
|
||||
--label "qcsd-production-trigger,automated"
|
||||
@@ -0,0 +1,24 @@
|
||||
# Delivery-metric snapshots
|
||||
|
||||
`scripts/collect-production-telemetry.sh` is an explicit, maintainer-invoked
|
||||
diagnostic. It is not production observability and must not run automatically
|
||||
after a release.
|
||||
|
||||
The script reads repository metadata through the GitHub API and writes a local
|
||||
JSON snapshot containing:
|
||||
|
||||
- release count per week;
|
||||
- release creation-to-publication time;
|
||||
- failed npm-publish workflow runs;
|
||||
- lifecycle time for closed issues labelled `bug`; and
|
||||
- open bug counts.
|
||||
|
||||
These values are only rough delivery-process proxies. In particular, they do
|
||||
not measure commit-to-production lead time, production incidents, service
|
||||
recovery time, availability, latency, errors, traffic, or user impact. Run the
|
||||
script manually only when those limitations fit the question being asked, and
|
||||
review its JSON before retaining or sharing it.
|
||||
|
||||
Generated snapshots belong under `docs/telemetry/production/` and are not
|
||||
committed automatically, uploaded as workflow artifacts, turned into pull
|
||||
requests, or used to create issues.
|
||||
@@ -15,7 +15,9 @@
|
||||
# GITHUB_REPOSITORY Owner/repo (default: auto-detect from git remote)
|
||||
# GH_TOKEN / GITHUB_TOKEN GitHub authentication (gh CLI must be authenticated)
|
||||
#
|
||||
# Designed to run in GitHub Actions or locally with gh CLI.
|
||||
# Maintainer-invoked diagnostic only. This script deliberately has no GitHub
|
||||
# Actions trigger: its GitHub-derived values are delivery-process proxies, not
|
||||
# production observability. See docs/telemetry/README.md before using it.
|
||||
# Individual API failures produce null fields, not script crash.
|
||||
|
||||
# Note: no set -e — individual commands use || fallbacks for error tolerance
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { readdirSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const workflowDirectory = join(process.cwd(), '.github', 'workflows');
|
||||
|
||||
describe('QCSD telemetry automation boundary', () => {
|
||||
it('should_notCollectOrPersistQcsdTelemetry_when_releaseWorkflowsRun', () => {
|
||||
const workflowFiles = readdirSync(workflowDirectory)
|
||||
.filter((name) => name.endsWith('.yml') || name.endsWith('.yaml'));
|
||||
|
||||
const automatedTelemetryReferences = workflowFiles.flatMap((name) => {
|
||||
const content = readFileSync(join(workflowDirectory, name), 'utf8');
|
||||
return /collect-production-telemetry|telemetry\/production|qcsd-production-trigger/.test(content)
|
||||
? [name]
|
||||
: [];
|
||||
});
|
||||
|
||||
expect(automatedTelemetryReferences).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user