From 00fce762d06417171c60eee6fd1f29edbbdf7f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 16 Sep 2026 17:19:00 +0200 Subject: [PATCH] fix: close health workflow output gaps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/aw/shared/devops-investigate.lock.md | 4 +- .../workflows/devops-health-check.lock.yml | 60 +++++++-- .github/workflows/devops-health-check.md | 60 +++++++-- .../workflows/devops-health-groom.lock.yml | 2 +- .github/workflows/devops-health-groom.md | 5 +- .../devops-health-investigate.lock.yml | 4 +- .../workflows/devops-health-investigate.md | 2 +- eng/evaluation/test_token_failover.py | 119 +++++++++++++++++- 8 files changed, 229 insertions(+), 27 deletions(-) diff --git a/.github/aw/shared/devops-investigate.lock.md b/.github/aw/shared/devops-investigate.lock.md index ec1bf840..9c073645 100644 --- a/.github/aw/shared/devops-investigate.lock.md +++ b/.github/aw/shared/devops-investigate.lock.md @@ -54,8 +54,8 @@ When `finding_type == "pipeline"`: 6. **Identify the PR that introduced the breaking change**: - For each suspect commit, use `search_pull_requests` with the exact SHA. - - Verify candidates with `get_pull_request`, `get_pull_request_files`, and - `get_pull_request_diff`. + - Verify candidates with `pull_request_read`: use method `get` for metadata, + `get_files` for changed files, and `get_diff` for the patch. - Record the PR number, title, author, and merge date only for a verified match. - This helps attribute the regression and identify who can help fix it diff --git a/.github/workflows/devops-health-check.lock.yml b/.github/workflows/devops-health-check.lock.yml index 6970c2c6..77b185ea 100644 --- a/.github/workflows/devops-health-check.lock.yml +++ b/.github/workflows/devops-health-check.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"034dea00121fee0e569acea849c4f8abf516cc9af7f02aa57e409b0a0d6ab2f8","body_hash":"07d2dd03e43399087384ab6436b1e7ccebd9ff703fa6d67df59e12cb5d098e43","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a0faa7129533cae1cee7bd7fb83138a2d06383f1d1166bea4e2ebcf5bee349c0","body_hash":"7df4a3a8d38de00a38a3f9cdf542f28355084baf214f36990b7868988dd6a978","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["actions_get","actions_list","get_commit","get_file_contents","get_job_logs","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","publish_health_dashboard"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1788,12 +1788,36 @@ jobs: } const item = items[0]; + const validateLinkDestination = destination => { + if (destination.startsWith("#")) { + return; + } + if (destination.startsWith("//")) { + throw new Error(`Protocol-relative links are not allowed: ${destination}`); + } + const link = new URL(destination); + if (link.protocol !== "https:" || link.hostname !== "github.com") { + throw new Error(`Only github.com links are allowed: ${link.href}`); + } + }; const validateGitHubLinks = value => { for (const match of value.matchAll(/https?:\/\/[^\s)<>"']+/g)) { - const link = new URL(match[0].replace(/[.,;:!?]+$/, "")); - if (link.protocol !== "https:" || link.hostname !== "github.com") { - throw new Error(`Only github.com links are allowed: ${link.href}`); - } + validateLinkDestination( + match[0].replace(/[.,;:!?]+$/, "") + ); + } + if (/(^|[^:])\/\/[A-Za-z0-9]/m.test(value)) { + throw new Error("Protocol-relative links are not allowed"); + } + for (const match of value.matchAll( + /!?\[[^\]\r\n]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g + )) { + validateLinkDestination(match[1]); + } + for (const match of value.matchAll( + /(?:href|src)\s*=\s*["']([^"']+)["']/gi + )) { + validateLinkDestination(match[1]); } }; @@ -1831,9 +1855,27 @@ jobs: if (typeof expectedUpdatedAt !== "string" || !expectedUpdatedAt) { throw new Error("expected_updated_at is required"); } + const requiredDashboardPatterns = [ + /^# ๐Ÿฅ Daily Health Check โ€” (\d{4}-\d{2}-\d{2})$/gm, + /^## ๐Ÿ†• New Findings \([0-9]+\)$/gm, + /^## ๐Ÿ” Investigation Results$/gm, + /^## โœ… Resolved Since Yesterday \([0-9]+\)$/gm, + /^## ๐Ÿ“Œ Existing Findings \([0-9]+\)$/gm, + /^## ๐Ÿ“Š Trends \(7-day\)$/gm, + /^\| Finding ID \| Finding \| Severity \| Investigation \| First Seen \| Result \|$/gm, + ]; + const dashboardDateMatches = [ + ...dashboardBody.matchAll(requiredDashboardPatterns[0]), + ]; if ( (dashboardBody.match(/$" diff --git a/.github/workflows/devops-health-check.md b/.github/workflows/devops-health-check.md index 8ae9f113..6c0063ac 100644 --- a/.github/workflows/devops-health-check.md +++ b/.github/workflows/devops-health-check.md @@ -95,12 +95,36 @@ safe-outputs: } const item = items[0]; + const validateLinkDestination = destination => { + if (destination.startsWith("#")) { + return; + } + if (destination.startsWith("//")) { + throw new Error(`Protocol-relative links are not allowed: ${destination}`); + } + const link = new URL(destination); + if (link.protocol !== "https:" || link.hostname !== "github.com") { + throw new Error(`Only github.com links are allowed: ${link.href}`); + } + }; const validateGitHubLinks = value => { for (const match of value.matchAll(/https?:\/\/[^\s)<>"']+/g)) { - const link = new URL(match[0].replace(/[.,;:!?]+$/, "")); - if (link.protocol !== "https:" || link.hostname !== "github.com") { - throw new Error(`Only github.com links are allowed: ${link.href}`); - } + validateLinkDestination( + match[0].replace(/[.,;:!?]+$/, "") + ); + } + if (/(^|[^:])\/\/[A-Za-z0-9]/m.test(value)) { + throw new Error("Protocol-relative links are not allowed"); + } + for (const match of value.matchAll( + /!?\[[^\]\r\n]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g + )) { + validateLinkDestination(match[1]); + } + for (const match of value.matchAll( + /(?:href|src)\s*=\s*["']([^"']+)["']/gi + )) { + validateLinkDestination(match[1]); } }; @@ -138,9 +162,27 @@ safe-outputs: if (typeof expectedUpdatedAt !== "string" || !expectedUpdatedAt) { throw new Error("expected_updated_at is required"); } + const requiredDashboardPatterns = [ + /^# ๐Ÿฅ Daily Health Check โ€” (\d{4}-\d{2}-\d{2})$/gm, + /^## ๐Ÿ†• New Findings \([0-9]+\)$/gm, + /^## ๐Ÿ” Investigation Results$/gm, + /^## โœ… Resolved Since Yesterday \([0-9]+\)$/gm, + /^## ๐Ÿ“Œ Existing Findings \([0-9]+\)$/gm, + /^## ๐Ÿ“Š Trends \(7-day\)$/gm, + /^\| Finding ID \| Finding \| Severity \| Investigation \| First Seen \| Result \|$/gm, + ]; + const dashboardDateMatches = [ + ...dashboardBody.matchAll(requiredDashboardPatterns[0]), + ]; if ( (dashboardBody.match(/$" @@ -1253,7 +1295,7 @@ Before finishing, verify: tools. Process API responses and dashboard state in memory. Do not create scripts or intermediate files. - **CRITICAL โ€” Publisher body must be inline**: The `dashboard_body` field must contain the **complete, literal issue body text**. NEVER write it to a file or use a shell reference. -- **CRITICAL โ€” Investigation Results section**: The `## ๐Ÿ” Investigation Results` section MUST always appear in the issue body template. The downstream [grooming workflow](../workflows/devops-health-groom.md) manages this section via a `replace-island` block. Preserve existing active rows by fingerprint and append new `๐Ÿ”„ Dispatched` rows with their exact fingerprints. Do NOT wrap the section in island markers yourself. +- **CRITICAL โ€” Investigation Results section**: The `## ๐Ÿ” Investigation Results` section MUST always appear in the issue body template. The downstream [grooming workflow](../workflows/devops-health-groom.md) manages this section via a `replace-island` block. Preserve existing active rows by fingerprint and append new `โณ Pending` rows with their exact fingerprints and correlation markers. Do NOT wrap the section in island markers yourself. - **Be data-driven**: Include specific numbers, durations, percentages, and links. - **Be precise with fingerprints**: Use the exact fingerprint formulas from the knowledge file. Consistency is critical โ€” the same finding MUST produce the same fingerprint across runs. - **First run handling**: If the validated dashboard body has no valid previous diff --git a/.github/workflows/devops-health-groom.lock.yml b/.github/workflows/devops-health-groom.lock.yml index 9c387846..f6abbd07 100644 --- a/.github/workflows/devops-health-groom.lock.yml +++ b/.github/workflows/devops-health-groom.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"1cf4c454441fb59d4eecaf7d19810e98c18301522797336a0983b5d1fb9d316b","body_hash":"64108daa2583ca5902cadf11d65c866ea9251a2b2a110dd94038596b041f38f0","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"1cf4c454441fb59d4eecaf7d19810e98c18301522797336a0983b5d1fb9d316b","body_hash":"e55bc8f8119edd5f19e9288935749649ddfad632cab076514524e6a6320e95b5","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["actions_get","actions_list","get_commit","get_file_contents","get_job_logs","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","update_issue"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/devops-health-groom.md b/.github/workflows/devops-health-groom.md index 557e07d8..e83da483 100644 --- a/.github/workflows/devops-health-groom.md +++ b/.github/workflows/devops-health-groom.md @@ -167,7 +167,10 @@ Parse each comment into one of these categories: For each **Investigation** comment, extract: - `finding_id` from the `**Finding ID:** \`{id}\`` line -- `executive_summary` from the `**Executive Summary:**` line (everything after the label) +- `executive_summary` from the `**Executive Summary:**` line. Collapse + whitespace to one line, limit it to 512 characters, and replace `]`, `|`, + carriage returns, and newlines with safe plain-text equivalents before using + it as a Markdown link label. - `correlation_id` from the `**Correlation:**` line - `comment_url` = the comment's `html_url` - `comment_id` = the comment's `id` diff --git a/.github/workflows/devops-health-investigate.lock.yml b/.github/workflows/devops-health-investigate.lock.yml index 49a0411f..4c635038 100644 --- a/.github/workflows/devops-health-investigate.lock.yml +++ b/.github/workflows/devops-health-investigate.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"775b209d1bc08ad73be079caa01c974cebad8c17a56677a068df104351e3197e","body_hash":"57384d2ea6efcbed6fbd3cd31ab748f63576630308b0f81703b82c6467540d7d","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"722e7bb9333fe8399e288811bd681c8fd853b12d1d0bcb855392338ef5bf9d1c","body_hash":"421e044a12c72b5e7a1777b99685be999509819aa808b413719e85a28b2a16ea","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["actions_get","actions_list","get_commit","get_file_contents","get_job_logs","get_latest_release","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","publish_investigation_report"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -108,7 +108,7 @@ permissions: {} concurrency: group: gh-aw-${{ github.workflow }}-${{ inputs.finding_id }} -run-name: DevOps Health Investigation โ€” ${{ inputs.correlation_id }} +run-name: DevOps Health Investigation ยท ${{ inputs.correlation_id }} env: OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }} diff --git a/.github/workflows/devops-health-investigate.md b/.github/workflows/devops-health-investigate.md index 0427376d..641cc9eb 100644 --- a/.github/workflows/devops-health-investigate.md +++ b/.github/workflows/devops-health-investigate.md @@ -6,7 +6,7 @@ description: > Dispatched by the health check orchestrator. It reports evidence, root cause, blast radius, and a proposed remediation without modifying repository files or executing repository code. -run-name: "DevOps Health Investigation โ€” ${{ inputs.correlation_id }}" +run-name: "DevOps Health Investigation ยท ${{ inputs.correlation_id }}" on: permissions: {} diff --git a/eng/evaluation/test_token_failover.py b/eng/evaluation/test_token_failover.py index 67a06c44..a5b5ea49 100644 --- a/eng/evaluation/test_token_failover.py +++ b/eng/evaluation/test_token_failover.py @@ -233,6 +233,7 @@ def run_health_publisher( existing_runs: list[dict[str, object]] | None = None, existing_comments: list[dict[str, object]] | None = None, initial_body: str = "", + complete_template: bool = True, ) -> dict[str, object]: node = shutil.which("node") if not node: @@ -242,8 +243,43 @@ def run_health_publisher( temp_path = Path(temp_dir) output_path = temp_path / "agent-output.json" harness_path = temp_path / "publisher-harness.cjs" + normalized_item = dict(item) + if complete_template: + body = str(normalized_item["dashboard_body"]) + missing_sections = [] + for pattern, heading in ( + (r"^## ๐Ÿ†• New Findings \([0-9]+\)$", "## ๐Ÿ†• New Findings (0)"), + ( + r"^## โœ… Resolved Since Yesterday \([0-9]+\)$", + "## โœ… Resolved Since Yesterday (0)", + ), + ( + r"^## ๐Ÿ“Œ Existing Findings \([0-9]+\)$", + "## ๐Ÿ“Œ Existing Findings (0)", + ), + (r"^## ๐Ÿ“Š Trends \(7-day\)$", "## ๐Ÿ“Š Trends (7-day)"), + ): + if not re.search(pattern, body, re.MULTILINE): + missing_sections.append(heading) + if missing_sections: + body = body.replace( + " +""" + result = run_health_publisher( + self, + { + "expected_updated_at": "2026-09-16T10:00:00Z", + "dashboard_body": incomplete_template_body, + "daily_comment": "## ๐Ÿ“‹ Health Check โ€” 2026-09-16", + "dispatches_json": "[]", + }, + complete_template=False, + ) + self.assertFalse(result["ok"]) + self.assertIn("Dashboard or daily comment structure", result["error"]) + self.assertEqual(result["calls"], []) + duplicate_finding = { "fingerprint": "infra:no-codeowners", "title": "Missing CODEOWNERS", @@ -1305,6 +1371,26 @@ class TokenFailoverTests(unittest.TestCase): self.assertIn("Duplicate row correlation", result["error"]) self.assertEqual(result["calls"], []) + dispatched_without_correlation = body.replace( + "โณ Pending", + "๐Ÿ”„ Dispatched", + ).replace( + " โณ Awaiting investigation result ", + " Investigation started", + ) + result = run_health_publisher( + self, + { + "expected_updated_at": "2026-09-16T10:00:00Z", + "dashboard_body": dispatched_without_correlation, + "daily_comment": "## ๐Ÿ“‹ Health Check โ€” 2026-09-16", + "dispatches_json": "[]", + }, + ) + self.assertFalse(result["ok"]) + self.assertIn("In-flight row has invalid correlation", result["error"]) + self.assertEqual(result["calls"], []) + def test_devops_health_publisher_reconciles_accepted_dispatch(self) -> None: finding = { "fingerprint": "pipeline:evaluation:evaluate:build:failure", @@ -1488,6 +1574,22 @@ class TokenFailoverTests(unittest.TestCase): ["get", "update", "repo", "comment"], ) + unsafe = run_health_publisher( + self, + { + "expected_updated_at": "2026-09-16T10:00:00Z", + "dashboard_body": body.replace( + "`owner/action@v1` should use a commit SHA.", + "[details](//attacker.example/path)", + ), + "daily_comment": "## ๐Ÿ“‹ Health Check โ€” 2026-09-16", + "dispatches_json": "[]", + }, + ) + self.assertFalse(unsafe["ok"]) + self.assertIn("Protocol-relative links are not allowed", unsafe["error"]) + self.assertEqual(unsafe["calls"], []) + def test_devops_health_investigation_is_report_only(self) -> None: investigate_source = ( REPO_ROOT / ".github" / "workflows" / "devops-health-investigate.md" @@ -1664,6 +1766,11 @@ class TokenFailoverTests(unittest.TestCase): investigate, ) self.assertNotIn("hc-{YYYY-MM-DD}", investigate) + self.assertIn( + 'run-name: "DevOps Health Investigation ยท ' + '${{ inputs.correlation_id }}"', + investigate, + ) self.assertIn("bounded `list_commits` and `get_commit`", investigate) self.assertIn("searching for the exact suspect commit SHA", investigate) investigate_knowledge = ( @@ -1680,10 +1787,18 @@ class TokenFailoverTests(unittest.TestCase): "`list_commits`", "`get_commit`", "`search_pull_requests`", - "`get_pull_request_files`", + "`pull_request_read`", + "`get_files`", + "`get_diff`", "`get_job_logs`", ): self.assertIn(available_tool, investigate_knowledge) + for unsupported_tool in ( + "`get_pull_request`", + "`get_pull_request_files`", + "`get_pull_request_diff`", + ): + self.assertNotIn(unsupported_tool, investigate_knowledge) workflow_tests = yaml.safe_load(TEST_WORKFLOW.read_text(encoding="utf-8")) triggers = workflow_tests.get("on", workflow_tests.get(True))