From 4f45bb0406d2a3fceda689ebf07b57cb48c8cc17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 16 Sep 2026 17:55:15 +0200 Subject: [PATCH] fix: preserve active health investigation rows Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../workflows/devops-health-groom.lock.yml | 84 +++++++++- .github/workflows/devops-health-groom.md | 82 ++++++++++ eng/evaluation/test_token_failover.py | 148 ++++++++++++++++++ 3 files changed, 313 insertions(+), 1 deletion(-) diff --git a/.github/workflows/devops-health-groom.lock.yml b/.github/workflows/devops-health-groom.lock.yml index 8668ca3a..0b47f9df 100644 --- a/.github/workflows/devops-health-groom.lock.yml +++ b/.github/workflows/devops-health-groom.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"fa23f14b3a8dd65005189e3a634fdb7926bd5e74ea31f657d4d2f1165426c456","body_hash":"2064bcfa4c63e1bef3639078cdffa0dd9e5a0c03aa422f697b8184a7a215413a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"3cdc7e2ddd6f2b8f743783b151988ab8a27337b6d9e56e11e09cd9b2eb566386","body_hash":"2064bcfa4c63e1bef3639078cdffa0dd9e5a0c03aa422f697b8184a7a215413a","compiler_version":"v0.88.7","strict":true,"agent_id":"copilot","agent_model":"${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'gpt-5.6-sol' }}","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"5e508589e03a7757a7e05b26e834292f5445bfb6","version":"v0.88.7"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14","digest":"sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.14@sha256:f7df036c86575527b61f3f7df91c4412349a12b2a74988d929eafa2999230c98"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14","digest":"sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.14@sha256:6f95e2234dd9bd6333a8ff28ccea7ecf0204acd4a09108723844dbd2bf6268c5"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14","digest":"sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.14@sha256:2ce8df3abf3e9b76e9c0cf5863da41f1ab3f89b20ad14b988806ab89e7bf2cd5"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.18","digest":"sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.18@sha256:85b940556a8faa4e1fdbef124bfd75f2c4ebd855a10b88a1c3b6f3e97f6f1a53"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d","pinned_image":"ghcr.io/github/gh-aw-node@sha256:33e1ec1d967ac1f28c2cedc24ce103dea3226840626de345d3fe579e96cf5c7d"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["actions_get","actions_list","get_commit","get_file_contents","get_job_logs","get_latest_release","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_releases","list_starred_repositories","list_tags","search_code","search_issues","search_repositories"]},{"name":"safeoutputs","tools":["missing_data","missing_tool","noop","publish_groomed_dashboard"]}]} # This file was automatically generated by gh-aw (v0.88.7). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1848,6 +1848,88 @@ jobs: const islandPattern = /(^|\n)## 🔍 Investigation Results\n[\s\S]*?(?=\n## |\n/ + )?.[1], + }); + } + return rows; + }; + const stateMatches = [ + ...(issue.body || "").matchAll( + //g + ), + ]; + let activeIds = null; + if (stateMatches.length > 1) { + throw new Error("Dashboard state marker is duplicated"); + } + if (stateMatches.length === 1) { + let state; + try { + state = JSON.parse(stateMatches[0][1]); + } catch (error) { + throw new Error(`Dashboard state JSON is invalid: ${error.message}`); + } + if (!Array.isArray(state.active_findings)) { + throw new Error("Dashboard active findings are invalid"); + } + activeIds = new Set( + state.active_findings.map(finding => finding?.fingerprint) + ); + if (activeIds.has(undefined) || activeIds.size !== state.active_findings.length) { + throw new Error("Dashboard active finding IDs are invalid"); + } + } + const newRows = parseRows(section); + const priorIsland = (issue.body || "").match(islandPattern)?.[0] || ""; + const priorRows = parseRows(priorIsland); + for (const [findingId, priorRow] of priorRows) { + const mustPreserve = activeIds === null || activeIds.has(findingId); + if (!mustPreserve) { + continue; + } + const nextRow = newRows.get(findingId); + if ( + !nextRow || + ( + priorRow.correlation && + nextRow.correlation !== priorRow.correlation + ) || + ( + priorRow.status === "✅ Done" && + ( + nextRow.status !== "✅ Done" || + nextRow.result !== priorRow.result + ) + ) + ) { + throw new Error( + `Active Investigation Results row was not preserved for ${findingId}` + ); + } + } + if ( + activeIds !== null && + [...newRows.keys()].some(findingId => !activeIds.has(findingId)) + ) { + throw new Error("Investigation Results contains a non-active finding"); + } let nextBody; if (islandPattern.test(issue.body || "")) { nextBody = (issue.body || "").replace( diff --git a/.github/workflows/devops-health-groom.md b/.github/workflows/devops-health-groom.md index 1b7438dd..b674a58d 100644 --- a/.github/workflows/devops-health-groom.md +++ b/.github/workflows/devops-health-groom.md @@ -139,6 +139,88 @@ safe-outputs: const islandPattern = /(^|\n)## 🔍 Investigation Results\n[\s\S]*?(?=\n## |\n/ + )?.[1], + }); + } + return rows; + }; + const stateMatches = [ + ...(issue.body || "").matchAll( + //g + ), + ]; + let activeIds = null; + if (stateMatches.length > 1) { + throw new Error("Dashboard state marker is duplicated"); + } + if (stateMatches.length === 1) { + let state; + try { + state = JSON.parse(stateMatches[0][1]); + } catch (error) { + throw new Error(`Dashboard state JSON is invalid: ${error.message}`); + } + if (!Array.isArray(state.active_findings)) { + throw new Error("Dashboard active findings are invalid"); + } + activeIds = new Set( + state.active_findings.map(finding => finding?.fingerprint) + ); + if (activeIds.has(undefined) || activeIds.size !== state.active_findings.length) { + throw new Error("Dashboard active finding IDs are invalid"); + } + } + const newRows = parseRows(section); + const priorIsland = (issue.body || "").match(islandPattern)?.[0] || ""; + const priorRows = parseRows(priorIsland); + for (const [findingId, priorRow] of priorRows) { + const mustPreserve = activeIds === null || activeIds.has(findingId); + if (!mustPreserve) { + continue; + } + const nextRow = newRows.get(findingId); + if ( + !nextRow || + ( + priorRow.correlation && + nextRow.correlation !== priorRow.correlation + ) || + ( + priorRow.status === "✅ Done" && + ( + nextRow.status !== "✅ Done" || + nextRow.result !== priorRow.result + ) + ) + ) { + throw new Error( + `Active Investigation Results row was not preserved for ${findingId}` + ); + } + } + if ( + activeIds !== null && + [...newRows.keys()].some(findingId => !activeIds.has(findingId)) + ) { + throw new Error("Investigation Results contains a non-active finding"); + } let nextBody; if (islandPattern.test(issue.body || "")) { nextBody = (issue.body || "").replace( diff --git a/eng/evaluation/test_token_failover.py b/eng/evaluation/test_token_failover.py index 7f1954f7..cdb12b26 100644 --- a/eng/evaluation/test_token_failover.py +++ b/eng/evaluation/test_token_failover.py @@ -116,6 +116,98 @@ def investigation_publisher_script() -> str: ) +def groom_publisher_script() -> str: + source = ( + REPO_ROOT / ".github" / "workflows" / "devops-health-groom.md" + ).read_text(encoding="utf-8") + frontmatter = yaml.safe_load(source.split("---", 2)[1]) + publisher = frontmatter["safe-outputs"]["jobs"]["publish-groomed-dashboard"] + return next( + step["with"]["script"] + for step in publisher["steps"] + if step.get("name") == "Verify and publish groomed dashboard" + ) + + +def run_groom_publisher( + test_case: unittest.TestCase, + *, + prior_body: str, + section: str, +) -> dict[str, object]: + node = shutil.which("node") + if not node: + test_case.skipTest("Node.js is required for publisher behavior tests") + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + output_path = temp_path / "agent-output.json" + harness_path = temp_path / "groom-publisher-harness.cjs" + output_path.write_text( + json.dumps( + { + "items": [ + { + "type": "publish_groomed_dashboard", + "expected_updated_at": "2026-09-16T10:00:00Z", + "investigation_section": section, + } + ] + } + ), + encoding="utf-8", + ) + harness_path.write_text( + f""" +const calls = []; +const github = {{ + rest: {{ + issues: {{ + get: async args => {{ + calls.push({{ type: "get", args }}); + return {{ + data: {{ + state: "open", + title: "🏥 Repository Health Dashboard", + labels: [{{ name: "devops-health" }}], + updated_at: "2026-09-16T10:00:00Z", + body: {json.dumps(prior_body)} + }} + }}; + }}, + update: async args => {{ + calls.push({{ type: "update", body: args.body }}); + return {{ data: {{}} }}; + }} + }} + }} +}}; +const context = {{ repo: {{ owner: "dotnet", repo: "skills" }} }}; +(async () => {{ +{groom_publisher_script()} +}})() + .then(() => console.log(JSON.stringify({{ ok: true, calls }}))) + .catch(error => console.log(JSON.stringify({{ + ok: false, + error: error.message, + calls + }}))); +""", + encoding="utf-8", + ) + environment = os.environ.copy() + environment["GH_AW_AGENT_OUTPUT"] = str(output_path) + completed = subprocess.run( + [node, str(harness_path)], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + env=environment, + ) + return json.loads(completed.stdout.strip()) + + def run_investigation_publisher( test_case: unittest.TestCase, *, @@ -690,6 +782,10 @@ class TokenFailoverTests(unittest.TestCase): "Dashboard identity or version validation failed", groom_script, ) + self.assertIn( + "Active Investigation Results row was not preserved", + groom_script, + ) groom_manifest = json.loads( groom_lock_text.splitlines()[1].removeprefix("# gh-aw-manifest: ") ) @@ -882,6 +978,58 @@ class TokenFailoverTests(unittest.TestCase): " ".join(shared_health.split()), ) + def test_devops_health_groom_publisher_preserves_active_rows(self) -> None: + finding_id = "pipeline:evaluation:evaluate:test:failure" + correlation = "hc-500-1" + row = ( + f"| `{finding_id}` | Evaluation tests failed | 🔴 Critical | " + "⏳ Pending | 2026-09-16 | ⏳ Awaiting investigation result " + f" |" + ) + section = f"""## 🔍 Investigation Results + +| Finding ID | Finding | Severity | Investigation | First Seen | Result | +|------------|---------|----------|---------------|------------|--------| +{row}""" + prior_body = f"""# 🏥 Daily Health Check — 2026-09-16 + +{section} + + +""" + empty_section = """## 🔍 Investigation Results + +| Finding ID | Finding | Severity | Investigation | First Seen | Result | +|------------|---------|----------|---------------|------------|--------|""" + + rejected = run_groom_publisher( + self, + prior_body=prior_body, + section=empty_section, + ) + self.assertFalse(rejected["ok"]) + self.assertIn( + "Active Investigation Results row was not preserved", + rejected["error"], + ) + self.assertEqual( + [call["type"] for call in rejected["calls"]], + ["get"], + ) + + accepted = run_groom_publisher( + self, + prior_body=prior_body, + section=section, + ) + self.assertTrue(accepted["ok"]) + self.assertEqual( + [call["type"] for call in accepted["calls"]], + ["get", "update"], + ) + def test_devops_health_publisher_rejects_invalid_state(self) -> None: body = """# 🏥 Daily Health Check — 2026-09-16