From d8d3f3d3690cdafdbcaed9f714a7b3364c7fddad Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:56:58 +0000 Subject: [PATCH] Fix duplicate entries in Skills Loaded (and Agents Invoked) columns of evaluation table (#462) * Initial plan * Fix duplicate entries in Skills Loaded column of evaluation table Agent-Logs-Url: https://github.com/dotnet/skills/sessions/2430176d-bf62-461d-bb79-781e34d7d084 Co-authored-by: danmoseley <6385855+danmoseley@users.noreply.github.com> * Remove accidentally committed .nuget binary Agent-Logs-Url: https://github.com/dotnet/skills/sessions/2430176d-bf62-461d-bb79-781e34d7d084 Co-authored-by: danmoseley <6385855+danmoseley@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: danmoseley <6385855+danmoseley@users.noreply.github.com> --- .gitignore | 1 + eng/skill-validator/src/Evaluate/Reporter.cs | 6 +- .../tests/Evaluate/OverfittingJudgeTests.cs | 100 ++++++++++++++++++ 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 67ebb93f..70fdc097 100644 --- a/.gitignore +++ b/.gitignore @@ -417,3 +417,4 @@ FodyWeavers.xsd *.msm *.msp .skill-validator-results/ +.nuget/ diff --git a/eng/skill-validator/src/Evaluate/Reporter.cs b/eng/skill-validator/src/Evaluate/Reporter.cs index 3c644021..c5c8235f 100644 --- a/eng/skill-validator/src/Evaluate/Reporter.cs +++ b/eng/skill-validator/src/Evaluate/Reporter.cs @@ -523,7 +523,8 @@ public static class Reporter if (anyPluginRun && s.SkillActivationPlugin is { } saPlug) { string plugActivation = FormatActivationCell(saPlug, s.ExpectActivation); - skillsCol += $" / {plugActivation}"; + if (plugActivation != skillsCol) + skillsCol += $" / {plugActivation}"; } // Agents invoked column — show subagent activations @@ -531,7 +532,8 @@ public static class Reporter if (anyPluginRun && s.SubagentActivationPlugin is { } saPlugAgent) { string plugAgents = FormatSubagentCell(saPlugAgent); - agentsCol += $" / {plugAgents}"; + if (plugAgents != agentsCol) + agentsCol += $" / {plugAgents}"; } var footnote = BuildVerdictFootnote(s, qualityDelta); diff --git a/eng/skill-validator/tests/Evaluate/OverfittingJudgeTests.cs b/eng/skill-validator/tests/Evaluate/OverfittingJudgeTests.cs index e926c34e..19ba13ea 100644 --- a/eng/skill-validator/tests/Evaluate/OverfittingJudgeTests.cs +++ b/eng/skill-validator/tests/Evaluate/OverfittingJudgeTests.cs @@ -714,6 +714,106 @@ public class OverfittingJudgeTests Assert.DoesNotContain("### ❌ Skill validation errors", md); } + [Fact] + public void MarkdownTable_SkillsLoaded_NoDuplicateWhenIsolatedAndPluginIdentical() + { + var activation = new SkillActivationInfo( + Activated: true, + DetectedSkills: new List { "my-skill" }, + ExtraTools: new List { "report_intent", "skill" }, + SkillEventCount: 1); + + var verdicts = new List + { + new() + { + SkillName = "my-skill", + SkillPath = "/test", + Passed = true, + Scenarios = new List + { + new() + { + ScenarioName = "sc1", + Baseline = new RunResult( + new RunMetrics { AgentOutput = "baseline" }, + new JudgeResult(new List(), 3.5, "OK")), + SkilledIsolated = new RunResult( + new RunMetrics { AgentOutput = "skilled" }, + new JudgeResult(new List(), 4.5, "Good")), + SkilledPlugin = new RunResult( + new RunMetrics { AgentOutput = "skilled-plugin" }, + new JudgeResult(new List(), 4.5, "Good")), + ImprovementScore = 0.25, + Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0), + SkillActivationIsolated = activation, + SkillActivationPlugin = activation, + } + }, + OverallImprovementScore = 0.25, + Reason = "Pass", + } + }; + + var md = Reporter.GenerateMarkdownSummary(verdicts); + + // Skills loaded cell should contain the activation info exactly once (no " / " separator) + Assert.DoesNotContain("my-skill; tools: report_intent, skill / ✅ my-skill", md); + Assert.Contains("✅ my-skill; tools: report_intent, skill", md); + } + + [Fact] + public void MarkdownTable_SkillsLoaded_ShowsBothWhenIsolatedAndPluginDiffer() + { + var isolatedActivation = new SkillActivationInfo( + Activated: true, + DetectedSkills: new List { "my-skill" }, + ExtraTools: new List { "report_intent" }, + SkillEventCount: 1); + var pluginActivation = new SkillActivationInfo( + Activated: true, + DetectedSkills: new List { "my-skill" }, + ExtraTools: new List { "report_intent", "skill" }, + SkillEventCount: 1); + + var verdicts = new List + { + new() + { + SkillName = "my-skill", + SkillPath = "/test", + Passed = true, + Scenarios = new List + { + new() + { + ScenarioName = "sc1", + Baseline = new RunResult( + new RunMetrics { AgentOutput = "baseline" }, + new JudgeResult(new List(), 3.5, "OK")), + SkilledIsolated = new RunResult( + new RunMetrics { AgentOutput = "skilled" }, + new JudgeResult(new List(), 4.5, "Good")), + SkilledPlugin = new RunResult( + new RunMetrics { AgentOutput = "skilled-plugin" }, + new JudgeResult(new List(), 4.5, "Good")), + ImprovementScore = 0.25, + Breakdown = new MetricBreakdown(0, 0, 0, 0, 0, 0, 0), + SkillActivationIsolated = isolatedActivation, + SkillActivationPlugin = pluginActivation, + } + }, + OverallImprovementScore = 0.25, + Reason = "Pass", + } + }; + + var md = Reporter.GenerateMarkdownSummary(verdicts); + + // Both activation entries should appear separated by " / " + Assert.Contains("✅ my-skill; tools: report_intent / ✅ my-skill; tools: report_intent, skill", md); + } + // --- Prompt overfitting detection tests --- [Fact]