mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Make evaluate a subcommand with tests-dir as required argument (#398)
* Make evaluate a subcommand with tests-dir as required argument
Change EvaluateCommand from RootCommand to a regular Command('evaluate')
so it is a subcommand like check, consolidate, and rejudge. Convert
--tests-dir from an optional Option to a required positional Argument
placed after the skill paths.
Update .csproj RunArguments, evaluation-run workflow, README, and
CONTRIBUTING.md to reflect the new CLI syntax.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
* Default --parallel-skills/scenarios/runs to 3
Update the default from 1 to 3 for all three parallelism options and
remove the now-redundant overrides from RunArguments in the .csproj.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
* Replace RunArguments with launchSettings.json, publish in skill-check
Add Properties/launchSettings.json with default evaluate args for local
development. Remove RunArguments from .csproj. Update skill-check.yml to
publish the validator and invoke the binary directly instead of using
dotnet run.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
* Keep --tests-dir as a required option, not positional
A variadic paths argument greedily consumes all positional values,
making a trailing positional tests-dir impossible to parse. Keep
--tests-dir as a named required option which matches the previous UX.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
* Move testsDirOpt after resultsDirOpt, drop evaluate from CONTRIBUTING examples
Reorder testsDirOpt declaration and registration to follow resultsDirOpt.
Remove the evaluate subcommand from CONTRIBUTING.md dotnet run examples
since the launch profile already provides it.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---------
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Viktor Hofer
@@ -211,7 +211,7 @@ jobs:
|
||||
ARGS="$ARGS --verbose"
|
||||
fi
|
||||
|
||||
artifacts/publish/SkillValidator/release/skill-validator $ARGS --tests-dir ./tests/${{ matrix.entry.plugin }} ./${{ matrix.entry.skills_path }}
|
||||
artifacts/publish/SkillValidator/release/skill-validator evaluate $ARGS --tests-dir ./tests/${{ matrix.entry.plugin }} ./${{ matrix.entry.skills_path }}
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
|
||||
@@ -24,11 +24,13 @@ jobs:
|
||||
with:
|
||||
global-json-file: global.json
|
||||
|
||||
- name: Build skill-validator
|
||||
run: dotnet publish eng/skill-validator/src/SkillValidator.csproj
|
||||
|
||||
- name: Run skill-validator check
|
||||
run: |
|
||||
plugin_args=$(for d in plugins/*/; do echo "--plugin $d"; done)
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- \
|
||||
check \
|
||||
artifacts/publish/SkillValidator/release/skill-validator check \
|
||||
$plugin_args \
|
||||
--allowed-external-deps eng/skill-validator/allowed-external-deps.txt
|
||||
|
||||
|
||||
+5
-5
@@ -266,19 +266,19 @@ Prerequisites: .NET 10 SDK or later and `gh auth login`.
|
||||
|
||||
```bash
|
||||
# Run tests for a single plugin
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
|
||||
# Run tests for a single skill (pass the skill directory directly)
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills/common-build-errors
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills/common-build-errors
|
||||
|
||||
# Fewer runs for faster iteration (default is 5)
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj --runs 3 --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate --runs 3 --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
|
||||
# Use a specific model
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj --model claude-opus-4.6 --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate --model claude-opus-4.6 --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
|
||||
# Run with verbose logging
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills --verbose
|
||||
dotnet run --project eng/skill-validator/src/SkillValidator.csproj -- evaluate --verbose --tests-dir tests/dotnet-msbuild plugins/dotnet-msbuild/skills
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
|
||||
@@ -43,13 +43,46 @@ dnx Microsoft.DotNet.SkillValidator --source ./path/to/downloaded/ evaluate --te
|
||||
|
||||
## Usage
|
||||
|
||||
The tool has two main subcommands:
|
||||
The tool has several subcommands:
|
||||
|
||||
- **`check`** — Static analysis of skills, plugins, and agents (no LLM, no token required)
|
||||
- **`evaluate`** — LLM-based evaluation testing (requires a Copilot token)
|
||||
- **`check`** — Static analysis of skills, plugins, and agents (no LLM, no token required)
|
||||
- **`consolidate`** — Merge results from matrix jobs into a single summary
|
||||
- **`rejudge`** — Re-run judging on previously saved sessions
|
||||
|
||||
All examples below use the `skill-validator` binary directly. If running from source, replace `skill-validator` with `dotnet run --project eng/skill-validator/src --`:
|
||||
|
||||
### LLM evaluation (`evaluate`)
|
||||
|
||||
```bash
|
||||
# Show evaluate help
|
||||
skill-validator evaluate --help
|
||||
|
||||
# Evaluate a skill (--tests-dir is required)
|
||||
skill-validator evaluate --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills/my-skill
|
||||
|
||||
# Verbose output with per-scenario breakdowns
|
||||
skill-validator evaluate --verbose --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Custom model and threshold
|
||||
skill-validator evaluate --model claude-sonnet-4.5 --min-improvement 0.2 --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Use a different model for judging vs agent runs
|
||||
skill-validator evaluate --model gpt-5.3-codex --judge-model claude-opus-4.6-fast --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Multiple runs for stability
|
||||
skill-validator evaluate --runs 5 --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Override the default results directory (.skill-validator-results)
|
||||
skill-validator evaluate --results-dir ./my-results --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# File reporters can also be specified explicitly.
|
||||
skill-validator evaluate --reporter junit --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Verdict-warn-only mode (verdict failures return exit 0, execution errors still fail)
|
||||
skill-validator evaluate --verdict-warn-only --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
```
|
||||
|
||||
### Static analysis (`check`)
|
||||
|
||||
```bash
|
||||
@@ -75,37 +108,6 @@ skill-validator check --plugin ./plugins/my-plugin --allowed-external-deps ./eng
|
||||
skill-validator check --verbose --plugin ./plugins/my-plugin
|
||||
```
|
||||
|
||||
### LLM evaluation (`evaluate`)
|
||||
|
||||
```bash
|
||||
# Show evaluate help
|
||||
skill-validator --help
|
||||
|
||||
# Evaluate a skill (--tests-dir is required)
|
||||
skill-validator --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills/my-skill
|
||||
|
||||
# Verbose output with per-scenario breakdowns
|
||||
skill-validator --verbose --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Custom model and threshold
|
||||
skill-validator --model claude-sonnet-4.5 --min-improvement 0.2 --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Use a different model for judging vs agent runs
|
||||
skill-validator --model gpt-5.3-codex --judge-model claude-opus-4.6-fast --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Multiple runs for stability
|
||||
skill-validator --runs 5 --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Override the default results directory (.skill-validator-results)
|
||||
skill-validator --results-dir ./my-results --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# File reporters can also be specified explicitly.
|
||||
skill-validator --reporter junit --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
|
||||
# Verdict-warn-only mode (verdict failures return exit 0, execution errors still fail)
|
||||
skill-validator --verdict-warn-only --tests-dir ./tests/my-plugin ./plugins/my-plugin/skills
|
||||
```
|
||||
|
||||
## `check` flags
|
||||
|
||||
| Flag | Default | Description |
|
||||
@@ -129,9 +131,9 @@ skill-validator --verdict-warn-only --tests-dir ./tests/my-plugin ./plugins/my-p
|
||||
| `--judge-mode <mode>` | `pairwise` | Judge mode: `pairwise`, `independent`, or `both` |
|
||||
| `--min-improvement <n>` | `0.1` | Minimum improvement score (0–1) |
|
||||
| `--runs <n>` | `5` | Runs per scenario (averaged for stability) |
|
||||
| `--parallel-skills <n>` | `1` | Max concurrent skills to evaluate |
|
||||
| `--parallel-scenarios <n>` | `1` | Max concurrent scenarios per skill |
|
||||
| `--parallel-runs <n>` | `1` | Max concurrent runs per scenario |
|
||||
| `--parallel-skills <n>` | `3` | Max concurrent skills to evaluate |
|
||||
| `--parallel-scenarios <n>` | `3` | Max concurrent scenarios per skill |
|
||||
| `--parallel-runs <n>` | `3` | Max concurrent runs per scenario |
|
||||
| `--confidence-level <n>` | `0.95` | Confidence level for statistical intervals (0–1) |
|
||||
| `--judge-timeout <n>` | `300` | Judge LLM timeout in seconds |
|
||||
| `--require-completion` | `true` | Fail if skill regresses task completion |
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace SkillValidator.Commands;
|
||||
|
||||
public static class EvaluateCommand
|
||||
{
|
||||
public static RootCommand Create()
|
||||
public static Command Create()
|
||||
{
|
||||
var pathsArg = new Argument<string[]>("paths") { Description = "Paths to skill directories or parent directories", Arity = ArgumentArity.OneOrMore };
|
||||
var minImprovementOpt = new Option<double>("--min-improvement") { Description = "Minimum improvement score to pass (0-1)", DefaultValueFactory = _ => 0.1 };
|
||||
@@ -21,13 +21,13 @@ public static class EvaluateCommand
|
||||
var judgeModeOpt = new Option<string>("--judge-mode") { Description = "Judge mode: pairwise, independent, or both", DefaultValueFactory = _ => "pairwise" }
|
||||
.AcceptOnlyFromAmong("pairwise", "independent", "both");
|
||||
var runsOpt = new Option<int>("--runs") { Description = "Number of runs per scenario for averaging", DefaultValueFactory = _ => 5 };
|
||||
var parallelSkillsOpt = new Option<int>("--parallel-skills") { Description = "Max concurrent skills to evaluate", DefaultValueFactory = _ => 1 };
|
||||
var parallelScenariosOpt = new Option<int>("--parallel-scenarios") { Description = "Max concurrent scenarios per skill", DefaultValueFactory = _ => 1 };
|
||||
var parallelRunsOpt = new Option<int>("--parallel-runs") { Description = "Max concurrent runs per scenario", DefaultValueFactory = _ => 1 };
|
||||
var parallelSkillsOpt = new Option<int>("--parallel-skills") { Description = "Max concurrent skills to evaluate", DefaultValueFactory = _ => 3 };
|
||||
var parallelScenariosOpt = new Option<int>("--parallel-scenarios") { Description = "Max concurrent scenarios per skill", DefaultValueFactory = _ => 3 };
|
||||
var parallelRunsOpt = new Option<int>("--parallel-runs") { Description = "Max concurrent runs per scenario", DefaultValueFactory = _ => 3 };
|
||||
var judgeTimeoutOpt = new Option<int>("--judge-timeout") { Description = "Judge timeout in seconds", DefaultValueFactory = _ => 300 };
|
||||
var confidenceLevelOpt = new Option<double>("--confidence-level") { Description = "Confidence level for statistical intervals (0-1)", DefaultValueFactory = _ => 0.95 };
|
||||
var resultsDirOpt = new Option<string>("--results-dir") { Description = "Directory to save results to", DefaultValueFactory = _ => ".skill-validator-results" };
|
||||
var testsDirOpt = new Option<string?>("--tests-dir") { Description = "Directory containing test subdirectories" };
|
||||
var testsDirOpt = new Option<string>("--tests-dir") { Description = "Directory containing test subdirectories", Required = true };
|
||||
var reporterOpt = new Option<string[]>("--reporter") { Description = "Reporter (console, json, junit, markdown). Can be repeated.", AllowMultipleArgumentsPerToken = true };
|
||||
var noOverfittingCheckOpt = new Option<bool>("--no-overfitting-check") { Description = "Disable LLM-based overfitting analysis (on by default)" };
|
||||
var overfittingFixOpt = new Option<bool>("--overfitting-fix") { Description = "Generate a fixed eval.yaml with improved rubric items/assertions" };
|
||||
@@ -36,7 +36,7 @@ public static class EvaluateCommand
|
||||
var noiseMaxDegradationOpt = new Option<double>("--noise-max-degradation") { Description = "Maximum acceptable average quality degradation (0-1) in noise test (only positive degradations count)", DefaultValueFactory = _ => 0.2 };
|
||||
var noiseMaxScenarioDegradationOpt = new Option<double>("--noise-max-scenario-degradation") { Description = "Maximum acceptable quality degradation (0-1) for any single noise-test scenario", DefaultValueFactory = _ => 0.4 };
|
||||
|
||||
var command = new RootCommand("Evaluate agent skills via LLM-based testing")
|
||||
var command = new Command("evaluate", "Evaluate agent skills via LLM-based testing")
|
||||
{
|
||||
pathsArg,
|
||||
minImprovementOpt,
|
||||
@@ -102,7 +102,7 @@ public static class EvaluateCommand
|
||||
Reporters = reporters,
|
||||
SkillPaths = paths,
|
||||
ResultsDir = parseResult.GetValue(resultsDirOpt),
|
||||
TestsDir = parseResult.GetValue(testsDirOpt) ?? throw new InvalidOperationException("--tests-dir is required"),
|
||||
TestsDir = parseResult.GetValue(testsDirOpt)!,
|
||||
OverfittingCheck = !parseResult.GetValue(noOverfittingCheckOpt),
|
||||
OverfittingFix = parseResult.GetValue(overfittingFixOpt),
|
||||
KeepSessions = parseResult.GetValue(keepSessionsOpt),
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
using System.CommandLine;
|
||||
using SkillValidator.Commands;
|
||||
|
||||
var rootCommand = EvaluateCommand.Create();
|
||||
var rootCommand = new RootCommand("Validate that agent skills meaningfully improve agent performance");
|
||||
rootCommand.Add(EvaluateCommand.Create());
|
||||
rootCommand.Add(CheckCommand.Create());
|
||||
rootCommand.Add(ConsolidateCommand.Create());
|
||||
rootCommand.Add(RejudgeCommand.Create());
|
||||
|
||||
@@ -16,9 +16,6 @@
|
||||
<!-- Publishing -->
|
||||
<RuntimeIdentifiers>win-x64;win-arm64;linux-x64;linux-arm64;osx-arm64</RuntimeIdentifiers>
|
||||
<PublishAot>true</PublishAot>
|
||||
|
||||
<!-- dotnet run args for local invocation -->
|
||||
<RunArguments>--results-dir "$([MSBuild]::NormalizePath('$(ArtifactsPath)', 'TestResults', '$(AssemblyName)'))" --parallel-skills 3 --parallel-scenarios 3 --parallel-runs 3</RunArguments>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user