skills/proffesor-for-testing__agentic-qe
1
0
Fork 0
mirror of https://github.com/proffesor-for-testing/agentic-qe.git synced 2026-09-19 08:45:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore(evals): finish eval model-ID migration across remaining surfaces

Browse Source 
Follow-up to d95f0d7b / 2f4d119c — the earlier passes updated .claude/skills
and assets/skills but missed several other eval surfaces that still pinned
retired IDs (claude-3.5-sonnet, claude-3-haiku):

- `.github/workflows/test-qe-browser.yml` — the `aqe eval run --model` flag
  (inert in command-eval mode, but a real dead-ID leftover) -> claude-sonnet-4-6.
- `plugins/agentic-qe-fleet/skills/` — the tracked plugin-marketplace skill copy
  (8 eval suites + a sample-output fixture); chaos skills get the opus-4-8 ceiling.
- `docs/schemas/skill-eval.schema.json` — models `enum` was missing claude-opus-4-8
  and still allowed retired/deprecated IDs; trimmed to current+active, default
  flipped to claude-sonnet-4-6. Plus description examples in the sibling schemas
  and the modelUsed values in docs/examples/*.json fixtures.
- `docs/templates/skill-frontmatter.example.yaml` validation_models list.
- `src/cli/commands/eval.ts` — the `aqe eval` `--model` help example and the
  `run-all --models` default (`claude-3.5-sonnet` -> `claude-sonnet-4-6`).

Verified: typecheck clean; all edited JSON/YAML parse; zero retired IDs remain in
any eval suite, schema, template, workflow, or eval-CLI default. Intentionally
left untouched: the consensus-provider subsystem (pricing tables / type-unions /
defaults — separate routing layer, bucket-2 data) and historical release notes /
QE audit reports (which document the old IDs by design).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dragan Spiridonov
2026-06-12 14:52:24 +00:00
parent b57c0c0026
commit 56206c7be2
17 changed files with 30 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/test-qe-browser.yml
+1 -1
View File
@@ -186,7 +186,7 @@ jobs:
run: |
node dist/cli/bundle.js eval run \
--skill qe-browser \
--model claude-3.5-sonnet \
--model claude-sonnet-4-6 \
--output /tmp/qe-browser-eval.json
env:
CI: 'true'
docs/examples/security-testing-output.example.json
+1 -1
View File
@@ -380,7 +380,7 @@
"executionTimeMs": 45230,
"toolsUsed": ["semgrep", "npm-audit", "owasp-zap", "trivy"],
"agentId": "qe-security-scanner",
"modelUsed": "claude-3.5-sonnet",
"modelUsed": "claude-sonnet-4-6",
"inputHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"targetUrl": "https://example.com",
"targetPath": "src/",
docs/examples/testability-scoring-output.example.json
+1 -1
View File
@@ -318,7 +318,7 @@
"executionTimeMs": 12450,
"toolsUsed": ["playwright", "chrome-devtools", "axe-core"],
"agentId": "qe-quality-analyzer",
"modelUsed": "claude-3.5-sonnet",
"modelUsed": "claude-sonnet-4-6",
"inputHash": "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730",
"targetUrl": "https://example.com",
"environment": "ci",
docs/schemas/skill-eval.schema.json
+2 -4
View File
@@ -26,19 +26,17 @@
"items": {
"type": "string",
"enum": [
"claude-opus-4-8",
"claude-opus-4-7",
"claude-sonnet-4-6",
"claude-haiku-4-5",
"claude-opus-4-5",
"claude-sonnet-4",
"claude-3.5-sonnet",
"claude-3-haiku",
"gpt-4o",
"gpt-4o-mini",
"gpt-4-turbo"
]
},
"default": ["claude-3.5-sonnet"],
"default": ["claude-sonnet-4-6"],
"minItems": 1,
"uniqueItems": true,
"description": "Models to run evaluation against for cross-model validation"
docs/schemas/skill-frontmatter.schema.json
+1 -1
View File
@@ -264,7 +264,7 @@
"validation_models": {
"type": "array",
"items": { "type": "string" },
"description": "Models used for evaluation testing (e.g., ['claude-3.5-sonnet', 'claude-3-haiku'])"
"description": "Models used for evaluation testing (e.g., ['claude-sonnet-4-6', 'claude-haiku-4-5'])"
},
"pass_rate": {
"type": "number",
docs/schemas/skill-output.template.json
+1 -1
View File
@@ -120,7 +120,7 @@
},
"modelUsed": {
"type": "string",
"description": "LLM model used for execution (e.g., claude-3.5-sonnet)"
"description": "LLM model used for execution (e.g., claude-sonnet-4-6)"
},
"inputHash": {
"type": "string",
docs/templates/skill-frontmatter.example.yaml
Vendored
+2 -2
View File
@@ -202,8 +202,8 @@ validation:
# Multi-model testing configuration
validation_models:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6
- claude-haiku-4-5
- gpt-4o
# Evaluation metrics
plugins/agentic-qe-fleet/skills/chaos-engineering-resilience/evals/chaos-engineering-resilience.yaml
+3 -3
View File
@@ -34,9 +34,9 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet # Primary model (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality threshold)
- gpt-4o # Cross-vendor validation
- claude-opus-4-8 # Capability ceiling (high-stakes skill)
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/mutation-testing/evals/mutation-testing.yaml
+2 -3
View File
@@ -25,9 +25,8 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet # Primary model (high accuracy expected)
- claude-3-haiku # Fast model (ensure minimum quality)
- gpt-4o # Cross-vendor validation
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/mutation-testing/test-data/sample-output.json
+1 -1
View File
@@ -272,7 +272,7 @@
"executionTimeMs": 125430,
"toolsUsed": ["stryker"],
"agentId": "qe-mutation-tester",
"modelUsed": "claude-3.5-sonnet",
"modelUsed": "claude-sonnet-4-6",
"targetPath": "src/auth",
"testPath": "tests/auth",
"incremental": false,
plugins/agentic-qe-fleet/skills/qe-chaos-resilience/evals/qe-chaos-resilience.yaml
+3 -2
View File
@@ -32,8 +32,9 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet # Primary model (high accuracy expected)
- claude-3-haiku # Fast model (minimum quality threshold)
- claude-opus-4-8 # Capability ceiling (high-stakes skill)
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/qe-coverage-analysis/evals/qe-coverage-analysis.yaml
+2 -2
View File
@@ -32,8 +32,8 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/qe-quality-assessment/evals/qe-quality-assessment.yaml
+2 -2
View File
@@ -31,8 +31,8 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/qe-test-execution/evals/qe-test-execution.yaml
+2 -2
View File
@@ -31,8 +31,8 @@ description: >
# =============================================================================
models_to_test:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
# =============================================================================
# MCP Integration Configuration
plugins/agentic-qe-fleet/skills/qe-test-generation/evals/qe-test-generation.yaml
+2 -2
View File
@@ -5,8 +5,8 @@ description: >
Tests unit test generation, integration test creation, and edge case coverage.
models_to_test:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
mcp_integration:
enabled: true
plugins/agentic-qe-fleet/skills/risk-based-testing/evals/risk-based-testing.yaml
+2 -2
View File
@@ -5,8 +5,8 @@ description: >
Tests risk scoring, test prioritization, and coverage optimization.
models_to_test:
- claude-3.5-sonnet
- claude-3-haiku
- claude-sonnet-4-6 # Primary (high accuracy expected)
- claude-haiku-4-5 # Fast model (minimum quality floor)
mcp_integration:
enabled: true
src/cli/commands/eval.ts
+2 -2
View File
@@ -695,7 +695,7 @@ export function createEvalCommand(): Command {
.requiredOption('-s, --skill <skill>', 'Skill name to evaluate')
.requiredOption(
'-m, --model <model>',
'Model to use (e.g., claude-3.5-sonnet)'
'Model to use (e.g., claude-sonnet-4-6)'
)
.option('-p, --parallel', 'Enable parallel execution', false)
.option('-w, --workers <n>', 'Number of parallel workers', parseInt, 5)
@@ -731,7 +731,7 @@ export function createEvalCommand(): Command {
.option(
'--models <models>',
'Comma-separated models to test',
'claude-3.5-sonnet'
'claude-sonnet-4-6'
)
.option('-p, --parallel', 'Enable parallel execution', true)
.option('-w, --workers <n>', 'Number of parallel workers', parseInt, 5)