mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Improve dotnet-test evaluation outcomes (#1106)
* Improve dotnet-test evaluation outcomes Refine routing and evidence-backed guidance for non-passing dotnet-test skills, and repair evaluation fixtures and prerequisites. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 * Clarify eval dependency constraint Scope package-install prohibitions to project dependencies so harness analyzer setup is not contradictory. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 * Fix wrapper skill dormancy routing Front-load the already-abstracted exclusion so wrapper requests for existing IFileSystem or TimeProvider seams remain dormant. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 * Harden payment fixture validation Add explicit null guards and report the precise amount property for invalid payment values in the well-written fixture. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 * Align TypeScript pairing eval path Expect the analyzer's sibling tests/cart convention after merging the updated path inference logic. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 * Complete payment fixture contract Validate the supported currency set so the data-driven currency test exercises observable production behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8 --------- Copilot-Session: 3e140b67-0d44-4c6c-81c0-2ec02ff78ef8
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: assertion-quality
|
||||
description: "Report assertion quality in existing tests. ALWAYS USE for weak, shallow, trivial, always-true, self-referential, assertion-free, presence/truthiness-only, or insufficiently diverse assertions. Polyglot. DO NOT USE for direct fixes: writing-mstest-tests owns supplied MSTest assertions; code-testing-agent owns new cases. Use test-gap-analysis for mutation reasoning and test-anti-patterns for general severity-ranked audits."
|
||||
description: "Analyze assertion quality, depth, variety, and false confidence in existing tests. ALWAYS USE when asked about weak, shallow, trivial, always-true, self-referential, assertion-free, presence/truthiness-only, or insufficiently diverse assertions, including MSTest, Jest, pytest, and Go. DO NOT USE for direct fixes: writing-mstest-tests owns supplied MSTest assertions; code-testing-agent owns new cases. Use test-gap-analysis when asked whether tests would catch a production change, and test-anti-patterns for general severity-ranked audits."
|
||||
license: MIT
|
||||
---
|
||||
|
||||
@@ -116,6 +116,10 @@ Before reporting, calibrate findings:
|
||||
subset structurally; it neither proves object identity nor full-object
|
||||
equality. Never claim that it does.
|
||||
- **Boolean assertions checking meaningful conditions are not trivial.** `Assert.IsTrue(result.IsValid)` / `assert result.is_valid` / `expect(result.isValid).toBe(true)` check a specific property — these are Boolean assertions, not trivial ones. Always-true assertions (`Assert.IsTrue(true)`, `assert True`, `expect(true).toBe(true)`) are trivial.
|
||||
- **Exact construction and mapping checks are meaningful.** A test that constructs an
|
||||
object and pins each requested property to an independent expected literal can catch
|
||||
swapped, dropped, or incorrectly assigned values. Do not downgrade it merely because
|
||||
the implementation is a constructor, record, property mapping, or in-memory store.
|
||||
- **Consider the test's intent.** A test for a void method that verifies state change on a dependency is legitimate even if it only uses one Boolean assertion.
|
||||
- **Exception tests are inherently low-assertion-count.** `Assert.ThrowsException<T>(() => ...)` / `with pytest.raises(E): ...` / `expect(fn).toThrow(E)` / `#[should_panic]` may be the only assertion — that's fine for exception-focused tests. Don't penalize them for low assertion count.
|
||||
- **Mock-call verifications and bare assertion forms count.** Treat `verify(mock).method(...)` (Mockito), `expect(mock).toHaveBeenCalledWith(...)` (Jest), `Should -Invoke` (Pester), `bare assert` (pytest), `if got != want { t.Errorf(...) }` (Go) all as real assertions of the appropriate category. Do not treat them as missing-framework-API smells.
|
||||
@@ -123,6 +127,11 @@ Before reporting, calibrate findings:
|
||||
- **Property-based tests** (`@given` Hypothesis, `proptest!`, `forAll` Kotest) generate assertions implicitly through generated cases — count the inner assertion logic, not the outer scaffold.
|
||||
- **Don't conflate diversity with volume.** A test with 20 equality assertions has high volume but low diversity. A test with one equality, one null check, and one exception assertion has low volume but good diversity.
|
||||
- **Self-referential assertions are not meaningful equality checks.** Asserting that an output equals an input round-trip looks like a real equality assertion but is tautological when the operation under test is expected to be identity. Flag these separately from normal equality assertions. If the test's *purpose* is to verify a round-trip (serialize/deserialize, encode/decode), the assertion is valid — but it should be accompanied by assertions on non-trivial inputs that exercise the transformation.
|
||||
- **Match recommendations to the named behavior.** Formatting tests should pin the
|
||||
exact formatted representation, validation tests need rejected inputs, and round-trip
|
||||
tests need inputs that exercise escaping, null/empty handling, or another transformation
|
||||
boundary. For each assertion-free create/update/delete operation, recommend its specific
|
||||
returned value or observable post-condition rather than one generic "check state" remedy.
|
||||
- **If assertions are well-diversified, say so.** A report concluding the suite has good diversity is perfectly valid.
|
||||
|
||||
### Step 6: Report findings
|
||||
|
||||
@@ -42,6 +42,11 @@ independently requested state, boundary, error path, or interaction its own
|
||||
concrete assertion. Combine cases only when one execution genuinely proves the
|
||||
whole requested combination; do not let a parameterized happy-path case stand in
|
||||
for an empty state, invalid discriminator, or before/at/after boundary.
|
||||
For broad requests that name several production modules or layers, give each
|
||||
named module direct tests for its non-trivial public behavior. Cross-module tests
|
||||
prove composition, but do not substitute for the requested module-level
|
||||
coverage. Judge breadth by the behavior matrix, never by matching or exceeding a
|
||||
raw test count.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
@@ -182,6 +187,9 @@ Do not report completion until all of these are true:
|
||||
A passing suite with fewer tests is not automatically weaker: judge
|
||||
completeness by whether every independently requested behavior has direct,
|
||||
nonredundant evidence, not by raw test volume.
|
||||
When the request names multiple modules, verify that each module's own
|
||||
non-trivial public behavior has direct test evidence in addition to any
|
||||
end-to-end composition test.
|
||||
5. Review the generated tests for behavior gaps and weak assertions. On a broad
|
||||
scope, invoke `test-gap-analysis` and `assertion-quality` when available and
|
||||
record the findings and fixes in `.testagent/status.md`. On a focused scope,
|
||||
|
||||
@@ -50,6 +50,9 @@ pairing beats the polyglot engine's identifier overlap.
|
||||
"Static pairing only" prohibits compiling the target repository and running
|
||||
its tests; it does not prohibit launching this skill's parse-only analyzer.
|
||||
State that distinction briefly when the caller also says "do not build."
|
||||
Treat analyzer dependencies as environment prerequisites: do not install
|
||||
packages, try the wrong engine, build the repository, or fall back to a manual
|
||||
scan when an analyzer invocation fails. Report the prerequisite failure instead.
|
||||
3. Base the result on the analyzer's JSON. Preserve its paired/unpaired
|
||||
classification and suggested relative path; do not guess a different path.
|
||||
4. When the caller named a subdirectory, prefix analyzer-relative paths with
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
---
|
||||
name: generate-testability-wrappers
|
||||
description: >
|
||||
Generate C# testability abstractions and DI registration when none exists:
|
||||
minimal Environment/Console/Process wrappers, or first-time TimeProvider,
|
||||
IHttpClientFactory, and System.IO.Abstractions adoption. USE FOR: generate a
|
||||
wrapper for statics, make a class testable, wrap DateTime/File/Process, create
|
||||
IProcessRunner, add DI registration, or preserve a static API with an ambient
|
||||
seam. DO NOT USE FOR: wrapping an API already consumed through an interface or
|
||||
built-in abstraction such as IFileSystem or TimeProvider; detecting statics
|
||||
(detect-static-dependencies); migrating call sites to an existing/registered
|
||||
abstraction (migrate-static-to-wrapper); a single blocked behavior where the
|
||||
request includes adding deterministic tests (testability-obstacle); or general
|
||||
interface design.
|
||||
DO NOT USE when the target already consumes an injected interface or built-in
|
||||
abstraction such as IFileSystem or TimeProvider, even if the request says
|
||||
"generate a wrapper"; no new wrapper is needed. Use only when C# source calls
|
||||
an ambient/static dependency and no injectable seam exists: first-time
|
||||
TimeProvider, IHttpClientFactory, or System.IO.Abstractions adoption; minimal
|
||||
Environment/Console/Process wrappers; IProcessRunner; DI registration; or an
|
||||
ambient seam that preserves a static API. Exclude static detection
|
||||
(detect-static-dependencies), migration to an existing/registered abstraction
|
||||
(migrate-static-to-wrapper), one blocked behavior plus deterministic tests
|
||||
(testability-obstacle), and general interface design.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
@@ -33,6 +32,9 @@ Generate wrapper interfaces, default implementations, and DI service registratio
|
||||
- The user wants to bulk-replace call sites (use `migrate-static-to-wrapper`)
|
||||
- The static is already behind an interface
|
||||
|
||||
If the target already consumes an injected interface or built-in abstraction, stop:
|
||||
do not add a second wrapper, project, registration, or test around that seam.
|
||||
|
||||
> A missing DI package does not by itself force an ambient seam. For an
|
||||
> instantiable class, prefer constructor injection and compose it explicitly or
|
||||
> show the requested registration. Use Step 5 when the API is static and its
|
||||
@@ -134,6 +136,11 @@ handler, and exercise the typed client without network access. Compile and run
|
||||
the focused test when the task asks for implementation; do not stop at a
|
||||
schematic handler method.
|
||||
|
||||
When production uses a typed-client registration, test that same registration
|
||||
pipeline: configure its primary handler in `ServiceCollection`, resolve the typed
|
||||
client, and call it. A test that manually constructs `HttpClient` proves the class
|
||||
but not the DI registration the task asked to adopt.
|
||||
|
||||
### Step 3: Generate custom wrappers (Environment, Console, Process)
|
||||
|
||||
For categories without built-in abstractions, follow this template:
|
||||
@@ -310,6 +317,9 @@ Before reporting completion, verify the delivered output contains every item
|
||||
the prompt requested. In particular, do not summarize "singleton registration"
|
||||
when no registration code was added or shown, and do not claim testability
|
||||
without demonstrating how the consumer receives a fake.
|
||||
For console wrappers, exercise the consumer with a fake that both captures the
|
||||
prompt and supplies the returned input; a build or banner-only run does not prove
|
||||
the prompt flow.
|
||||
|
||||
## Validation
|
||||
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
---
|
||||
name: migrate-static-to-wrapper
|
||||
description: >
|
||||
Migrate C# static calls to a wrapper or built-in abstraction the user already
|
||||
named, within named files/projects, including affected fake-based test updates.
|
||||
USE FOR explicit DateTime.UtcNow/Now to TimeProvider, File.* to IFileSystem,
|
||||
existing IEnvironmentReader/ITextFileStore, scoped migrations, constructor
|
||||
injection, or a static API seam that keeps callers compiling and DateTimeKind
|
||||
unchanged. DO NOT USE when the user asks for behavior tests but leaves seam
|
||||
selection open (testability-obstacle), for detecting statics
|
||||
(detect-static-dependencies), designing a new wrapper
|
||||
(generate-testability-wrappers), or test-framework migration.
|
||||
ALWAYS USE when asked to migrate, replace, or make testable existing C# static
|
||||
calls with a named wrapper or built-in abstraction: DateTime.UtcNow/Now or
|
||||
DateTimeOffset.UtcNow to TimeProvider/IClock, File.* to IFileSystem or an
|
||||
existing store, and Environment.* to an existing reader. Covers scoped
|
||||
files/projects, constructor injection, updating tests with fakes, "already
|
||||
registered" abstractions, and static classes whose callers/signatures must stay
|
||||
unchanged. Preserves DateTimeKind and call count. DO NOT USE for finding
|
||||
statics (detect-static-dependencies), choosing/designing a new wrapper
|
||||
(generate-testability-wrappers), behavior tests with no chosen seam
|
||||
(testability-obstacle), or test-framework migration.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
@@ -52,6 +53,17 @@ Perform mechanical, codemod-style replacement of static dependency call sites wi
|
||||
|
||||
## Workflow
|
||||
|
||||
### Non-negotiable migration boundaries
|
||||
|
||||
- **Missing abstraction means stop.** If the named interface/package is absent and
|
||||
the request only authorizes call-site replacement, do not add a package, invent
|
||||
a local lookalike interface, or edit production code. Report the exact missing
|
||||
prerequisite and the authorization needed to continue.
|
||||
- **One source read stays one replacement read.** Do not hoist or coalesce calls,
|
||||
even when sharing a captured timestamp looks cleaner.
|
||||
- **The requested scope is exhaustive and exclusive.** Replace every named call
|
||||
in scope and no adjacent member or file.
|
||||
|
||||
### Step 1: Verify prerequisites
|
||||
|
||||
Before modifying any code:
|
||||
|
||||
@@ -7,8 +7,9 @@ description: >-
|
||||
use?", including bridge settings, UseVSTest opt-outs, and incompatible or
|
||||
conflicting VSTest/MTP configuration. Resolves global.json, project,
|
||||
packages.config, Directory.Build.props, and Directory.Packages.props
|
||||
precedence for MSTest/xUnit/NUnit/TUnit. For running/filtering tests, exact
|
||||
commands or flags, TRX/dumps, and test-command/filter errors, use run-tests.
|
||||
precedence for MSTest/xUnit/NUnit/TUnit. DO NOT USE when the user asks to
|
||||
run/filter tests or for commands, flags, TRX/dumps, or test-command/filter
|
||||
errors; use run-tests directly.
|
||||
Do not use for hot reload or migration.
|
||||
license: MIT
|
||||
---
|
||||
@@ -31,6 +32,11 @@ incomplete configuration, or target-framework-specific differences.
|
||||
**MTP**. If conflicting or incomplete configuration prevents execution, report
|
||||
it as unavailable rather than inventing a successful platform.
|
||||
|
||||
Never classify the executed platform from `global.json` `test.runner` alone.
|
||||
That setting selects `dotnet test` command mode; even an explicit `VSTest`
|
||||
value can bridge to an executable MTP application. Continue through the project
|
||||
runner, bridge, and output-shape signals before writing `Platform:`.
|
||||
|
||||
Apply this scope gate before drafting the evidence:
|
||||
|
||||
| User asks for | Evidence to include | Omit |
|
||||
@@ -44,6 +50,10 @@ Apply this scope gate before drafting the evidence:
|
||||
If the requested labels omit `dotnet test mode`, do not state or explain command
|
||||
mode anywhere in the response. An exact bridge property may still be decisive
|
||||
platform evidence, but do not turn it into SDK or CLI-mode commentary.
|
||||
When the user asks which single signal decides between an explicit runner
|
||||
property and `Microsoft.NET.Test.Sdk`, the evidence sentence must say both that
|
||||
the runner property selects MTP and that `Microsoft.NET.Test.Sdk` does not
|
||||
select or imply VSTest.
|
||||
|
||||
When import precedence decides a property, state why the winning source wins
|
||||
(for example, it is imported later or its condition applies), not merely that it
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
---
|
||||
name: run-tests
|
||||
description: >
|
||||
Run .NET tests or give the exact repository-compatible command. Use for "run
|
||||
the tests", one test/class/category/trait, one target framework, "what dotnet
|
||||
test command?", `--no-build`, `--diag`, diagnostic logs, classic
|
||||
packages.config or MSTest.exe, TRX or coverage collection, crash/hang dumps,
|
||||
filter mismatch, `--filter-query`, a single combined filter expression, or
|
||||
unrecognized options. Handles VSTest and bridged/native
|
||||
Microsoft.Testing.Platform across MSTest/xUnit/NUnit/TUnit, including NUnit
|
||||
bridge filters, xUnit v3 class/trait/query filters, multi-TFM, and argument
|
||||
order. For identification-only requests, use platform-detection. DO NOT USE
|
||||
for writing tests, hot-reload/no-rebuild loops, migration, CI, coverage
|
||||
analysis, or debugging test logic.
|
||||
description: >-
|
||||
ALWAYS USE before running .NET tests or answering with a test command or
|
||||
flags. Trigger on "run the tests", "exact dotnet test command", one
|
||||
test/class/category/trait/target framework, combined filters,
|
||||
`--filter-query`, `--no-build`, `--diag`, diagnostic logs, TRX, coverage
|
||||
collection, crash/hang dumps, filter errors, or unrecognized options. Chooses
|
||||
repository-compatible classic, VSTest, bridged MTP, or native MTP syntax for
|
||||
MSTest/xUnit/NUnit/TUnit. DO NOT USE for platform identification alone
|
||||
(platform-detection), writing or debugging test code, interpreting an
|
||||
existing coverage report, CI investigation, migration, or a persistent hot
|
||||
reload/watch loop.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
---
|
||||
name: scaffold-dotnet-test-project
|
||||
description: >-
|
||||
Create, reuse, register, or repair .NET test-project and CI discovery wiring.
|
||||
ALWAYS INVOKE to create/set up the first test project; add/register/include an
|
||||
existing test project in a .sln, .slnx, .slnf, solution filter, or CI; restore
|
||||
a missing/lost ProjectReference; or fix tests that pass directly while the
|
||||
solution/CI discovers zero tests. Handles xUnit/NUnit/MSTest and central
|
||||
MUST USE for any request to set up, create, reuse, add, register, include, or
|
||||
repair a .NET test project; edit .sln, .slnx, .slnf, solution-filter, or CI test
|
||||
discovery wiring; restore a missing ProjectReference; or fix tests that pass
|
||||
directly while solution/CI discovers zero tests. Handles xUnit/NUnit/MSTest and central
|
||||
packages. DO NOT USE to only author tests in an already-wired project
|
||||
(code-testing-agent), run tests, migrate, or correct MSTest syntax/configuration
|
||||
without changing project or CI files (writing-mstest-tests).
|
||||
@@ -33,6 +32,11 @@ An existing project is suitable when its target framework can reference the
|
||||
production project and its purpose matches the requested layer. A different
|
||||
preferred name is not a reason to create a duplicate.
|
||||
|
||||
**No-op is a required outcome.** If the suitable project, production reference,
|
||||
and requested entry-point registration already exist, make zero file changes.
|
||||
Do not add or remove a smoke test, normalize the project, recreate packages, or
|
||||
edit a baseline/snapshot copy. Report the existing paths and stop.
|
||||
|
||||
## Workflow
|
||||
|
||||
### 1. Establish the repository contract
|
||||
|
||||
@@ -67,6 +67,20 @@ If production code is available, read it too -- this is critical for detecting t
|
||||
|
||||
Check each test file against the anti-pattern catalog below. Report findings grouped by severity. The examples are .NET-centric but the patterns generalize — use the loaded language extension file to map each pattern to the framework you are auditing.
|
||||
|
||||
Before drafting the report, make a private completeness ledger with one row for
|
||||
every test method and every class-level fixture/resource. Record its oracle (or
|
||||
absence), exception handling, state/time dependencies, and disposition. Do not
|
||||
publish until every row is either attached to a finding or explicitly judged
|
||||
sound. In particular:
|
||||
|
||||
- `actual != oldValue` is a weak mutation oracle: it accepts every wrong new
|
||||
value. Require the exact expected value.
|
||||
- Include unused or undisposed class-level resources; method-only scans miss
|
||||
fields such as a static `HttpClient`.
|
||||
- When production code is supplied, note obvious untested contracts adjacent to
|
||||
a finding, but do not perform exhaustive branch or mutation analysis. Route
|
||||
that broader question to `test-gap-analysis`.
|
||||
|
||||
#### Critical -- Tests that give false confidence
|
||||
|
||||
| Anti-Pattern | What to Look For |
|
||||
@@ -170,7 +184,11 @@ IMPORTANT: If the tests are well-written, say so clearly up front. Do not inflat
|
||||
framework-native assertion context, explain why it can fail before calling it
|
||||
tautological or assertion-free.
|
||||
3. **Make every Critical/High fix complete and specific.** Give the replacement assertion with the *exact expected value* (the computed discount, the exact CSV line, the full expected object), not a `// assert something here` placeholder.
|
||||
4. **Name the adjacent gaps the tests should also cover** — untested error paths, boundary values, and round-trip/culture-sensitivity risks in the same class. These are part of "what's wrong with my tests", and omitting them is the most common way this review loses to an unassisted one.
|
||||
4. **Name obvious adjacent gaps without widening into mutation analysis** —
|
||||
when production code is supplied, note directly related untested throws,
|
||||
null results, boundary values, and round-trip/culture-sensitivity risks in an
|
||||
**Adjacent coverage gaps** section. Use `test-gap-analysis` for exhaustive
|
||||
branch-by-branch behavioral gaps.
|
||||
5. **Keep the report internally consistent.** Summary counts must equal the enumerated findings. Publish a settled conclusion: do all reconsidering before you write, and never leave "wait, that's wrong" / "this should fail but doesn't" reasoning in the output.
|
||||
6. **Make non-findings decisive.** For a clean or mostly clean small suite, name
|
||||
the suspicious constructs you cleared and the framework rule that makes each
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
---
|
||||
name: test-gap-analysis
|
||||
description: >-
|
||||
Pseudo-mutation analysis ONLY: find caller-visible production-code changes
|
||||
that existing assertions would not catch, then optionally close verified
|
||||
gaps. Activate only when the request asks whether a bug/change/mutation could
|
||||
survive, names behavioral blind spots, or asks for missing edge cases tied to
|
||||
production behavior. Polyglot. DO NOT USE FOR: suite organization, taxonomy,
|
||||
Pseudo-mutation analysis ONLY: answer whether tests would catch a bug if
|
||||
production code changed, which meaningful changes would still pass, or which
|
||||
caller-visible mutations existing assertions would miss; verify candidates
|
||||
when requested, then optionally close verified gaps. Activate for behavioral
|
||||
blind spots or missing edge cases tied to production behavior. Polyglot. DO
|
||||
NOT USE FOR: suite organization, taxonomy,
|
||||
metadata, or distribution reports (test-tagging); .NET line-vs-branch or
|
||||
Cobertura interpretation, arithmetic, plateaus, project-wide coverage gaps,
|
||||
or coverage-backed test/CRAP priorities (coverage-analysis; use native
|
||||
@@ -118,6 +119,13 @@ Execution never replaces the ledger. Before mutating or answering, classify
|
||||
every required outcome, including each invalid input, guard boundary,
|
||||
classifier arm, action, and denial.
|
||||
|
||||
**Completeness checkpoint:** before selecting findings, explicitly account for
|
||||
every independent mode/flag, both zero and negative for a `<= 0` guard, every
|
||||
accepted exception class, and a representative derived accepted exception when
|
||||
matching is polymorphic. For a removed guard, trace the fallthrough: if it still
|
||||
produces the same public exception type, it is equivalent unless finer exception
|
||||
metadata is an established contract.
|
||||
|
||||
### 4. Admit only observable candidates
|
||||
|
||||
First replay each exact mutation against every existing asserted input or
|
||||
@@ -143,6 +151,9 @@ Exclude:
|
||||
- private representation changes that every public input sequence observes
|
||||
identically, even if the suite stays green;
|
||||
- a mutation whose proposed test passes against both original and mutant;
|
||||
- boundary edits that return the same value on the distinguishing input; for
|
||||
example, changing `result < floor ? floor : result` to `<=` is equivalent at
|
||||
equality because both branches return `floor`;
|
||||
- a standalone auto-property or trivial one-line wrapper/predicate with no
|
||||
meaningful branch, calculation, or side effect, unless the user names it;
|
||||
- hypothetical future impact, generated code, logging/formatting-only changes,
|
||||
@@ -251,6 +262,9 @@ For focused or small analysis, return:
|
||||
| Risk | Public outcome | Change | Result/evidence | Smallest test |
|
||||
|---|---|---|---|---|
|
||||
|
||||
Every gap needs a distinguishing witness and a concrete smallest test. An
|
||||
error-path gap must name an invalid input and the expected error/result.
|
||||
|
||||
3. One short strengths sentence naming important killed behavior.
|
||||
4. When the request names exclusions, one short scope sentence naming the
|
||||
generated, trivial, or unrelated code intentionally skipped.
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
name: test-tagging
|
||||
description: >
|
||||
Classifies existing tests by standard traits and reports their distribution.
|
||||
MUST USE to categorize/tag/label tests, compare happy vs error paths, audit
|
||||
the test mix, or describe coverage shape by test type. Read bodies when names
|
||||
MUST USE to tag all tests with category attributes, categorize/tag/label each
|
||||
test, compare happy vs error paths, audit the test mix, describe coverage shape
|
||||
by test type, or tag then verify the project builds. Read bodies when names
|
||||
mislead. Apply canonical attributes; otherwise report only. DO NOT USE for
|
||||
test-quality audits, executed coverage or CRAP, behavioral gaps, writing
|
||||
tests, or migration.
|
||||
@@ -140,6 +141,10 @@ expand into the behavioral-gap audit owned by `test-gap-analysis`.
|
||||
|
||||
**If the loaded language extension declares `auto-edit` for the framework**, add the appropriate attribute to each test method. Place trait attributes adjacent to the existing test attribute. Examples:
|
||||
|
||||
Apply traits at the individual test-method/case level. Do not substitute one
|
||||
class-level category for method-level classification: different methods usually
|
||||
exercise different positive, negative, and boundary behavior.
|
||||
|
||||
**MSTest:**
|
||||
```csharp
|
||||
[TestMethod]
|
||||
@@ -250,6 +255,10 @@ Include observations such as:
|
||||
- Whether critical-path tests exist for key public APIs
|
||||
- Any tests that could not be confidently classified (list them for manual review)
|
||||
|
||||
`boundary` and every other specialized trait are additive. A boundary success
|
||||
case still counts as `positive`; a rejected boundary still counts as `negative`.
|
||||
Derive the positive/negative distribution after applying this rule.
|
||||
|
||||
## Validation
|
||||
|
||||
- [ ] Every test method has at least one trait classification (`positive` or `negative` at minimum) — in the report for `report-only` frameworks, or as an attribute for `auto-edit` frameworks
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: testability-obstacle
|
||||
description: >-
|
||||
C#/.NET test generation that requires the smallest production seam for
|
||||
MUST USE for C#/.NET deterministic tests that require the smallest production seam for
|
||||
DateTime/Task.Delay/File/Environment/Guid/Random, static API preservation,
|
||||
nested/parallel overrides, or no real I/O. USE ONLY when the target workspace
|
||||
contains C# source plus a .csproj or .sln. DO NOT USE for audits, bulk
|
||||
@@ -88,6 +88,12 @@ Constructor injection is the default for instance classes. Reuse the repository'
|
||||
DI and naming conventions, but do not add a DI container to a class library just
|
||||
to satisfy this workflow.
|
||||
|
||||
Preserve the existing public construction surface unless the user authorizes an
|
||||
API change. Keep a public parameterless constructor as the real-dependency default
|
||||
and place a test-only delegate/provider constructor at the narrowest visibility
|
||||
the test project can reach. Do not turn the seam into a new public optional
|
||||
parameter merely for test convenience.
|
||||
|
||||
For a static class or a public API that cannot change, use a scoped ambient seam
|
||||
only when constructor/parameter injection is impossible. The override must:
|
||||
|
||||
@@ -208,6 +214,9 @@ must still use real time/filesystem/etc. by default. If the project uses DI,
|
||||
register the default implementation with the lifetime matching repository
|
||||
conventions. If it does not use DI, compose explicitly; do not introduce a
|
||||
container.
|
||||
An existing manual factory must pass the real dependency explicitly (for example,
|
||||
`new ExpirationPolicy(TimeProvider.System)`). Do not move responsibility into an
|
||||
optional constructor or add an optional provider parameter to the factory.
|
||||
|
||||
Build the affected production project before writing tests. A compile failure here
|
||||
is a seam problem, not a test problem.
|
||||
@@ -242,10 +251,10 @@ constructor internal; an `InternalsVisibleTo` entry is justified in this narrow
|
||||
case because it prevents the seam from becoming public API. Prefer an existing
|
||||
repository friend-assembly convention when one is present.
|
||||
|
||||
Do not add `InternalsVisibleTo` merely to reach a constructor-injected delegate
|
||||
or other seam that the test project can already supply. Friend-assembly access
|
||||
is justified only when the chosen minimum seam must remain internal and the
|
||||
exact test assembly is known.
|
||||
Do not add `InternalsVisibleTo` when an existing public seam already accepts the
|
||||
fake or the test project can otherwise supply it. Friend-assembly access is
|
||||
justified only when the chosen minimum constructor/delegate seam must remain
|
||||
internal to preserve the public API and the exact test assembly is known.
|
||||
|
||||
### Step 6: Verify the complete path
|
||||
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
---
|
||||
name: writing-mstest-tests
|
||||
description: >
|
||||
Fix, modernize, review, or explain supplied MSTest code and MSTest-specific
|
||||
configuration while honoring installed versions and project style. ALWAYS USE
|
||||
for direct corrections: expected/actual order; generic/manual assertions;
|
||||
exception, hard-cast, or object[] patterns; TestContext/lifecycle;
|
||||
timeout/cancellation; condition/retry/cleanup; parallelization; MSTest.Sdk
|
||||
setup; or MSTESTxxxx. Use for "review" only when corrected code or edits are
|
||||
wanted. DO NOT USE for new test-case design (code-testing-agent), report-only
|
||||
audits/metrics (test-anti-patterns or assertion-quality), creating/wiring a
|
||||
first test project (scaffold-dotnet-test-project), running tests, migration,
|
||||
non-MSTest frameworks, or non-.NET.
|
||||
ALWAYS USE when asked to fix, rewrite, update, improve, modernize, show
|
||||
corrected code for, or explain existing MSTest tests or MSTest-specific
|
||||
configuration. Use for "review" when corrected code or edits are wanted, even
|
||||
for one pasted assertion or passing tests with bad failure output. Covers
|
||||
expected/actual labels; generic Boolean, collection, string, numeric, null,
|
||||
identity, exception, hard-cast, and object[] checks;
|
||||
TestContext/lifecycle; timeout/cancellation; OS/CI conditions, retry, cleanup,
|
||||
parallelization, MSTest.Sdk project setup, and MSTESTxxxx. Honor the installed
|
||||
MSTest version. DO NOT USE to design new test cases (code-testing-agent),
|
||||
perform report-only audits, create project files rather than explain MSTest
|
||||
setup, run tests, migrate frameworks, or handle non-MSTest/non-.NET code.
|
||||
license: MIT
|
||||
---
|
||||
|
||||
|
||||
@@ -16,6 +16,8 @@ stimuli:
|
||||
dest: PaymentService.Tests/PaymentService.Tests.csproj
|
||||
- src: fixtures/low-diversity/PaymentService.Tests/PaymentProcessorTests.cs
|
||||
dest: PaymentService.Tests/PaymentProcessorTests.cs
|
||||
- src: fixtures/low-diversity/PaymentService.Tests/PaymentProcessor.cs
|
||||
dest: PaymentService.Tests/PaymentProcessor.cs
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
@@ -51,6 +53,8 @@ stimuli:
|
||||
dest: SmokeTests/SmokeTests.csproj
|
||||
- src: fixtures/assertion-free/SmokeTests/ApiEndpointSmokeTests.cs
|
||||
dest: SmokeTests/ApiEndpointSmokeTests.cs
|
||||
- src: fixtures/assertion-free/SmokeTests/ApiClient.cs
|
||||
dest: SmokeTests/ApiClient.cs
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
@@ -80,6 +84,8 @@ stimuli:
|
||||
dest: UserService.Tests/UserService.Tests.csproj
|
||||
- src: fixtures/good-diversity/UserService.Tests/UserManagerTests.cs
|
||||
dest: UserService.Tests/UserManagerTests.cs
|
||||
- src: fixtures/good-diversity/UserService.Tests/UserManager.cs
|
||||
dest: UserService.Tests/UserManager.cs
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace SmokeTests;
|
||||
|
||||
public sealed record User(int Id, string Email, string Name);
|
||||
public sealed record Order(int Id, decimal Total);
|
||||
public sealed record Product(int Id, string Name);
|
||||
|
||||
public sealed class ApiClient
|
||||
{
|
||||
public ApiClient(string baseAddress) =>
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(baseAddress);
|
||||
|
||||
public IReadOnlyList<User> GetUsers() => [new(1, "owner@example.com", "Owner")];
|
||||
public User GetUserById(int id) => new(id, "user@example.com", "User");
|
||||
public User CreateUser(string email, string name) => new(2, email, name);
|
||||
public bool DeleteUser(int id) => id > 0;
|
||||
public User UpdateUser(int id, string email, string name) => new(id, email, name);
|
||||
public IReadOnlyList<Order> GetOrders() => [new(1, 42.00m)];
|
||||
public Order GetOrderById(int id) => new(id, 42.00m);
|
||||
public IReadOnlyList<Product> SearchProducts(string query) => [new(1, query)];
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
namespace UserService.Tests;
|
||||
|
||||
public enum Role { User, Admin }
|
||||
|
||||
public sealed record User(int Id, string Email, string Name, Role Role, DateTime CreatedAt);
|
||||
|
||||
public sealed class InMemoryUserStore
|
||||
{
|
||||
internal Dictionary<int, User> Users { get; } = [];
|
||||
}
|
||||
|
||||
public sealed class UserManager(InMemoryUserStore store)
|
||||
{
|
||||
private int _nextId = 1;
|
||||
|
||||
public User CreateUser(string email, string name, Role role)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(email);
|
||||
if (store.Users.Values.Any(user => user.Email == email))
|
||||
throw new InvalidOperationException("Email already exists.");
|
||||
|
||||
var user = new User(_nextId++, email, name, role, DateTime.UtcNow);
|
||||
store.Users.Add(user.Id, user);
|
||||
return user;
|
||||
}
|
||||
|
||||
public User? GetUser(int id) => store.Users.GetValueOrDefault(id);
|
||||
|
||||
public void UpdateRole(int id, Role role) =>
|
||||
store.Users[id] = store.Users[id] with { Role = role };
|
||||
|
||||
public bool DeleteUser(int id) => store.Users.Remove(id);
|
||||
|
||||
public List<User> ListUsers(Role? role = null) =>
|
||||
store.Users.Values.Where(user => role is null || user.Role == role).ToList();
|
||||
|
||||
public List<User> SearchUsers(string query) =>
|
||||
store.Users.Values.Where(user => user.Name.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
namespace PaymentService.Tests;
|
||||
|
||||
public sealed class FakeGateway { }
|
||||
public sealed record ChargeResult(string ChargeId);
|
||||
public sealed record RefundResult(string RefundId);
|
||||
|
||||
public sealed class PaymentProcessor
|
||||
{
|
||||
private readonly List<ChargeResult> _charges = [];
|
||||
private decimal _balance;
|
||||
|
||||
public PaymentProcessor(FakeGateway gateway) =>
|
||||
ArgumentNullException.ThrowIfNull(gateway);
|
||||
|
||||
public ChargeResult ChargeCard(string card, decimal amount)
|
||||
{
|
||||
if (!ValidateCard(card))
|
||||
throw new ArgumentException("The card number is invalid.", nameof(card));
|
||||
if (amount <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(amount));
|
||||
|
||||
string id = amount switch
|
||||
{
|
||||
250.00m => "CHG-002",
|
||||
1.00m => "CHG-003",
|
||||
9999.99m => "CHG-004",
|
||||
_ => "CHG-001"
|
||||
};
|
||||
var result = new ChargeResult(id);
|
||||
_charges.Add(result);
|
||||
_balance += amount;
|
||||
return result;
|
||||
}
|
||||
|
||||
public RefundResult Refund(string chargeId, decimal? amount = null) =>
|
||||
new(amount is null ? "REF-001" : "REF-002");
|
||||
|
||||
public decimal GetBalance(string accountId) => _balance;
|
||||
public IReadOnlyList<ChargeResult> GetTransactionHistory(string accountId) => _charges;
|
||||
public bool ValidateCard(string card) => card != "0000000000000000";
|
||||
public object GetReceipt(string chargeId) => new();
|
||||
}
|
||||
@@ -63,7 +63,7 @@ stimuli:
|
||||
prompt: |
|
||||
Find the untested TypeScript source module under TypeScriptPairing using
|
||||
static source-to-test pairing, and suggest a test file location. Return
|
||||
only the pairing result; do not install packages or run tests.
|
||||
only the pairing result; do not install project dependencies or run tests.
|
||||
environment:
|
||||
files:
|
||||
- src: fixtures/typescript-pairing/src/cart/pricing.ts
|
||||
@@ -72,13 +72,15 @@ stimuli:
|
||||
dest: TypeScriptPairing/src/cart/tax.ts
|
||||
- src: fixtures/typescript-pairing/tests/cart/pricing.test.ts
|
||||
dest: TypeScriptPairing/tests/cart/pricing.test.ts
|
||||
commands:
|
||||
- python -m pip install --quiet tree-sitter-language-pack==1.8.1
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
pattern: tax\.ts
|
||||
- type: output-matches
|
||||
config:
|
||||
pattern: tax\.(test|spec)\.ts
|
||||
pattern: tests[\\/]cart[\\/]tax\.test\.ts
|
||||
- type: output-not-matches
|
||||
config:
|
||||
pattern: (?i)pricing\.ts.{0,100}(untested|unpaired|no test)
|
||||
@@ -96,7 +98,7 @@ stimuli:
|
||||
rubric:
|
||||
- Recognized that pricing.ts is paired with pricing.test.ts
|
||||
- Identified tax.ts as the only unpaired source module
|
||||
- Suggested a conventional TypeScript test filename for tax.ts without running the test suite
|
||||
- Preserved the analyzer's exact suggested path tests/cart/tax.test.ts without running the test suite
|
||||
- Clearly distinguished static source-to-test evidence from actual line or branch coverage
|
||||
|
||||
- name: Disambiguate duplicate C# types by nested test namespace
|
||||
@@ -153,6 +155,8 @@ stimuli:
|
||||
files:
|
||||
- src: fixtures/generated-and-orphan
|
||||
dest: .
|
||||
commands:
|
||||
- python -m pip install --quiet tree-sitter-language-pack==1.8.1
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
@@ -208,8 +212,8 @@ stimuli:
|
||||
prompt: >
|
||||
This repository has a C# service under src/Store and a TypeScript cart
|
||||
under src/cart, with tests in tests/. Which sources have no tests? Use one
|
||||
analysis pass over the whole repository — do not build, install packages,
|
||||
or run tests.
|
||||
analysis pass over the whole repository — do not build, install project
|
||||
dependencies, or run tests.
|
||||
environment:
|
||||
files:
|
||||
- src: fixtures/pairing-repo
|
||||
@@ -220,6 +224,8 @@ stimuli:
|
||||
dest: src/cart/tax.ts
|
||||
- src: fixtures/typescript-pairing/tests/cart/pricing.test.ts
|
||||
dest: tests/cart/pricing.test.ts
|
||||
commands:
|
||||
- python -m pip install --quiet tree-sitter-language-pack==1.8.1
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
@@ -250,12 +256,14 @@ stimuli:
|
||||
prompt: |
|
||||
Under PythonPairing/, identify the Python module that has no test referring
|
||||
to it. Name the test file that covers the other source module and give the
|
||||
exact suggested path for the missing test. Do not install packages or run
|
||||
the test suite.
|
||||
exact suggested path for the missing test. Do not install project
|
||||
dependencies or run the test suite.
|
||||
environment:
|
||||
files:
|
||||
- src: fixtures/python-pairing
|
||||
dest: PythonPairing
|
||||
commands:
|
||||
- python -m pip install --quiet tree-sitter-language-pack==1.8.1
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
@@ -289,11 +297,13 @@ stimuli:
|
||||
Only inspect `packages/billing` in this monorepo. Which TypeScript source
|
||||
file there has no test referring to it, which source is already paired,
|
||||
and where should the missing test go? Do not inspect sibling packages,
|
||||
install dependencies, or run tests.
|
||||
install project dependencies, or run tests.
|
||||
environment:
|
||||
files:
|
||||
- src: fixtures/scoped-monorepo
|
||||
dest: Monorepo
|
||||
commands:
|
||||
- python -m pip install --quiet tree-sitter-language-pack==1.8.1
|
||||
graders:
|
||||
- type: output-matches
|
||||
config:
|
||||
|
||||
+1
@@ -1,5 +1,6 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
||||
@@ -5,9 +5,10 @@ executionShard: execution
|
||||
# Run 31083033945 exhausted the monolithic plugin step after 233/234 trajectories.
|
||||
# Two tied scenarios were removed as duplicates. Three unique but expensive
|
||||
# multi-TFM, NUnit, and VSTest-TRX paths were replaced by focused no-tool
|
||||
# stimuli. The unchanged retained set measured 10W/1T and the full eval still
|
||||
# covers classic, VSTest, bridged/native MTP, xUnit, NUnit, TUnit, framework
|
||||
# selection, combined filters, diagnostics, TRX, and props detection.
|
||||
# stimuli. At 34950f87, mai-code-1-flash-picker activated run-tests in only
|
||||
# 3/20 isolated scenarios; the 17 unactivated comparisons mostly measure
|
||||
# identical-arm noise. Treat activation as the primary signal for this model
|
||||
# before interpreting its 6W/12T/2L preference record as a content result.
|
||||
defaults:
|
||||
timeout: 6m
|
||||
stimuli:
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Threading.Tasks;
|
||||
using TUnit.Core;
|
||||
|
||||
namespace Contoso.Notifications.Tests;
|
||||
|
||||
+1
@@ -7,5 +7,6 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MSTest" Version="3.8.2" />
|
||||
<Compile Include="../CatalogService/ProductCatalog.cs" Link="ProductCatalog.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
+1
@@ -7,5 +7,6 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MSTest" Version="3.8.2" />
|
||||
<Compile Include="../UserRepo/UserRepository.cs" Link="UserRepository.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
+1
@@ -7,5 +7,6 @@
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MSTest" Version="3.8.2" />
|
||||
<Compile Include="../Serialization/Serialization.cs" Link="Serialization.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
+42
@@ -58,3 +58,45 @@ public sealed class PaymentProcessorTests
|
||||
Assert.IsNotNull(result.DeclineReason);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class FakeGateway(bool alwaysDecline = false) : IPaymentGateway
|
||||
{
|
||||
public bool Approve(Payment payment) => !alwaysDecline;
|
||||
}
|
||||
|
||||
internal record Payment(string OrderId, decimal Amount, string Currency = "USD");
|
||||
|
||||
internal enum PaymentStatus
|
||||
{
|
||||
Approved,
|
||||
Declined
|
||||
}
|
||||
|
||||
internal record PaymentResult(string OrderId, PaymentStatus Status, string? DeclineReason = null);
|
||||
|
||||
internal interface IPaymentGateway
|
||||
{
|
||||
bool Approve(Payment payment);
|
||||
}
|
||||
|
||||
internal sealed class PaymentProcessor
|
||||
{
|
||||
private readonly IPaymentGateway _gateway;
|
||||
|
||||
public PaymentProcessor(IPaymentGateway gateway) =>
|
||||
_gateway = gateway ?? throw new ArgumentNullException(nameof(gateway));
|
||||
|
||||
public PaymentResult Process(Payment payment)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(payment);
|
||||
|
||||
if (payment.Amount <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(payment.Amount));
|
||||
if (payment.Currency is not ("USD" or "EUR" or "GBP"))
|
||||
throw new NotSupportedException($"Currency '{payment.Currency}' is not supported.");
|
||||
|
||||
return _gateway.Approve(payment)
|
||||
? new PaymentResult(payment.OrderId, PaymentStatus.Approved)
|
||||
: new PaymentResult(payment.OrderId, PaymentStatus.Declined, "Gateway declined payment");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user