diff --git a/.github/skills/assertion-quality/skill.md b/.github/skills/assertion-quality/skill.md new file mode 100644 index 0000000..fa8b630 --- /dev/null +++ b/.github/skills/assertion-quality/skill.md @@ -0,0 +1,176 @@ +--- +name: assertion-quality +description: "Analyzes the variety and depth of assertions across test suites in any language. Use when the user asks to evaluate assertion quality, find shallow tests, identify assertion-free tests (no assertions or only trivial ones like Assert.IsNotNull / toBeTruthy()), flag self-referential or tautological assertions, measure assertion diversity, or audit whether tests verify different facets of behavior. Polyglot: .NET, Python, TS/JS, Java, Go, Ruby, Rust, Swift, Kotlin, PowerShell, C++. DO NOT USE FOR: writing new tests (use code-testing-agent / writing-mstest-tests), mutation reasoning about whether tests would catch a bug (use test-gap-analysis), or a general severity-ranked anti-pattern audit (use test-anti-patterns), fixing or rewriting assertions, or writing, fixing, or modernizing MSTest tests, assertions, or attributes (use writing-mstest-tests)." +license: MIT +--- + +# Assertion Diversity Analysis + +Analyze test code in any supported language to measure how varied and meaningful the assertions are. Produce a metrics report that reveals whether tests verify different facets of correctness — not just "output equals X" but also structure, exceptions, state transitions, side effects, and invariants. + +> **Language-specific guidance**: Call the `test-analysis-extensions` skill to discover available extension files, then read the file matching the target codebase's language and framework (e.g., `dotnet.md` for .NET, `python.md` for pytest, `typescript.md` for Jest, `go.md` for the standard `testing` package). You MUST read the relevant extension file before classifying assertions, because assertion APIs differ significantly across frameworks. + +## Why Assertion Diversity Matters + +Low assertion diversity signals shallow testing. Tests may pass while bugs hide in unasserted logic. Common symptoms: + +| Problem | Symptom | Consequence | +|---------|---------|-------------| +| Trivial assertions | Test contains only `Assert.IsNotNull(result)` / `assert result is not None` / `expect(x).toBeDefined()` | Test passes but doesn't verify correctness | +| Single-value obsession | Always check one field or return value | Bugs in unasserted logic slip through | +| No negative assertions | Never check what shouldn't happen | Regressions sneak in through false positives | +| No state checks | Don't verify object state changes | Missed side-effects or lifecycle issues | +| No structural checks | Only assert top-level value | Bugs in nested objects go unnoticed | +| Assertion-free tests | Tests that call but don't verify | Code coverage lies; false security | + +## When to Use + +- User asks to evaluate assertion quality or depth +- User asks "are my tests actually testing anything meaningful?" +- User wants to know if test assertions are too shallow or trivial +- User asks for assertion coverage metrics or diversity analysis +- User suspects tests give false confidence despite passing +- The `code-testing-generator` agent (or any test-generation workflow) calls this skill as a pre-completion self-review step on freshly generated tests, before declaring the run finished + +## When Not to Use + +- User wants to write new tests (use `code-testing-agent` for any language, or `writing-mstest-tests` for MSTest specifically) +- User wants to detect anti-patterns beyond assertions (use `test-anti-patterns`) +- User wants to fix or rewrite assertions (help them directly) +- User asks about code coverage percentages (out of scope — this analyzes assertion quality, not line coverage) + +## Inputs + +| Input | Required | Description | +|-------|----------|-------------| +| Test code | Yes | One or more test files or a test project directory to analyze | +| Production code | No | The code under test, to evaluate whether assertions cover the important behaviors | + +## Workflow + +### Step 1: Detect language and load extension + +Identify the target codebase's language and test framework. Call the `test-analysis-extensions` skill and read the matching extension file (e.g., `extensions/dotnet.md` for .NET, `extensions/python.md` for pytest, `extensions/typescript.md` for Jest/Vitest, `extensions/go.md` for Go). The extension file lists the framework-specific assertion APIs you will classify in Step 3. + +### Step 2: Gather the test code + +Read all test files the user provides. If the user points to a directory or project, scan for all test files using the markers in the language extension file (e.g., `[TestMethod]` for MSTest, `def test_*` for pytest, `it()` / `test()` for Jest, `func TestXxx` for Go). + +### Step 3: Classify every assertion + +For each test method, identify all assertions and classify them into these language-neutral categories: + +| Category | What it verifies | Examples across languages | +|----------|------------------|----------------------------| +| **Equality** | Return value matches expected | `Assert.AreEqual` (MSTest), `Assert.Equal` (xUnit), `assert x == y` (pytest), `expect(x).toBe(y)` (Jest), `assertEquals` (JUnit), `if got != want { t.Error... }` / `assert.Equal(t, want, got)` (Go), `x shouldBe y` (Kotest), `Should -Be` (Pester), `EXPECT_EQ` (GoogleTest) | +| **Boolean** | Condition holds | `Assert.IsTrue`, `assert flag` (Python), `expect(x).toBeTruthy()` (Jest), `assertTrue` (JUnit), `assert.True(t, ok)` (testify), `x.shouldBeTrue()` (Kotest), `Should -BeTrue` (Pester), `EXPECT_TRUE` | +| **Null / None / Nil** | Presence/absence of value | `Assert.IsNull` (.NET), `assert x is None` (pytest), `expect(x).toBeNull()` (Jest), `assertNull` (JUnit), `assert.Nil(t, v)` (testify), `XCTAssertNil` (XCTest), `Should -BeNullOrEmpty` (Pester) | +| **Exception / Error** | Error handling behavior | `Assert.Throws()`, `pytest.raises(E)`, `expect(fn).toThrow(E)`, `assertThrows`, `assert.Error(t, err)` / `assert.ErrorIs`, `#[should_panic]` (Rust), `XCTAssertThrowsError`, `Should -Throw`, `EXPECT_THROW` | +| **Type checks** | Runtime type correctness | `Assert.IsInstanceOfType`, `assert isinstance(x, T)`, `expect(x).toBeInstanceOf(T)`, `assertInstanceOf`, `assert.IsType(t, T{}, v)`, `assert!(matches!(value, Pattern))` (Rust), `Should -BeOfType` | +| **String** | Text content and format | `StringAssert.Contains`, `assert sub in s`, `expect(s).toMatch(/x/)`, `assertTrue(s.contains(...))`, `assert.Contains(t, s, sub)`, `s shouldContain sub`, `Should -Match`, `EXPECT_THAT(s, HasSubstr(...))` | +| **Collection** | Collection contents and structure | `CollectionAssert.Contains`, `assert item in collection`, `expect(arr).toContain(x)`, `assertIterableEquals`, `assert.Contains(t, slice, item)`, `col shouldContainExactly listOf(...)`, `Should -Contain`, `EXPECT_THAT(c, ElementsAre(...))` | +| **Comparison** | Ordering and magnitude | `Assert.IsTrue(x > y)`, `Is.GreaterThan`, `assert x > y`, `expect(x).toBeGreaterThan(y)`, `assertTrue(x > y)`, `assert.Greater(t, x, y)` (testify) | +| **Approximate** | Floating-point or tolerance-based | `Assert.AreEqual(expected, actual, delta)`, `pytest.approx(y)`, `expect(x).toBeCloseTo(y)`, `assertEquals(x, y, delta)`, `assert.InDelta(t, x, y, delta)`, `EXPECT_NEAR`, `EXPECT_DOUBLE_EQ` | +| **Negative** | What should NOT happen | `Assert.AreNotEqual`, `assert x != y`, `expect(x).not.toBe(y)`, `assertNotEquals`, `assert.NotEqual(t, x, y)`, `refute` (Minitest / Ruby), `Should -Not -Be` | +| **State / Side-effect** | State transitions and side effects | Assertions on object properties after mutation; mock-call verifications: `mock.Verify(...)` (Moq), `mock_method.assert_called_with(...)` (Python `unittest.mock`), `expect(mock).toHaveBeenCalledWith(...)` (Jest), `verify(mock).method(...)` (Mockito), `Should -Invoke` (Pester), `expect { code }.to change(obj, :attr)` (RSpec) | +| **Structural / Deep** | Deep object correctness | `Assert.AreEqual` with rich-equality types, `assertThat(obj).usingRecursiveComparison()` (AssertJ), `.toEqual({...})` (Jest deep equality), `cmp.Diff` (Go go-cmp), snapshot tests (`.toMatchSnapshot()`, `syrupy`, `SnapshotTesting`), `assertThat(col).extracting(...)` (AssertJ chains) | + +A single assertion can belong to multiple categories (e.g., `Assert.AreNotEqual` is both Equality and Negative; `expect(mock).toHaveBeenCalledWith(...)` is both State/Side-effect and a specific-call assertion). + +Read the loaded language extension file for the exact framework-specific list of assertion APIs. + +### Step 4: Compute metrics + +Calculate these metrics for the test suite: + +#### Per-test metrics +- **Assertion count**: Number of assertions in each test method +- **Assertion categories**: Which categories each test uses + +#### Suite-wide metrics +- **Average assertions per test**: Total assertions / total test methods +- **Assertion type spread**: Number of distinct assertion categories used across the suite (out of 12) +- **Tests with zero assertions**: Count and percentage of test methods with no assertions at all +- **Tests with only trivial assertions**: Count and percentage of tests where every assertion is only a null check or `Assert.IsTrue(true)` — trivial means no meaningful value verification +- **Tests with self-referential assertions**: Count and percentage of tests whose assertions compare an input to a round-tripped or identity-transformed version of itself (e.g., `Assert.AreEqual(input, Parse(input.ToString()))`) or assert a field against itself (`Assert.AreEqual(dto.Name, dto.Name)`). These are tautological — they verify the plumbing, not the behavior. +- **Tests with negative assertions**: Count and percentage (target: at least 10% of tests should verify what should NOT happen) +- **Tests with exception assertions**: Count and percentage +- **Tests with state/side-effect assertions**: Count and percentage +- **Tests with structural/deep assertions**: Count and percentage +- **Single-category tests**: Count and percentage of tests that use only one assertion category + +### Step 5: Apply calibration rules + +Before reporting, calibrate findings: + +- **Trivial means truly trivial.** A null/None/nil check alone is trivial (`Assert.IsNotNull(result)`, `assert result is not None`, `expect(x).toBeDefined()`). But a null check followed by a meaningful value assertion is not trivial — the null check is a guard before the real assertion. Only flag a test as "trivial" if it has no meaningful value assertions. +- **Boolean assertions checking meaningful conditions are not trivial.** `Assert.IsTrue(result.IsValid)` / `assert result.is_valid` / `expect(result.isValid).toBe(true)` check a specific property — these are Boolean assertions, not trivial ones. Always-true assertions (`Assert.IsTrue(true)`, `assert True`, `expect(true).toBe(true)`) are trivial. +- **Consider the test's intent.** A test for a void method that verifies state change on a dependency is legitimate even if it only uses one Boolean assertion. +- **Exception tests are inherently low-assertion-count.** `Assert.ThrowsException(() => ...)` / `with pytest.raises(E): ...` / `expect(fn).toThrow(E)` / `#[should_panic]` may be the only assertion — that's fine for exception-focused tests. Don't penalize them for low assertion count. +- **Mock-call verifications and bare assertion forms count.** Treat `verify(mock).method(...)` (Mockito), `expect(mock).toHaveBeenCalledWith(...)` (Jest), `Should -Invoke` (Pester), `bare assert` (pytest), `if got != want { t.Errorf(...) }` (Go) all as real assertions of the appropriate category. Do not treat them as missing-framework-API smells. +- **Snapshot assertions** (`.toMatchSnapshot()`, `syrupy`, `SnapshotTesting`) count as Structural/Deep assertions. Flag stale or never-updated snapshots separately. +- **Property-based tests** (`@given` Hypothesis, `proptest!`, `forAll` Kotest) generate assertions implicitly through generated cases — count the inner assertion logic, not the outer scaffold. +- **Don't conflate diversity with volume.** A test with 20 equality assertions has high volume but low diversity. A test with one equality, one null check, and one exception assertion has low volume but good diversity. +- **Self-referential assertions are not meaningful equality checks.** Asserting that an output equals an input round-trip looks like a real equality assertion but is tautological when the operation under test is expected to be identity. Flag these separately from normal equality assertions. If the test's *purpose* is to verify a round-trip (serialize/deserialize, encode/decode), the assertion is valid — but it should be accompanied by assertions on non-trivial inputs that exercise the transformation. +- **If assertions are well-diversified, say so.** A report concluding the suite has good diversity is perfectly valid. + +### Step 6: Report findings + +**Scale the report depth to the size and complexity of the suite.** The structure below is the full template for a substantial suite (roughly 15+ tests or a multi-file project). For a small or simple input (a single file with only a handful of tests), do not emit every section — a padded multi-section dashboard on a trivial input reads as noise and buries the answer. Instead, answer the user's question directly and concisely: which tests are assertion-free or trivial-only, the overall assertion-quality verdict, and concrete recommendations (still distinguishing intentional smoke tests from tests masquerading as real verification). Use only the sections that carry real signal for the input at hand; a short metric summary plus the assertion-free list and recommendations is often enough. Never omit the rubric-relevant substance (assertion-free/trivial identification, the quality verdict, and concrete recommendations) — only trim structural overhead that adds no information. + +Present the analysis in this structure: + +1. **Summary Dashboard** — A quick-reference table of key metrics: + ``` + | Metric | Value | Assessment | + |-------------------------------|--------|------------| + | Total tests | 25 | — | + | Average assertions per test | 2.4 | Moderate | + | Assertion type spread | 5/12 | Low | + | Tests with zero assertions | 3 (12%)| Concerning | + | Tests with only trivial asserts | 4 (16%)| Acceptable | + | Tests with negative assertions | 2 (8%) | Below target | + | Single-category tests | 15 (60%)| High | + ``` + +2. **Category Breakdown** — For each assertion category, show: + - How many tests use it + - Representative examples from the code + - Whether it's overused or underused relative to the code under test + +3. **Gap Analysis** — Based on the production code (if available), identify: + - Behaviors that are tested but only with equality checks + - Error paths with no exception assertions + - State-changing methods with no state verification + - Collections returned but never checked for contents + +4. **Recommendations** — Prioritized list of improvements: + - Which tests would benefit most from additional assertion types + - Which assertion categories are missing and why they matter + - Concrete examples of assertions that could be added + +5. **Assertion-free tests** — If any exist, list each one with its method name and what it appears to be testing, so the user can decide whether to add assertions or mark them as intentional smoke tests. + +## Validation + +- [ ] Every assertion in the test suite was classified into at least one category +- [ ] Metrics are computed correctly (counts add up) +- [ ] Trivial-assertion tests are correctly identified (not over-flagged) +- [ ] Exception tests are not penalized for low assertion count +- [ ] Boolean assertions on meaningful properties are not classified as trivial +- [ ] Recommendations are concrete (name specific test methods and suggest specific assertion types) +- [ ] If the suite has good diversity, the report acknowledges this + +## Common Pitfalls + +| Pitfall | Solution | +|---------|----------| +| Penalizing exception tests for low assertion count | Exception assertions are complete on their own — skip count warnings for these | +| Flagging null/None/nil checks before value checks as trivial | Only flag tests where the null/None/nil check is the ONLY assertion | +| Counting any Boolean assertion as trivial | Only always-true assertions (`Assert.IsTrue(true)`, `assert True`, `expect(true).toBe(true)`) are trivial | +| Ignoring framework differences | Each framework has distinct assertion APIs — always read the matching language extension first. MSTest's `Assert.AreEqual`, xUnit's `Assert.Equal`, NUnit's `Is.EqualTo`, pytest's bare `assert ==`, Jest's `expect().toBe()`, Go's `if … { t.Error… }` all map to the **Equality** category | +| Treating bare assertion forms as missing-framework | Bare `assert` (pytest), `if got != want { t.Error... }` (Go), and `assert!()` (Rust) are canonical — count them in the right category | +| Treating mock-call verifications as assertion-free | `verify(mock).method(...)`, `expect(mock).toHaveBeenCalledWith(...)`, `Should -Invoke` are State/Side-effect assertions | +| Recommending diversity for diversity's sake | Only suggest adding assertion types that would catch real bugs in the code under test | +| Missing implicit assertions | Exception assertions are both Exception and Negative; snapshot/property-based tests are real assertions with implicit structure | +| Async tests with unawaited assertions | TUnit, Jest with `.resolves`/`.rejects`, pytest-asyncio, Swift Testing, and Kotest all silently pass tests where assertions are not `await`ed — treat as assertion-free even when assertion calls are present | diff --git a/.github/skills/code-testing-agent/extensions/dotnet.md b/.github/skills/code-testing-agent/extensions/dotnet.md new file mode 100644 index 0000000..e362ad8 --- /dev/null +++ b/.github/skills/code-testing-agent/extensions/dotnet.md @@ -0,0 +1,111 @@ +# .NET Extension + +Language-specific guidance for .NET (C#/F#/VB) test generation. + +## Build Commands + +| Scope | Command | +|-------|---------| +| Specific test project | `dotnet build MyProject.Tests.csproj` | +| Full solution (final validation) | `dotnet build MySolution.sln --no-incremental` | +| From repo root (no .sln) | `dotnet build --no-incremental` | + +- Use `--no-restore` if dependencies are already restored +- Use `-v:q` (quiet) to reduce output noise +- Always use `--no-incremental` for the final validation build — incremental builds hide errors like CS7036 + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests | `dotnet test` | +| Filtered | `dotnet test --filter "FullyQualifiedName~ClassName"` | +| After build | `dotnet test --no-build` | + +- Use `--no-build` if already built +- Use `-v:q` for quieter output + +## Lint Command + +```bash +dotnet format --include path/to/file.cs +dotnet format MySolution.sln # full solution +``` + +## Project Reference Validation + +Before writing test code, read the test project's `.csproj` to verify it has `` entries for the assemblies your tests will use. If a reference is missing, add it: + +```xml + + + +``` + +This prevents CS0234 ("namespace not found") and CS0246 ("type not found") errors. + +## Common CS Error Codes + +| Error | Meaning | Fix | +|-------|---------|-----| +| CS0234 | Namespace not found | Add `` to the source project in the test `.csproj` | +| CS0246 | Type not found | Add `using Namespace;` or add missing `` | +| CS0103 | Name not found | Check spelling, add `using` statement | +| CS1061 | Missing member | Verify method/property name matches the source code exactly | +| CS0029 | Type mismatch | Cast or change the type to match the expected signature | +| CS7036 | Missing required parameter | Read the constructor/method signature and pass all required arguments | + +## `.csproj` / `.sln` Handling + +- During phase implementation, build only the specific test `.csproj` for speed +- For the final validation, build the full `.sln` with `--no-incremental` +- Full-solution builds catch cross-project reference errors invisible in scoped builds + +## MSTest Template + +```csharp +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace ProjectName.Tests; + +[TestClass] +public sealed class ClassNameTests +{ + [TestMethod] + public void MethodName_Scenario_ExpectedResult() + { + // Arrange + var sut = new ClassName(); + + // Act + var result = sut.MethodName(input); + + // Assert + Assert.AreEqual(expected, result); + } + + [TestMethod] + [DataRow(2, 3, 5, DisplayName = "Positive numbers")] + [DataRow(-1, 1, 0, DisplayName = "Negative and positive")] + public void Add_ValidInputs_ReturnsSum(int a, int b, int expected) + { + // Act + var result = _sut.Add(a, b); + + // Assert + Assert.AreEqual(expected, result); + } +} +``` + +## Coverage XML Parsing + +If `.testagent/initial_coverage.xml` exists, it uses Cobertura/VS format: + +- `module` elements with `line_coverage` attribute — identifies which assemblies have low coverage +- `function` elements with `line_coverage="0.00"` — identifies completely untested methods +- `range` elements with `covered="no"` — identifies specific uncovered lines + +## Skip Coverage Tools + +Do not configure or run code coverage measurement tools (coverlet, dotnet-coverage, XPlat Code Coverage). These tools have inconsistent cross-configuration behavior and waste significant time. Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-agent/skill.md b/.github/skills/code-testing-agent/skill.md new file mode 100644 index 0000000..565c4f5 --- /dev/null +++ b/.github/skills/code-testing-agent/skill.md @@ -0,0 +1,197 @@ +--- +name: code-testing-agent +description: >- + Generates comprehensive, workable unit tests for any programming language + using a multi-agent pipeline. Use when asked to generate tests, write unit + tests, improve test coverage, add test coverage, create test files, or test a + codebase. Supports C#, TypeScript, JavaScript, Python, Go, Rust, Java, and + more. Orchestrates research, planning, and implementation phases to produce + tests that compile, pass, and follow project conventions. +--- + +# Code Testing Generation Skill + +An AI-powered skill that generates comprehensive, workable unit tests for any programming language using a coordinated multi-agent pipeline. + +## When to Use This Skill + +Use this skill when you need to: + +- Generate unit tests for an entire project or specific files +- Improve test coverage for existing codebases +- Create test files that follow project conventions +- Write tests that actually compile and pass +- Add tests for new features or untested code + +## When Not to Use + +- Running or executing existing tests (use the `run-tests` skill) +- Migrating between test frameworks (use migration skills) +- Writing tests specifically for MSTest patterns (use `writing-mstest-tests`) +- Debugging failing test logic + +## How It Works + +This skill coordinates multiple specialized agents in a **Research → Plan → Implement** pipeline: + +### Pipeline Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ TEST GENERATOR │ +│ Coordinates the full pipeline and manages state │ +└─────────────────────┬───────────────────────────────────────┘ + │ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ +┌───────────┐ ┌───────────┐ ┌───────────────┐ +│ RESEARCHER│ │ PLANNER │ │ IMPLEMENTER │ +│ │ │ │ │ │ +│ Analyzes │ │ Creates │ │ Writes tests │ +│ codebase │→ │ phased │→ │ per phase │ +│ │ │ plan │ │ │ +└───────────┘ └───────────┘ └───────┬───────┘ + │ + ┌─────────┬───────┼───────────┐ + ▼ ▼ ▼ ▼ + ┌─────────┐ ┌───────┐ ┌───────┐ ┌───────┐ + │ BUILDER │ │TESTER │ │ FIXER │ │LINTER │ + │ │ │ │ │ │ │ │ + │ Compiles│ │ Runs │ │ Fixes │ │Formats│ + │ code │ │ tests │ │ errors│ │ code │ + └─────────┘ └───────┘ └───────┘ └───────┘ +``` + +## Step-by-Step Instructions + +### Step 1: Determine the user request + +Make sure you understand what user is asking and for what scope. +When the user does not express strong requirements for test style, coverage goals, or conventions, source the guidelines from [unit-test-generation.prompt.md](unit-test-generation.prompt.md). This prompt provides best practices for discovering conventions, parameterization strategies, coverage goals (aim for 80%), and language-specific patterns. + +### Step 2: Invoke the Test Generator + +Start by calling the `code-testing-generator` agent with your test generation request: + +``` +Generate unit tests for [path or description of what to test], following the [unit-test-generation.prompt.md](unit-test-generation.prompt.md) guidelines +``` + +The Test Generator will manage the entire pipeline automatically. + +### Step 3: Research Phase (Automatic) + +The `code-testing-researcher` agent analyzes your codebase to understand: + +- **Language & Framework**: Detects C#, TypeScript, Python, Go, Rust, Java, etc. +- **Testing Framework**: Identifies MSTest, xUnit, Jest, pytest, go test, etc. +- **Project Structure**: Maps source files, existing tests, and dependencies +- **Build Commands**: Discovers how to build and test the project + +Output: `.testagent/research.md` + +### Step 4: Planning Phase (Automatic) + +The `code-testing-planner` agent creates a structured implementation plan: + +- Groups files into logical phases (2-5 phases typical) +- Prioritizes by complexity and dependencies +- Specifies test cases for each file +- Defines success criteria per phase + +Output: `.testagent/plan.md` + +### Step 5: Implementation Phase (Automatic) + +The `code-testing-implementer` agent executes each phase sequentially: + +1. **Read** source files to understand the API +2. **Write** test files following project patterns +3. **Build** using the `code-testing-builder` sub-agent to verify compilation +4. **Test** using the `code-testing-tester` sub-agent to verify tests pass +5. **Fix** using the `code-testing-fixer` sub-agent if errors occur +6. **Lint** using the `code-testing-linter` sub-agent for code formatting + +Each phase completes before the next begins, ensuring incremental progress. + +### Coverage Types + +- **Happy path**: Valid inputs produce expected outputs +- **Edge cases**: Empty values, boundaries, special characters +- **Error cases**: Invalid inputs, null handling, exceptions + +## State Management + +All pipeline state is stored in `.testagent/` folder: + +| File | Purpose | +| ------------------------ | ---------------------------- | +| `.testagent/research.md` | Codebase analysis results | +| `.testagent/plan.md` | Phased implementation plan | +| `.testagent/status.md` | Progress tracking (optional) | + +## Examples + +### Example 1: Full Project Testing + +``` +Generate unit tests for my Calculator project at C:\src\Calculator +``` + +### Example 2: Specific File Testing + +``` +Generate unit tests for src/services/UserService.ts +``` + +### Example 3: Targeted Coverage + +``` +Add tests for the authentication module with focus on edge cases +``` + +## Agent Reference + +| Agent | Purpose | +| -------------------------- | -------------------- | +| `code-testing-generator` | Coordinates pipeline | +| `code-testing-researcher` | Analyzes codebase | +| `code-testing-planner` | Creates test plan | +| `code-testing-implementer` | Writes test files | +| `code-testing-builder` | Compiles code | +| `code-testing-tester` | Runs tests | +| `code-testing-fixer` | Fixes errors | +| `code-testing-linter` | Formats code | + +## Requirements + +- Project must have a build/test system configured +- Testing framework should be installed (or installable) +- VS Code with GitHub Copilot extension + +## Troubleshooting + +### Tests don't compile + +The `code-testing-fixer` agent will attempt to resolve compilation errors. Check `.testagent/plan.md` for the expected test structure. Check the `extensions/` folder for language-specific error code references (e.g., `extensions/dotnet.md` for .NET). + +### Tests fail + +Most failures in generated tests are caused by **wrong expected values in assertions**, not production code bugs: + +1. Read the actual test output +2. Read the production code to understand correct behavior +3. Fix the assertion, not the production code +4. Never mark tests `[Ignore]` or `[Skip]` just to make them pass + +### Wrong testing framework detected + +Specify your preferred framework in the initial request: "Generate Jest tests for..." + +### Environment-dependent tests fail + +Tests that depend on external services, network endpoints, specific ports, or precise timing will fail in CI environments. Focus on unit tests with mocked dependencies instead. + +### Build fails on full solution + +During phase implementation, build only the specific test project for speed. After all phases, run a full non-incremental workspace build to catch cross-project errors. diff --git a/.github/skills/code-testing-agent/unit-test-generation.prompt.md b/.github/skills/code-testing-agent/unit-test-generation.prompt.md new file mode 100644 index 0000000..ccdbbbc --- /dev/null +++ b/.github/skills/code-testing-agent/unit-test-generation.prompt.md @@ -0,0 +1,173 @@ +--- +description: >- + Best practices and guidelines for generating comprehensive, + parameterized unit tests with 80% code coverage across any programming + language +--- + +# Unit Test Generation Prompt + +You are an expert code generation assistant specialized in writing concise, effective, and logical unit tests. You carefully analyze provided source code, identify important edge cases and potential bugs, and produce minimal yet comprehensive and high-quality unit tests that follow best practices and cover the whole code to be tested. Aim for 80% code coverage. + +## Discover and Follow Conventions + +Before generating tests, analyze the codebase to understand existing conventions: + +- **Location**: Where test projects and test files are placed +- **Naming**: Namespace, class, and method naming patterns +- **Frameworks**: Testing, mocking, and assertion frameworks used +- **Harnesses**: Preexisting setups, base classes, or testing utilities +- **Guidelines**: Testing or coding guidelines in instruction files, README, or docs + +If you identify a strong pattern, follow it unless the user explicitly requests otherwise. If no pattern exists and there's no user guidance, use your best judgment. + +## Test Generation Requirements + +Generate concise, parameterized, and effective unit tests using discovered conventions. + +- **Prefer mocking** over generating one-off testing types +- **Prefer unit tests** over integration tests, unless integration tests are clearly needed and can run locally +- **Traverse code thoroughly** to ensure high coverage (80%+) of the entire scope +- Continue generating tests until you reach the coverage target or have covered all non-trivial public surface area + +### Key Testing Goals + +| Goal | Description | +| ----------------------------- | ---------------------------------------------------------------------------------------------------- | +| **Minimal but Comprehensive** | Avoid redundant tests | +| **Logical Coverage** | Focus on meaningful edge cases, domain-specific inputs, boundary values, and bug-revealing scenarios | +| **Core Logic Focus** | Test positive cases and actual execution logic; avoid low-value tests for language features | +| **Balanced Coverage** | Don't let negative/edge cases outnumber tests of actual logic | +| **Best Practices** | Use Arrange-Act-Assert pattern and proper naming (`Method_Condition_ExpectedResult`) | +| **Buildable & Complete** | Tests must compile, run, and contain no hallucinated or missed logic | + +## Parameterization + +- Prefer parameterized tests (e.g., `[DataRow]`, `[Theory]`, `@pytest.mark.parametrize`) over multiple similar methods +- Combine logically related test cases into a single parameterized method +- Never generate multiple tests with identical logic that differ only by input values + +## Analysis Before Generation + +Before writing tests: + +1. **Analyze** the code line by line to understand what each section does +2. **Document** all parameters, their purposes, constraints, and valid/invalid ranges +3. **Identify** potential edge cases and error conditions +4. **Describe** expected behavior under different input conditions +5. **Note** dependencies that need mocking +6. **Consider** concurrency, resource management, or special conditions +7. **Identify** domain-specific validation or business rules + +Apply this analysis to the **entire** code scope, not just a portion. + +## Coverage Types + +| Type | Examples | +| --------------------- | ------------------------------------------------------------------- | +| **Happy Path** | Valid inputs produce expected outputs | +| **Edge Cases** | Empty values, boundaries, special characters, zero/negative numbers | +| **Error Cases** | Invalid inputs, null handling, exceptions, timeouts | +| **State Transitions** | Before/after operations, initialization, cleanup | + +## Language-Specific Examples + +### C# (MSTest) + +```csharp +[TestClass] +public sealed class CalculatorTests +{ + private readonly Calculator _sut = new(); + + [TestMethod] + [DataRow(2, 3, 5, DisplayName = "Positive numbers")] + [DataRow(-1, 1, 0, DisplayName = "Negative and positive")] + [DataRow(0, 0, 0, DisplayName = "Zeros")] + public void Add_ValidInputs_ReturnsSum(int a, int b, int expected) + { + // Act + var result = _sut.Add(a, b); + + // Assert + Assert.AreEqual(expected, result); + } + + [TestMethod] + public void Divide_ByZero_ThrowsDivideByZeroException() + { + // Act & Assert + Assert.ThrowsException(() => _sut.Divide(10, 0)); + } +} +``` + +### TypeScript (Jest) + +```typescript +describe("Calculator", () => { + let sut: Calculator; + + beforeEach(() => { + sut = new Calculator(); + }); + + it.each([ + [2, 3, 5], + [-1, 1, 0], + [0, 0, 0], + ])("add(%i, %i) returns %i", (a, b, expected) => { + expect(sut.add(a, b)).toBe(expected); + }); + + it("divide by zero throws error", () => { + expect(() => sut.divide(10, 0)).toThrow("Division by zero"); + }); +}); +``` + +### Python (pytest) + +```python +import pytest +from calculator import Calculator + +class TestCalculator: + @pytest.fixture + def sut(self): + return Calculator() + + @pytest.mark.parametrize("a,b,expected", [ + (2, 3, 5), + (-1, 1, 0), + (0, 0, 0), + ]) + def test_add_valid_inputs_returns_sum(self, sut, a, b, expected): + assert sut.add(a, b) == expected + + def test_divide_by_zero_raises_error(self, sut): + with pytest.raises(ZeroDivisionError): + sut.divide(10, 0) +``` + +## Output Requirements + +- Tests must be **complete and buildable** with no placeholder code +- Follow the **exact conventions** discovered in the target codebase +- Include **appropriate imports** and setup code +- Add **brief comments** explaining non-obvious test purposes +- Place tests in the **correct location** following project structure + +## Build and Verification + +- **Scoped builds during development**: Build the specific test project during implementation for faster iteration +- **Final full-workspace build**: After all test generation is complete, run a full non-incremental build from the workspace root to catch cross-project errors +- **API signature verification**: Before calling any method in test code, verify the exact parameter types, count, and order by reading the source code +- **Project reference validation**: Before writing test code, verify the test project references all source projects the tests will use. Check the `extensions/` folder for language-specific guidance (e.g., `extensions/dotnet.md` for .NET) + +## Test Scope Guidelines + +- **Write unit tests, not integration/acceptance tests**: Focus on testing individual classes and methods with mocked dependencies +- **No external dependencies**: Never write tests that call external URLs, bind to network ports, require service discovery, or depend on precise timing +- **Mock everything external**: HTTP clients, database connections, file systems, network endpoints — all should be mocked in unit tests +- **Fix assertions, not production code**: When tests fail, read the production code, understand its actual behavior, and update the test assertion diff --git a/.github/skills/code-testing-extensions/extensions/cpp-examples.md b/.github/skills/code-testing-extensions/extensions/cpp-examples.md new file mode 100644 index 0000000..186fe9b --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/cpp-examples.md @@ -0,0 +1,292 @@ +# C++ Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a C++ codebase using CMake + Catch2. These show what each pipeline phase produces for a small library project. + +> GoogleTest follows the same shape. Replace `TEST_CASE` / `SECTION` / `REQUIRE` with `TEST` / `EXPECT_*` and link the test executable to `GTest::gtest_main` instead of `Catch2::Catch2WithMain`. + +## Source Under Test + +A simple `InvoiceService` in a CMake project: + +```text +CMakeLists.txt +include/contoso/billing/ + invoice.hpp + invoice_repository.hpp + invoice_service.hpp +src/invoice_service.cpp +tests/CMakeLists.txt (links Catch2::Catch2WithMain) +``` + +```cpp +// src/invoice_service.cpp +#include "contoso/billing/invoice_service.hpp" + +#include +#include +#include + +namespace contoso::billing { + +InvoiceService::InvoiceService(InvoiceRepository& repository, Clock clock) + : repository_(repository), clock_(std::move(clock)) {} + +double InvoiceService::calculate_total(const Invoice& invoice) const { + if (invoice.line_items.empty()) { + throw std::invalid_argument("invoice has no line items"); + } + + double subtotal = 0.0; + for (const LineItem& item : invoice.line_items) { + subtotal += static_cast(item.quantity) * item.unit_price; + } + + return std::round((subtotal + subtotal * invoice.tax_rate) * 100.0) / 100.0; +} + +Invoice InvoiceService::get_by_id(int id) const { + auto invoice = repository_.find(id); + if (!invoice.has_value()) { + throw std::out_of_range("invoice not found"); + } + + return *invoice; +} + +void InvoiceService::mark_as_paid(int id) { + auto invoice = repository_.find(id); + if (!invoice.has_value()) { + throw std::out_of_range("invoice not found"); + } + if (invoice->status == InvoiceStatus::paid) { + throw std::logic_error("invoice is already paid"); + } + + invoice->status = InvoiceStatus::paid; + invoice->paid_at = clock_(); + repository_.update(*invoice); +} + +} // namespace contoso::billing +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: C++20 +- **Build System**: CMake (preset `ninja-debug` present) +- **Test Framework**: Catch2 v3 (detected via `find_package(Catch2 3 REQUIRED)`) + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Configure**: `cmake --preset ninja-debug` +- **Build tests**: `cmake --build --preset ninja-debug --target invoice_service_tests` +- **Test**: `ctest --preset ninja-debug --output-on-failure` +- **Coverage (if configured)**: rebuild with `--coverage`, then use `gcov` or `llvm-cov` + +## Files to Test + +### High Priority +| File | Classes/Functions | Testability | Notes | +|------|-------------------|-------------|-------| +| src/invoice_service.cpp | InvoiceService: calculate_total, get_by_id, mark_as_paid | High | Repository is an interface; clock dependency is injectable | + +## Testing Patterns +- No existing patterns; recommend Catch2 `TEST_CASE` blocks, `SECTION` cases, `Approx` for floating-point assertions, and a hand-written fake repository. +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate Catch2 tests for InvoiceService covering pure calculation logic, +repository lookup behavior, and the paid-state transition. + +## Commands +- **Build**: `cmake --build --preset ninja-debug --target invoice_service_tests` +- **Test**: `ctest --preset ninja-debug --output-on-failure` + +## Phase 1: InvoiceService + +### Files to Test +- **Source**: `src/invoice_service.cpp` +- **Test File**: `tests/invoice_service_tests.cpp` + +**Methods to Test**: +1. `calculate_total` — tax, zero tax, rounding, empty line items +2. `get_by_id` — existing invoice and missing invoice +3. `mark_as_paid` — success with fixed clock, already-paid, missing +``` + +## Sample Generated Test File + +```cpp +// tests/invoice_service_tests.cpp +#include "contoso/billing/invoice_service.hpp" + +#include +#include + +#include +#include +#include +#include + +using Catch::Matchers::ContainsSubstring; + +namespace contoso::billing { +namespace { + +class FakeRepository final : public InvoiceRepository { +public: + std::optional find(int id) override { + auto it = invoices.find(id); + return it == invoices.end() ? std::nullopt : std::optional{it->second}; + } + + void update(const Invoice& invoice) override { + updated = invoice; + invoices[invoice.id] = invoice; + } + + std::unordered_map invoices; + std::optional updated; +}; + +Invoice make_invoice(int id = 1) { + return Invoice{ + .id = id, + .status = InvoiceStatus::pending, + .tax_rate = 0.10, + .line_items = {LineItem{.quantity = 1, .unit_price = 100.0}}, + }; +} + +} // namespace + +TEST_CASE("InvoiceService calculates totals", "[invoice-service]") { + FakeRepository repository; + InvoiceService sut(repository, [] { return std::chrono::system_clock::time_point{}; }); + + SECTION("single item with tax") { + REQUIRE(sut.calculate_total(make_invoice()) == Catch::Approx(110.0)); + } + + SECTION("multi quantity with zero tax") { + Invoice invoice = make_invoice(); + invoice.tax_rate = 0.0; + invoice.line_items = {LineItem{.quantity = 3, .unit_price = 25.0}}; + + REQUIRE(sut.calculate_total(invoice) == Catch::Approx(75.0)); + } + + SECTION("rounds to two decimals") { + Invoice invoice = make_invoice(); + invoice.tax_rate = 0.07; + invoice.line_items = {LineItem{.quantity = 2, .unit_price = 9.99}}; + + REQUIRE(sut.calculate_total(invoice) == Catch::Approx(21.38).epsilon(0.001)); + } + + SECTION("empty line items throw") { + Invoice invoice = make_invoice(); + invoice.line_items.clear(); + + REQUIRE_THROWS_WITH(sut.calculate_total(invoice), ContainsSubstring("no line items")); + } +} + +TEST_CASE("InvoiceService uses the repository", "[invoice-service]") { + const auto fixed_time = std::chrono::system_clock::time_point{std::chrono::seconds{123}}; + FakeRepository repository; + repository.invoices.emplace(42, make_invoice(42)); + InvoiceService sut(repository, [fixed_time] { return fixed_time; }); + + SECTION("get_by_id returns an existing invoice") { + REQUIRE(sut.get_by_id(42).id == 42); + } + + SECTION("get_by_id throws when missing") { + REQUIRE_THROWS_WITH(sut.get_by_id(999), ContainsSubstring("not found")); + } + + SECTION("mark_as_paid updates status, date, and repository") { + repository.invoices.emplace(1, make_invoice(1)); + + sut.mark_as_paid(1); + + REQUIRE(repository.updated.has_value()); + REQUIRE(repository.updated->status == InvoiceStatus::paid); + REQUIRE(repository.updated->paid_at == fixed_time); + } + + SECTION("mark_as_paid rejects an already-paid invoice") { + Invoice invoice = make_invoice(2); + invoice.status = InvoiceStatus::paid; + repository.invoices.emplace(2, invoice); + + REQUIRE_THROWS_WITH(sut.mark_as_paid(2), ContainsSubstring("already paid")); + } +} + +} // namespace contoso::billing +``` + +## Sample Fix Cycle + +When the implementer hits a compile or runner issue, the fixer agent diagnoses and resolves it. + +**Build output:** + +```text +error: cannot declare variable 'repository' to be of abstract type 'FakeRepository' +note: missing pure virtual method 'InvoiceRepository::update' +``` + +**Fixer diagnosis:** The fake repository implemented `find` but not the full `InvoiceRepository` interface. + +**Fix applied:** Add `void update(const Invoice& invoice) override` to `FakeRepository` and record the updated invoice for assertions. + +**Rebuild + rerun:** `cmake --build --preset ninja-debug --target invoice_service_tests && ctest --preset ninja-debug --output-on-failure` → SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: contoso-billing (C++ / CMake) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 9 | +| Tests passing | 9 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `tests/invoice_service_tests.cpp` (2 Catch2 test cases, 9 sections) + +### Coverage +- InvoiceService.calculate_total — tax, zero tax, rounding, empty input +- InvoiceService.get_by_id — found and missing branches +- InvoiceService.mark_as_paid — success and already-paid branches + +### Build / Test Validation +- Configure: ✅ `cmake --preset ninja-debug` +- Build: ✅ `cmake --build --preset ninja-debug --target invoice_service_tests` +- Test: ✅ `ctest --preset ninja-debug --output-on-failure` +``` diff --git a/.github/skills/code-testing-extensions/extensions/cpp.md b/.github/skills/code-testing-extensions/extensions/cpp.md new file mode 100644 index 0000000..c48467e --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/cpp.md @@ -0,0 +1,509 @@ +# C++ Extension + +Language-specific guidance for C++ test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `*_test.cpp`, `*_tests.cpp`, `test_*.cc`, `tests/`, `test/`, `unittests/`, and any CTest/Bazel/Make test targets. Copy the framework, naming, fixtures, assertions, and helper style already in use. +2. **Build configuration** — inspect `CMakeLists.txt`, `CMakePresets.json`, `Makefile`, `WORKSPACE`, `MODULE.bazel`, `BUILD`, `BUILD.bazel`, `meson.build`, or CI scripts before inventing commands. +3. **Dependencies** — detect GoogleTest, GoogleMock, Catch2, doctest, Boost.Test, or a custom harness from package manifests and target links. +4. **Compiler and standard** — identify `CMAKE_CXX_STANDARD`, `-std=c++NN`, toolchain files, compiler wrappers, sanitizers, and warning-as-error flags. +5. **Coverage setup** — determine whether the repo already has `gcov`, `llvm-cov`, `gcovr`, `lcov`, or custom coverage targets. If not, add minimal, test-target-scoped coverage instrumentation. + +Generated C++ tests must earn coverage, not merely compile. Target uncovered functions, branches, error paths, and boundary conditions; shallow smoke tests that only construct objects rarely move line coverage. + +## Environment and Toolchain Detection + +| Indicator | Meaning | +|-----------|---------| +| `CMakeLists.txt` | CMake project; prefer configured build directories and existing presets | +| `CMakePresets.json` | Use `cmake --preset ` and `cmake --build --preset ` when present | +| `Makefile` without CMake | Use `make`, `make test`, or the repo's documented targets | +| `WORKSPACE`, `MODULE.bazel`, `BUILD(.bazel)` | Bazel project; tests are `cc_test` targets | +| `compile_commands.json` | Exact compiler flags and include directories used by the build | +| `CMAKE_CXX_STANDARD` / `-std=c++17` | Maximum language features allowed in tests | +| `clang++` | Prefer `llvm-cov gcov` for coverage data collection | +| `g++` | Use matching `gcov` from the same GCC toolchain version | + +Useful discovery commands: + +```bash +cmake --version +g++ --version +clang++ --version +find . \( -name CMakeLists.txt -o -name CMakePresets.json -o -name Makefile -o -name BUILD -o -name BUILD.bazel \) +find . \( -name '*test*.cpp' -o -name '*test*.cc' -o -name '*tests*.cpp' -o -name '*tests*.cc' \) +``` + +Do not change the project's C++ standard to make a test compile. Match the production target's standard and include directories. + +## Test Framework Detection + +| Indicator | Framework | +|-----------|-----------| +| `find_package(GTest)` / `GTest::gtest_main` / `gtest_discover_tests` | GoogleTest | +| `GTest::gmock` / `#include ` | GoogleMock for mocks | +| `find_package(Catch2 3)` / `Catch2::Catch2WithMain` / `catch_discover_tests` | Catch2 v3 | +| `#include ` | Catch2 v3 test source | +| `#include ` | GoogleTest test source | +| `add_test(NAME ... COMMAND ...)` | Manual CTest registration | +| `cc_test(` | Bazel C++ test target | + +Use the framework already present. Do not add Catch2 to a GoogleTest repo or GoogleTest to a Catch2 repo just because it is familiar. + +## Build Commands + +Prefer repo scripts and presets first. Otherwise use the smallest command that compiles the changed test target. + +| Scope | CMake command | +|-------|---------------| +| Configure debug build | `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug` | +| Configure with tests | `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON` | +| Configure with coverage option | `cmake -S . -B build-coverage -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DCODE_COVERAGE=ON` | +| Build all | `cmake --build build` | +| Build one test target | `cmake --build build --target my_component_tests` | +| Parallel build | `cmake --build build --parallel` | +| Clean rebuild | `cmake --build build --target clean && cmake --build build` | + +Make equivalents when no CMake build exists: + +| Scope | Make command | +|-------|--------------| +| Build all | `make` | +| Build tests | `make test` or `make tests` | +| Build one target | `make my_component_tests` | +| Coverage build | `make clean && CXXFLAGS="--coverage -O0 -g" LDFLAGS="--coverage" make tests` | + +Bazel equivalents: + +| Scope | Bazel command | +|-------|---------------| +| Build tests | `bazel build //...` | +| Run all tests | `bazel test //...` | +| Run one test | `bazel test //path/to:target_test` | + +## Test Commands + +| Scope | Command | +|-------|---------| +| All CTest tests | `ctest --test-dir build --output-on-failure` | +| Verbose CTest failure | `ctest --test-dir build --output-on-failure -V` | +| One CTest test by regex | `ctest --test-dir build -R my_component --output-on-failure` | +| Direct GoogleTest binary | `./build/tests/my_component_tests` | +| GoogleTest filter | `./build/tests/my_component_tests --gtest_filter=SuiteName.TestName` | +| GoogleTest list tests | `./build/tests/my_component_tests --gtest_list_tests` | +| Direct Catch2 binary | `./build/tests/my_component_tests` | +| Catch2 filter by name | `./build/tests/my_component_tests "test case name"` | +| Catch2 filter by tag | `./build/tests/my_component_tests "[parser]"` | +| Catch2 list tests | `./build/tests/my_component_tests --list-tests` | + +For CTest, run from the configured build tree or pass `--test-dir build`; running `ctest` from the source root often reports zero tests. + +## Lint Command + +Use the repo's lint script first (`make lint`, `cmake --build build --target lint`, `ninja clang-tidy`). Otherwise detect tools from configuration: + +- `.clang-format` present → `clang-format -i path/to/test.cpp` +- `.clang-tidy` present → run the repo's clang-tidy target or `clang-tidy path/to/test.cpp -- -Iinclude` +- CMake format target present → `cmake --build build --target format` + +Never silence warnings in generated tests with blanket pragmas. Fix the warning or match the existing project pattern. + +## Project Layout and Test File Placement + +Common layouts: + +```text +project/ +├── include/ # public headers +├── src/ # implementation files +├── tests/ # test sources and CMakeLists.txt +│ ├── CMakeLists.txt +│ └── widget_test.cpp +├── CMakeLists.txt +└── CMakePresets.json +``` + +| Source file | Preferred test file | +|-------------|---------------------| +| `src/widget.cpp` | `tests/widget_test.cpp` or existing repo pattern | +| `src/parser/tokenizer.cpp` | `tests/parser/tokenizer_test.cpp` | +| `include/lib/widget.hpp` | `tests/widget_test.cpp` using the public API | + +- Match existing suffixes: `_test.cpp`, `_tests.cpp`, `test_*.cpp`, or `.cc`. +- Keep tests near existing test CMake targets instead of creating an isolated harness. +- Prefer testing through public headers. Use white-box access only when existing tests already do so or coverage-critical internals cannot be reached otherwise. +- Shared fixtures/helpers belong in `tests/support/`, `tests/helpers/`, or the existing helper location, not production `src/` unless the repo already has test-only utilities. + +## GoogleTest Setup + +Minimal test source: + +```cpp +#include + +#include "calculator.hpp" + +TEST(CalculatorTest, Add_WithPositiveInputs_ReturnsSum) { + Calculator calculator; + + EXPECT_EQ(calculator.Add(2, 3), 5); +} +``` + +CMake registration with discovery: + +```cmake +enable_testing() +find_package(GTest REQUIRED) +include(GoogleTest) + +add_executable(calculator_tests + tests/calculator_test.cpp +) +target_link_libraries(calculator_tests + PRIVATE + calculator_lib + GTest::gtest_main +) +gtest_discover_tests(calculator_tests) +``` + +If the repo already has a test helper function such as `add_project_test(...)`, use it instead of writing raw `add_executable` blocks. + +## Catch2 v3 Setup + +Minimal test source: + +```cpp +#include + +#include "calculator.hpp" + +TEST_CASE("Calculator adds positive inputs", "[calculator]") { + Calculator calculator; + + CHECK(calculator.Add(2, 3) == 5); +} +``` + +CMake registration with discovery: + +```cmake +enable_testing() +find_package(Catch2 3 REQUIRED) +include(Catch) + +add_executable(calculator_tests + tests/calculator_test.cpp +) +target_link_libraries(calculator_tests + PRIVATE + calculator_lib + Catch2::Catch2WithMain +) +catch_discover_tests(calculator_tests) +``` + +Use `Catch2::Catch2WithMain` unless the repo already provides a custom `main`. Linking a framework main and defining your own `main` causes duplicate-symbol linker failures. + +## Coverage Instrumentation + +Coverage instrumentation is the highest-risk setup step. Add coverage flags to **both compilation and linking** for the test target. Adding flags only to `CXXFLAGS` often compiles but produces no `.gcda` files or fails with missing gcov runtime symbols at link time. + +### CMake target-scoped coverage option + +Prefer target-scoped flags over global `CMAKE_CXX_FLAGS` so production targets stay clean: + +```cmake +option(CODE_COVERAGE "Build tests with gcov-compatible coverage instrumentation" OFF) + +add_executable(calculator_tests + tests/calculator_test.cpp +) +target_link_libraries(calculator_tests + PRIVATE + calculator_lib + GTest::gtest_main +) + +if(CODE_COVERAGE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + target_compile_options(calculator_tests PRIVATE --coverage -O0 -g) + target_link_options(calculator_tests PRIVATE --coverage) +endif() +``` + +For libraries built into the test binary, instrument the library target too, otherwise coverage only reports test files: + +```cmake +if(CODE_COVERAGE AND CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + target_compile_options(calculator_lib PRIVATE --coverage -O0 -g) + target_link_options(calculator_lib PRIVATE --coverage) + target_compile_options(calculator_tests PRIVATE --coverage -O0 -g) + target_link_options(calculator_tests PRIVATE --coverage) +endif() +``` + +Equivalent long-form flags: + +```cmake +target_compile_options(calculator_tests PRIVATE -fprofile-arcs -ftest-coverage -O0 -g) +target_link_options(calculator_tests PRIVATE -fprofile-arcs -ftest-coverage) +``` + +### Coverage command sequence + +GCC/gcov path: + +```bash +cmake -S . -B build-coverage -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DCODE_COVERAGE=ON +cmake --build build-coverage --target calculator_tests +ctest --test-dir build-coverage --output-on-failure +gcovr --root . --filter 'src/' --filter 'include/' --exclude 'tests/' --print-summary +``` + +Clang path using gcov-compatible data: + +```bash +cmake -S . -B build-coverage -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DCODE_COVERAGE=ON +cmake --build build-coverage --target calculator_tests +ctest --test-dir build-coverage --output-on-failure +gcovr --root . --gcov-executable 'llvm-cov gcov' --filter 'src/' --filter 'include/' --exclude 'tests/' --print-summary +``` + +`lcov` / `genhtml` path: + +```bash +cmake -S . -B build-coverage -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=ON -DCODE_COVERAGE=ON +cmake --build build-coverage +ctest --test-dir build-coverage --output-on-failure +lcov --capture --directory build-coverage --output-file coverage.info +lcov --remove coverage.info '*/tests/*' '*/_deps/*' --output-file coverage.filtered.info +genhtml coverage.filtered.info --output-directory coverage-html +``` + +Make-only coverage path: + +```bash +make clean +CXXFLAGS="--coverage -O0 -g" LDFLAGS="--coverage" make tests +./tests/calculator_tests +gcovr --root . --print-summary +``` + +Important coverage rules: + +- Compile and link with the same compiler family. Clang-generated coverage data should be read with `llvm-cov gcov`, not system `gcov`. +- Run the instrumented test binary before collecting; `.gcda` files are written when the process exits normally. +- Use `-O0 -g` for coverage builds to keep line mapping stable. +- Exclude test files and vendored dependencies from coverage reports; include production `src/` and `include/`. +- If the code under test is a static library, object library, or source list linked into tests, instrument that target as well as the test executable. + +## Coverage-Targeting Guidance + +To avoid low coverage deltas: + +1. Read the coverage report and identify uncovered production files, functions, and branch lines. +2. Write tests that drive real behavior through public APIs or stable seams. +3. Prioritize branches: error handling, empty input, boundary values, invalid parse cases, feature flags, and state transitions. +4. Prefer one parameterized test that covers many meaningful paths over many tests that repeat the same happy path. +5. Assert observable outcomes, side effects, return codes, exceptions, and mock interactions. A test that only constructs an object usually adds little or no useful coverage. +6. Re-run the targeted coverage command and confirm the intended files moved. + +Do not chase coverage by testing implementation details that make the suite brittle when a public API can cover the same lines. + +## Test Patterns + +### GoogleTest assertions + +| Need | Pattern | +|------|---------| +| Non-fatal equality | `EXPECT_EQ(actual, expected)` | +| Fatal precondition | `ASSERT_NE(pointer, nullptr)` | +| Boolean | `EXPECT_TRUE(value)` / `EXPECT_FALSE(value)` | +| String equality | `EXPECT_STREQ(actual.c_str(), "expected")` | +| Floating point | `EXPECT_NEAR(actual, expected, 1e-6)` | +| Exception | `EXPECT_THROW(call(), std::invalid_argument)` | +| No exception | `EXPECT_NO_THROW(call())` | + +Use `ASSERT_*` only when the rest of the test cannot safely continue. + +### GoogleTest fixtures and parameterized tests + +```cpp +class ParserTest : public ::testing::Test { +protected: + Parser parser_; +}; + +TEST_F(ParserTest, Parse_WithEmptyInput_ReturnsEmptyResult) { + EXPECT_TRUE(parser_.Parse("").empty()); +} + +class ClampTest : public ::testing::TestWithParam> {}; + +TEST_P(ClampTest, Clamp_WithBoundaryInputs_ReturnsExpectedValue) { + const auto [value, min, max, expected] = GetParam(); + + EXPECT_EQ(Clamp(value, min, max), expected); +} + +INSTANTIATE_TEST_SUITE_P( + BoundaryCases, + ClampTest, + ::testing::Values( + std::make_tuple(-1, 0, 10, 0), + std::make_tuple(5, 0, 10, 5), + std::make_tuple(11, 0, 10, 10))); +``` + +### Catch2 assertions and generators + +```cpp +#include +#include +#include + +TEST_CASE("Clamp handles boundary inputs", "[math]") { + const auto [value, min, max, expected] = GENERATE( + std::tuple{-1, 0, 10, 0}, + std::tuple{5, 0, 10, 5}, + std::tuple{11, 0, 10, 10}); + + CHECK(Clamp(value, min, max) == expected); +} + +TEST_CASE("Divide rejects zero denominator", "[math]") { + REQUIRE_THROWS_AS(Divide(1.0, 0.0), std::invalid_argument); + CHECK(Divide(1.0, 3.0) == Catch::Approx(0.333333).epsilon(0.001)); +} +``` + +Use `REQUIRE` when execution must stop after failure; use `CHECK` for independent assertions. + +## Mocking Rules + +Use GoogleMock when the repo already uses gMock or GoogleTest with mocks: + +```cpp +#include + +class MockClock : public Clock { +public: + MOCK_METHOD(std::chrono::seconds, Now, (), (const, override)); +}; + +TEST(SchedulerTest, ShouldRun_WhenIntervalElapsed_ReturnsTrue) { + MockClock clock; + EXPECT_CALL(clock, Now()).WillOnce(::testing::Return(std::chrono::seconds{42})); + + Scheduler scheduler(clock); + + EXPECT_TRUE(scheduler.ShouldRun()); +} +``` + +Guidelines: + +- Mock interfaces with virtual methods and virtual destructors. +- Prefer small interfaces or constructor injection over global state. +- For code without virtuals, create seams with templates, function objects, adapters, or thin interfaces around external dependencies. +- Do not mock standard library containers or value objects; build real values. +- If a test needs more than three mocks, treat it as a design smell and look for a higher-level behavioral test. + +## Testing Internals + +If types are not well suited for testing only through their public surface, consider exposing internals to tests using a preprocessor-guarded `friend` declaration: + +```cpp +class MyClass { +#ifdef UNIT_TESTING + friend class MyClassTest; +#endif + // ... +}; +``` + +Define `UNIT_TESTING` only in the test build configuration so production builds remain unaffected: + +```cmake +target_compile_definitions(my_component_tests PRIVATE UNIT_TESTING) +``` + +Use this sparingly. Prefer public behavior tests and dependency seams before adding test-only friendship. + +## Common Errors + +| Error | Fix | +|-------|-----| +| `undefined reference to __gcov_init` / `__gcov_exit` | Add `--coverage` or `-fprofile-arcs -ftest-coverage` to link flags, not only compile flags | +| No `.gcda` files produced | Ensure the instrumented binary ran to normal exit, the production target was instrumented, and the build directory is writable | +| `profiling: ... cannot merge previous GCDA file` | Delete old coverage files or rebuild clean after changing compiler/options | +| `gcov: stamp mismatch` | Clean the build directory; `.gcno` and `.gcda` came from different builds | +| Clang coverage unreadable by `gcov` | Run `gcovr --gcov-executable 'llvm-cov gcov'` | +| `multiple definition of main` | Link `GTest::gtest_main` or `Catch2::Catch2WithMain`, or provide your own main, not both | +| CTest reports `No tests were found!!!` | Add `enable_testing()` and `gtest_discover_tests`, `catch_discover_tests`, or `add_test`; run `ctest --test-dir build` | +| `fatal error: gtest/gtest.h: No such file or directory` | Use the repo's dependency mechanism or add `find_package(GTest REQUIRED)` / FetchContent only as last resort | +| `undefined reference` to production symbols | Link the test target to the library under test with `target_link_libraries(test PRIVATE my_lib)` | +| ABI or standard mismatch | Match `CMAKE_CXX_STANDARD`, compiler, runtime, and flags between production and test targets | +| `EXPECT_EQ` prints unreadable custom types | Add `operator==` and, if useful, `operator<<` in the test namespace or production type namespace | +| Test passes directly but not under CTest | Check working directory assumptions; set `WORKING_DIRECTORY` in `add_test` or use paths relative to test data | +| Segfault in test cleanup | Avoid owning raw pointers in tests; use RAII objects and make mock lifetimes outlive the object under test | + +## Dependency Installation (Last Resort) + +Only add dependencies after investigation confirms the repo has no test framework or the expected framework is missing. Prefer existing package managers and lockfiles. + +CMake with installed packages: + +```cmake +find_package(GTest REQUIRED) +find_package(Catch2 3 REQUIRED) +``` + +CMake FetchContent fallback for GoogleTest: + +```cmake +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip) +FetchContent_MakeAvailable(googletest) +``` + +CMake FetchContent fallback for Catch2 v3: + +```cmake +include(FetchContent) +FetchContent_Declare( + Catch2 + URL https://github.com/catchorg/Catch2/archive/refs/tags/v3.5.4.zip) +FetchContent_MakeAvailable(Catch2) +list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras) +``` + +Package manager examples: + +| Manager | Command | +|---------|---------| +| vcpkg | `vcpkg install gtest catch2` | +| Conan | `conan install . --build=missing` | +| apt | `sudo apt-get install libgtest-dev catch2 gcovr lcov` | +| Homebrew | `brew install googletest catch2 gcovr lcov` | + +On Debian/Ubuntu, `libgtest-dev` historically installs only GoogleTest *sources* (no prebuilt libraries or CMake package config), so `find_package(GTest REQUIRED)` can still fail. On those systems either build/install GoogleTest from the packaged sources, add it via `FetchContent`, or use vcpkg/Conan instead of relying on apt alone. + +Do not vendor dependencies by copying source into the repo unless that is already the project's dependency policy. + +## Skip + +Skip or avoid these actions unless the repo explicitly requires them: + +- Do not replace the build system or create a parallel test harness outside CMake/Make/Bazel just for generated tests. +- Do not add coverage flags globally to release builds; keep coverage in a Debug/test-only configuration. +- Do not use system `gcov` with Clang-generated coverage data. +- Do not define a second test `main` when linking framework-provided main targets. +- Do not write tests that only instantiate objects without assertions or behavior coverage. +- Do not add a new test framework when an existing one is already configured. diff --git a/.github/skills/code-testing-extensions/extensions/dotnet-examples.md b/.github/skills/code-testing-extensions/extensions/dotnet-examples.md new file mode 100644 index 0000000..4eed0a9 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/dotnet-examples.md @@ -0,0 +1,370 @@ +# .NET Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a .NET/C# codebase. These show what each pipeline phase produces for a small project. + +## Source Under Test + +A simple `InvoiceService` in a .NET 9 project using MSTest: + +```text +src/ + Contoso.Billing/ + Contoso.Billing.csproj + InvoiceService.cs + Invoice.cs + IInvoiceRepository.cs +tests/ + Contoso.Billing.Tests/ + Contoso.Billing.Tests.csproj (exists, references Contoso.Billing) +Contoso.Billing.sln +``` + +```csharp +// InvoiceService.cs +namespace Contoso.Billing; + +public class InvoiceService(IInvoiceRepository repository) +{ + public decimal CalculateTotal(Invoice invoice) + { + if (invoice is null) throw new ArgumentNullException(nameof(invoice)); + if (invoice.LineItems.Count == 0) throw new InvalidOperationException("Invoice has no line items."); + + var subtotal = invoice.LineItems.Sum(li => li.Quantity * li.UnitPrice); + var tax = subtotal * invoice.TaxRate; + return Math.Round(subtotal + tax, 2); + } + + public async Task GetByIdAsync(int id) + { + var invoice = await repository.FindAsync(id); + return invoice ?? throw new KeyNotFoundException($"Invoice {id} not found."); + } + + public async Task MarkAsPaidAsync(int id) + { + var invoice = await repository.FindAsync(id) + ?? throw new KeyNotFoundException($"Invoice {id} not found."); + if (invoice.Status == InvoiceStatus.Paid) + throw new InvalidOperationException("Invoice is already paid."); + invoice.Status = InvoiceStatus.Paid; + invoice.PaidDate = DateTime.UtcNow; + await repository.UpdateAsync(invoice); + } +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: C:\src\Contoso.Billing +- **Language**: C# (.NET 9) +- **Framework**: .NET 9 (net9.0) +- **Test Framework**: MSTest 3.8 + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Build**: `dotnet build Contoso.Billing.sln` +- **Test**: `dotnet test Contoso.Billing.sln` +- **Lint**: `dotnet format Contoso.Billing.sln` + +## Project Structure +- Source: `src/Contoso.Billing/` +- Tests: `tests/Contoso.Billing.Tests/` (exists, empty) + +## Files to Test + +### High Priority +| File | Classes/Functions | Testability | Notes | +|------|-------------------|-------------|-------| +| src/Contoso.Billing/InvoiceService.cs | InvoiceService: CalculateTotal, GetByIdAsync, MarkAsPaidAsync | High | Core business logic, repository dependency needs mocking | + +### Low Priority / Skip +| File | Reason | +|------|--------| +| src/Contoso.Billing/Invoice.cs | Data model, no logic | +| src/Contoso.Billing/IInvoiceRepository.cs | Interface, no implementation | + +## Existing Tests +- No existing tests found + +## Existing Test Projects +- **Project file**: `tests/Contoso.Billing.Tests/Contoso.Billing.Tests.csproj` +- **Target source project**: `src/Contoso.Billing/Contoso.Billing.csproj` +- **Test files**: none + +## Testing Patterns +- No existing patterns; recommend sealed test classes, AAA structure, `Moq` for mocking IInvoiceRepository + +## Recommendations +- Start with InvoiceService.CalculateTotal (pure logic, easy to test) +- Then async methods (require mocking IInvoiceRepository) +``` + +## Sample Plan Output + +What `code-testing-planner` produces in `.testagent/plan.md`: + +```markdown +# Test Implementation Plan + +## Overview +Generate MSTest tests for the Contoso.Billing InvoiceService, covering all three +public methods across happy path, edge case, and error scenarios. Single phase +since there is only one source file. + +## Commands +- **Build**: `dotnet build tests/Contoso.Billing.Tests/Contoso.Billing.Tests.csproj` +- **Test**: `dotnet test tests/Contoso.Billing.Tests/Contoso.Billing.Tests.csproj` +- **Lint**: `dotnet format --include tests/Contoso.Billing.Tests/` + +## Phase Summary +| Phase | Focus | Files | Est. Tests | +|-------|-------|-------|------------| +| 1 | InvoiceService | 1 | 9-12 | + +--- + +## Phase 1: InvoiceService + +### Overview +Cover all public methods of InvoiceService. CalculateTotal is pure logic tested +with DataRow. Async methods require a mocked IInvoiceRepository. + +### Files to Test + +#### 1. InvoiceService.cs +- **Source**: `src/Contoso.Billing/InvoiceService.cs` +- **Test File**: `tests/Contoso.Billing.Tests/InvoiceServiceTests.cs` +- **Test Class**: `InvoiceServiceTests` + +**Methods to Test**: +1. `CalculateTotal` — Pure calculation logic + - Happy path: single line item returns quantity × price + tax + - Happy path: multiple line items summed correctly + - Edge case: zero tax rate returns subtotal only + - Error case: null invoice throws ArgumentNullException + - Error case: empty line items throws InvalidOperationException + +2. `GetByIdAsync` — Repository lookup + - Happy path: existing ID returns invoice + - Error case: non-existent ID throws KeyNotFoundException + +3. `MarkAsPaidAsync` — State transition + - Happy path: unpaid invoice transitions to Paid with PaidDate set + - Error case: already paid throws InvalidOperationException + - Error case: non-existent ID throws KeyNotFoundException + +### Success Criteria +- [ ] All test files created +- [ ] Tests compile with `dotnet build` +- [ ] All tests pass with `dotnet test` +``` + +## Sample Generated Test File + +What `code-testing-implementer` produces: + +```csharp +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Contoso.Billing; + +namespace Contoso.Billing.Tests; + +[TestClass] +public sealed class InvoiceServiceTests +{ + private readonly Mock _repositoryMock = new(); + private readonly InvoiceService _sut; + + public InvoiceServiceTests() + { + _sut = new InvoiceService(_repositoryMock.Object); + } + + // --- CalculateTotal --- + + [TestMethod] + [DataRow(1, 100.00, 0.10, 110.00, DisplayName = "Single item with 10% tax")] + [DataRow(3, 25.00, 0.0, 75.00, DisplayName = "Multiple quantity, zero tax")] + public void CalculateTotal_ValidLineItems_ReturnsExpectedTotal( + int quantity, double unitPrice, double taxRate, double expected) + { + // Arrange + var invoice = new Invoice + { + TaxRate = (decimal)taxRate, + LineItems = [new LineItem { Quantity = quantity, UnitPrice = (decimal)unitPrice }] + }; + + // Act + var total = _sut.CalculateTotal(invoice); + + // Assert + Assert.AreEqual((decimal)expected, total); + } + + [TestMethod] + public void CalculateTotal_NullInvoice_ThrowsArgumentNullException() + { + Assert.ThrowsExactly(() => _sut.CalculateTotal(null!)); + } + + [TestMethod] + public void CalculateTotal_EmptyLineItems_ThrowsInvalidOperationException() + { + // Arrange + var invoice = new Invoice { LineItems = [] }; + + // Act & Assert + Assert.ThrowsExactly(() => _sut.CalculateTotal(invoice)); + } + + // --- GetByIdAsync --- + + [TestMethod] + public async Task GetByIdAsync_ExistingId_ReturnsInvoice() + { + // Arrange + var expected = new Invoice { Id = 42 }; + _repositoryMock.Setup(r => r.FindAsync(42)).ReturnsAsync(expected); + + // Act + var result = await _sut.GetByIdAsync(42); + + // Assert + Assert.AreSame(expected, result); + } + + [TestMethod] + public async Task GetByIdAsync_NonExistentId_ThrowsKeyNotFoundException() + { + // Arrange + _repositoryMock.Setup(r => r.FindAsync(999)).ReturnsAsync((Invoice?)null); + + // Act & Assert + await Assert.ThrowsExactlyAsync( + () => _sut.GetByIdAsync(999)); + } + + // --- MarkAsPaidAsync --- + + [TestMethod] + public async Task MarkAsPaidAsync_UnpaidInvoice_SetsStatusAndDate() + { + // Arrange + var invoice = new Invoice { Id = 1, Status = InvoiceStatus.Pending }; + _repositoryMock.Setup(r => r.FindAsync(1)).ReturnsAsync(invoice); + + // Act + await _sut.MarkAsPaidAsync(1); + + // Assert + Assert.AreEqual(InvoiceStatus.Paid, invoice.Status); + Assert.IsNotNull(invoice.PaidDate); + _repositoryMock.Verify(r => r.UpdateAsync(invoice), Times.Once); + } + + [TestMethod] + public async Task MarkAsPaidAsync_AlreadyPaid_ThrowsInvalidOperationException() + { + // Arrange + var invoice = new Invoice { Id = 1, Status = InvoiceStatus.Paid }; + _repositoryMock.Setup(r => r.FindAsync(1)).ReturnsAsync(invoice); + + // Act & Assert + await Assert.ThrowsExactlyAsync( + () => _sut.MarkAsPaidAsync(1)); + } +} +``` + +## Sample Fix Cycle + +When the implementer encounters a build error, the fixer agent diagnoses and resolves it: + +**Build output:** + +```text +error CS0246: The type or namespace name 'Moq' could not be found (are you missing a using directive or an assembly reference?) +``` + +**Fixer diagnosis:** The test project is missing the Moq NuGet package. + +**Fix applied:** + +```bash +dotnet add tests/Contoso.Billing.Tests/Contoso.Billing.Tests.csproj package Moq +``` + +**Rebuild:** `dotnet build tests/Contoso.Billing.Tests/Contoso.Billing.Tests.csproj` → SUCCESS + +--- + +**Another common cycle:** + +**Build output:** + +```text +error CS7036: There is no argument given that corresponds to the required parameter 'repository' of 'InvoiceService.InvoiceService(IInvoiceRepository)' +``` + +**Fixer diagnosis:** Test code instantiated `new InvoiceService()` without passing the required constructor parameter. The source uses a primary constructor with an `IInvoiceRepository` dependency. + +**Fix applied:** + +```csharp +// Before (wrong) +var sut = new InvoiceService(); + +// After (fixed) +var repositoryMock = new Mock(); +var sut = new InvoiceService(repositoryMock.Object); +``` + +**Rebuild:** SUCCESS + +## Sample Final Report + +What `code-testing-generator` produces at Step 9: + +```markdown +## Test Generation Report + +**Project**: Contoso.Billing +**Strategy**: Single pass + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 9 | +| Tests passing | 9 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `tests/Contoso.Billing.Tests/InvoiceServiceTests.cs` (9 tests) + +### Coverage +- InvoiceService.CalculateTotal — 3 happy path, 2 error cases +- InvoiceService.GetByIdAsync — 1 happy path, 1 error case +- InvoiceService.MarkAsPaidAsync — 1 happy path, 1 error case + +### Build Validation +- Scoped build: ✅ passed +- Full solution build (`dotnet build --no-incremental`): ✅ passed + +### Next Steps +- Add integration tests for repository layer if needed +- Consider testing with multiple line items for CalculateTotal +``` diff --git a/.github/skills/code-testing-extensions/extensions/dotnet.md b/.github/skills/code-testing-extensions/extensions/dotnet.md new file mode 100644 index 0000000..9fd13c5 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/dotnet.md @@ -0,0 +1,138 @@ +# .NET Extension + +Language-specific guidance for .NET (C#/F#/VB) test generation. + +## Build Commands + +| Scope | Command | +|-------|---------| +| Specific test project | `dotnet build MyProject.Tests.csproj` | +| Full solution (final validation) | `dotnet build MySolution.sln --no-incremental` | +| From repo root (no .sln) | `dotnet build --no-incremental` | + +- Use `--no-restore` if dependencies are already restored +- Use `-v:q` (quiet) to reduce output noise +- Always use `--no-incremental` for the final validation build — incremental builds hide errors like CS7036 + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests | `dotnet test` | +| Filtered | `dotnet test --filter "FullyQualifiedName~ClassName"` | +| After build | `dotnet test --no-build` | + +- Use `--no-build` if already built +- Use `-v:q` for quieter output + +## Lint Command + +```bash +dotnet format --include path/to/file.cs +dotnet format MySolution.sln # full solution +``` + +## Project Reference Validation + +Before writing test code, read the test project's `.csproj` to verify it has `` entries for the assemblies your tests will use. If a reference is missing, add it: + +```xml + + + +``` + +This prevents CS0234 ("namespace not found") and CS0246 ("type not found") errors. + +## Common CS Error Codes + +| Error | Meaning | Fix | +|-------|---------|-----| +| CS0234 | Namespace not found | Add `` to the source project in the test `.csproj` | +| CS0246 | Type not found | Add `using Namespace;` or add missing `` | +| CS0103 | Name not found | Check spelling, add `using` statement | +| CS1061 | Missing member | Verify method/property name matches the source code exactly | +| CS0029 | Type mismatch | Cast or change the type to match the expected signature | +| CS7036 | Missing required parameter | Read the constructor/method signature and pass all required arguments | + +## `.csproj` / `.sln` Handling + +- During phase implementation, build only the specific test `.csproj` for speed +- For the final validation, build the full `.sln` with `--no-incremental` +- Full-solution builds catch cross-project reference errors invisible in scoped builds + +### Registering a new test project (MANDATORY when `dotnet new` was used) + +A new `.csproj` is **invisible** to `dotnet test `, to `dotnet test` run from the repo root, and to any CI/benchmark harness until it is added to the solution. Run `dotnet sln add` *immediately* after creating the project as part of Step 3 ("Register Test Project with Build System") — do not defer it to a later step. + +1. Use the exact solution or solution-filter target identified in `.testagent/research.md` or `.testagent/plan.md` — do not search for or substitute a different `.sln`, `.slnx`, or `.slnf` target. +2. If that target is a `.sln` or `.slnx`, run `dotnet sln add `. +3. If the target is a `.slnf` (solution filter), also ensure the new project is included in the filter; adding only to the underlying `.sln` may not be enough for test discovery. +4. Skip this if the project is already included in the solution or solution filter used for testing. +5. Prefer the researched test command. If you need to run the solution directly, use `dotnet test --solution ` only for repos on .NET SDK 10+ with MTP-style syntax; otherwise use the standard positional form `dotnet test `. + +### Harness Discovery Check + +Before reporting success, run the **harness-equivalent** discovery command from the repo root and confirm the test count went up by at least the number of tests you generated. The harness (CI, msbench, coverage tools) does not know which `.csproj` you targeted — it runs the solution-level command, so a test that passes via `dotnet test MyProject.Tests.csproj` is still worthless if `dotnet test --list-tests` doesn't enumerate it. + +```bash +# From repo root, against the solution identified in .testagent/research.md +dotnet test --list-tests --no-build 2>&1 | grep -c '^ [A-Za-z]' +``` + +If the delta is `0`, the new project isn't in the solution. Run `dotnet sln add ` and re-run the check. Do **not** report success until the harness command sees your new tests. + +## Test Framework Detection + +Detect the framework from the test project's `.csproj` package references and match its conventions: + +| Package Reference | Framework | Attributes | Assertion Style | +|-------------------|-----------|------------|-----------------| +| `MSTest.Sdk` or `MSTest.TestFramework` | MSTest | `[TestClass]`, `[TestMethod]`, `[DataRow]` | `Assert.AreEqual(expected, actual)` | +| `xunit` | xUnit | `[Fact]`, `[Theory]`, `[InlineData]` | `Assert.Equal(expected, actual)` | +| `NUnit` | NUnit | `[TestFixture]`, `[Test]`, `[TestCase]` | `Assert.That(actual, Is.EqualTo(expected))` | + +Use the repo's existing framework — do not introduce a different one. + +## MSTest Template + +```csharp +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace ProjectName.Tests; + +[TestClass] +public sealed class ClassNameTests +{ + [TestMethod] + public void MethodName_Scenario_ExpectedResult() + { + // Arrange + var sut = new ClassName(); + + // Act + var result = sut.MethodName(input); + + // Assert + Assert.AreEqual(expected, result); + } + + [TestMethod] + [DataRow(2, 3, 5, DisplayName = "Positive numbers")] + [DataRow(-1, 1, 0, DisplayName = "Negative and positive")] + public void Add_ValidInputs_ReturnsSum(int a, int b, int expected) + { + // Act + var result = _sut.Add(a, b); + + // Assert + Assert.AreEqual(expected, result); + } +} +``` + +## Skip Coverage Tools + +Do not configure or run code coverage measurement tools (coverlet, dotnet-coverage, XPlat Code Coverage) by default. These tools have inconsistent cross-configuration behavior and waste significant time. Coverage is measured separately by the evaluation harness. + +**Exception**: if the user or evaluation harness explicitly requires a Cobertura/XML coverage artifact (e.g., they ask for `coverlet.collector` or a `--collect:"XPlat Code Coverage"` run), add the `coverlet.collector` PackageReference to the generated .NET test csproj so the harness's coverage command can produce output. Do not run the coverage command yourself; leave that to the validation step. diff --git a/.github/skills/code-testing-extensions/extensions/go-examples.md b/.github/skills/code-testing-extensions/extensions/go-examples.md new file mode 100644 index 0000000..236f3b8 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/go-examples.md @@ -0,0 +1,396 @@ +# Go Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Go codebase. These show what each pipeline phase produces for a small package. + +## Source Under Test + +A simple `InvoiceService` in a Go module: + +```text +go.mod (module github.com/contoso/billing) +internal/billing/ + invoice.go + invoice_repository.go (defines the InvoiceRepository interface) + invoice_service.go +``` + +```go +// internal/billing/invoice_service.go +package billing + +import ( + "context" + "errors" + "fmt" + "math" + "time" +) + +type InvoiceService struct { + repository InvoiceRepository + now func() time.Time +} + +func NewInvoiceService(repo InvoiceRepository) *InvoiceService { + return &InvoiceService{repository: repo, now: time.Now} +} + +func (s *InvoiceService) CalculateTotal(invoice *Invoice) (float64, error) { + if invoice == nil { + return 0, errors.New("invoice must not be nil") + } + if len(invoice.LineItems) == 0 { + return 0, errors.New("invoice has no line items") + } + var subtotal float64 + for _, li := range invoice.LineItems { + subtotal += float64(li.Quantity) * li.UnitPrice + } + tax := subtotal * invoice.TaxRate + return math.Round((subtotal+tax)*100) / 100, nil +} + +func (s *InvoiceService) GetByID(ctx context.Context, id int) (*Invoice, error) { + invoice, err := s.repository.Find(ctx, id) + if err != nil { + return nil, err + } + if invoice == nil { + return nil, fmt.Errorf("invoice %d not found", id) + } + return invoice, nil +} + +func (s *InvoiceService) MarkAsPaid(ctx context.Context, id int) error { + invoice, err := s.repository.Find(ctx, id) + if err != nil { + return err + } + if invoice == nil { + return fmt.Errorf("invoice %d not found", id) + } + if invoice.Status == StatusPaid { + return errors.New("invoice is already paid") + } + invoice.Status = StatusPaid + invoice.PaidDate = s.now() + return s.repository.Update(ctx, invoice) +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/billing +- **Language**: Go 1.22 (from go.mod) +- **Module**: github.com/contoso/billing +- **Test Framework**: standard `testing` package (no testify/gomock detected in go.sum) + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Vet**: `go vet ./...` +- **Build**: `go build ./...` +- **Compile tests**: `go test -count=1 -run=^$ ./internal/billing` +- **Test**: `go test -count=1 ./internal/billing` + +## Project Structure +- Source: `internal/billing/` +- Tests: none + +## Files to Test + +### High Priority +| File | Functions | Testability | Notes | +|------|-----------|-------------|-------| +| internal/billing/invoice_service.go | InvoiceService.CalculateTotal, GetByID, MarkAsPaid | High | Uses InvoiceRepository interface — easy to fake with a hand-written struct | + +## Existing Tests +- No existing tests found + +## Testing Patterns +- No existing patterns; recommend white-box `package billing` tests with hand-written fake repository (no testify since the repo doesn't use it), table-driven `t.Run` subtests for CalculateTotal, and an injected `now func() time.Time` for MarkAsPaid. + +## Recommendations +- Inject `now` instead of stubbing `time.Now` globally — the struct already supports it +- Place tests in `internal/billing/invoice_service_test.go` (same package, white-box) +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate standard-library Go tests for InvoiceService using table-driven subtests +and a hand-written fake repository. Single phase since there is only one source file. + +## Commands +- **Compile tests**: `go test -count=1 -run=^$ ./internal/billing` +- **Test**: `go test -count=1 -v ./internal/billing` + +## Phase 1: InvoiceService + +### Files to Test + +#### 1. invoice_service.go +- **Source**: `internal/billing/invoice_service.go` +- **Test File**: `internal/billing/invoice_service_test.go` + +**Functions to Test**: +1. `CalculateTotal` — Table-driven + - Happy paths: single item, multi-item, rounding + - Error cases: nil invoice, empty line items +2. `GetByID` — happy + missing + repo error +3. `MarkAsPaid` — happy (verifies timestamp via injected clock) + already-paid + missing + repo error +``` + +## Sample Generated Test File + +```go +// internal/billing/invoice_service_test.go +package billing + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +type fakeRepository struct { + findFunc func(ctx context.Context, id int) (*Invoice, error) + updateFunc func(ctx context.Context, invoice *Invoice) error + updated *Invoice +} + +func (f *fakeRepository) Find(ctx context.Context, id int) (*Invoice, error) { + if f.findFunc != nil { + return f.findFunc(ctx, id) + } + return nil, nil +} + +func (f *fakeRepository) Update(ctx context.Context, invoice *Invoice) error { + f.updated = invoice + if f.updateFunc != nil { + return f.updateFunc(ctx, invoice) + } + return nil +} + +func TestInvoiceService_CalculateTotal(t *testing.T) { + tests := []struct { + name string + invoice *Invoice + want float64 + wantErr string + }{ + { + name: "single item with 10% tax", + invoice: &Invoice{TaxRate: 0.10, LineItems: []LineItem{{Quantity: 1, UnitPrice: 100}}}, + want: 110, + }, + { + name: "multi quantity zero tax", + invoice: &Invoice{TaxRate: 0, LineItems: []LineItem{{Quantity: 3, UnitPrice: 25}}}, + want: 75, + }, + { + name: "rounds half up", + invoice: &Invoice{TaxRate: 0.07, LineItems: []LineItem{{Quantity: 2, UnitPrice: 9.99}}}, + want: 21.38, + }, + { + name: "nil invoice errors", + invoice: nil, + wantErr: "invoice must not be nil", + }, + { + name: "empty line items errors", + invoice: &Invoice{TaxRate: 0, LineItems: []LineItem{}}, + wantErr: "no line items", + }, + } + sut := NewInvoiceService(&fakeRepository{}) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := sut.CalculateTotal(tt.invoice) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("CalculateTotal = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInvoiceService_GetByID(t *testing.T) { + ctx := context.Background() + want := &Invoice{ID: 42} + + t.Run("returns invoice when found", func(t *testing.T) { + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return want, nil }} + sut := NewInvoiceService(repo) + got, err := sut.GetByID(ctx, 42) + if err != nil || got != want { + t.Fatalf("got (%v, %v), want (%v, nil)", got, err, want) + } + }) + + t.Run("returns not-found error when missing", func(t *testing.T) { + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return nil, nil }} + sut := NewInvoiceService(repo) + _, err := sut.GetByID(ctx, 999) + if err == nil || !strings.Contains(err.Error(), "999") { + t.Fatalf("expected error mentioning 999, got %v", err) + } + }) + + t.Run("propagates repository error", func(t *testing.T) { + boom := errors.New("boom") + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return nil, boom }} + sut := NewInvoiceService(repo) + _, err := sut.GetByID(ctx, 1) + if !errors.Is(err, boom) { + t.Fatalf("expected boom error, got %v", err) + } + }) +} + +func TestInvoiceService_MarkAsPaid(t *testing.T) { + ctx := context.Background() + fixedTime := time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC) + + t.Run("transitions pending invoice to paid", func(t *testing.T) { + invoice := &Invoice{ID: 1, Status: StatusPending} + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return invoice, nil }} + sut := NewInvoiceService(repo) + sut.now = func() time.Time { return fixedTime } + + if err := sut.MarkAsPaid(ctx, 1); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if invoice.Status != StatusPaid { + t.Errorf("status = %v, want %v", invoice.Status, StatusPaid) + } + if !invoice.PaidDate.Equal(fixedTime) { + t.Errorf("paid date = %v, want %v", invoice.PaidDate, fixedTime) + } + if repo.updated != invoice { + t.Errorf("repository was not updated with the invoice") + } + }) + + t.Run("rejects already-paid invoice", func(t *testing.T) { + invoice := &Invoice{ID: 1, Status: StatusPaid} + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return invoice, nil }} + sut := NewInvoiceService(repo) + if err := sut.MarkAsPaid(ctx, 1); err == nil || !strings.Contains(err.Error(), "already paid") { + t.Fatalf("expected already-paid error, got %v", err) + } + if repo.updated != nil { + t.Errorf("update should not be called for already-paid invoice") + } + }) + + t.Run("returns not-found when missing", func(t *testing.T) { + repo := &fakeRepository{findFunc: func(_ context.Context, _ int) (*Invoice, error) { return nil, nil }} + sut := NewInvoiceService(repo) + if err := sut.MarkAsPaid(ctx, 999); err == nil || !strings.Contains(err.Error(), "999") { + t.Fatalf("expected not-found error, got %v", err) + } + }) +} +``` + +## Sample Fix Cycle + +When the implementer hits a compile or test-runner issue, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +internal/billing/invoice_service_test.go:14:6: cannot use &fakeRepository{} (value of type *fakeRepository) as type InvoiceRepository in argument to NewInvoiceService: + *fakeRepository does not implement InvoiceRepository (missing method Update) +``` + +**Fixer diagnosis:** The fake repository only implemented `Find`. Go enforces full interface implementation at compile time. Add the missing method. + +**Fix applied:** Add the `Update` method to `fakeRepository` (shown in the test file above). + +**Rebuild + rerun:** `go test -count=1 ./internal/billing` → SUCCESS + +--- + +**Another common cycle — wrong test selection regex:** + +**Test output:** + +```text +testing: warning: no tests to run +``` + +**Fixer diagnosis:** The agent used `go test -run TestInvoiceService_CalculateTotal/single_item` without `^...$` anchors. The Go test runner treats `-run` as a regex; the underscore makes the match too narrow. + +**Fix applied:** + +```bash +# Before — bare name without anchors, and an unquoted space would be parsed +# by the shell as two separate arguments +go test -run 'TestInvoiceService_CalculateTotal/single_item' + +# After — anchor the subtest name, replace spaces with underscores +go test -run '^TestInvoiceService_CalculateTotal$/^single_item_with_10%_tax$' ./internal/billing +``` + +**Rerun:** SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: billing (Go) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 11 | +| Tests passing | 11 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `internal/billing/invoice_service_test.go` (3 top-level tests, 11 subtests including 5 table cases) + +### Coverage +- InvoiceService.CalculateTotal — 3 happy + 2 error cases (table-driven) +- InvoiceService.GetByID — happy + missing + repo-error +- InvoiceService.MarkAsPaid — happy (with fixed clock) + already-paid + missing + +### Build / Test Validation +- `go vet ./...`: ✅ +- `go test -count=1 ./internal/billing`: ✅ PASS + +### Next Steps +- Add fuzz test (`FuzzCalculateTotal`) if rounding correctness is critical +- Consider extracting a `Clock` interface if more time-dependent logic appears +``` diff --git a/.github/skills/code-testing-extensions/extensions/go.md b/.github/skills/code-testing-extensions/extensions/go.md new file mode 100644 index 0000000..f084767 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/go.md @@ -0,0 +1,158 @@ +# Go Extension + +Language-specific guidance for Go test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `*_test.go` files and copy their style (table-driven layout, helper usage, assertion library, build tags) +2. **`go.mod` / `go.sum`** — module path, Go version, dependencies (e.g. `testify`, `gomock`, `mockery`) +3. **Build/CI scripts** — `Makefile`, `magefile.go`, `Taskfile.yml`, `.github/workflows/*.yml` +4. **`go.work`** — if present, you are in a workspace; tests for a module must run from that module's directory or use `-C` (Go 1.20+) + +Use whatever assertion style and test layout the repo already uses. Do not introduce `testify` if the repo uses the standard library only. + +## Toolchain Detection + +| Indicator | Meaning | +|-----------|---------| +| `go.mod` `go 1.x` directive | Minimum Go version — match it locally with `go version` | +| `go.work` at the root | Multi-module workspace; commands resolve dependent modules from sibling directories | +| `vendor/` directory | Vendored deps; many commands implicitly add `-mod=vendor` | +| `tools.go` with `//go:build tools` | Tool versions pinned in `go.mod` (e.g. `mockgen`); install with `go install` from the listed paths | + +## Build Commands + +| Scope | Command | +|-------|---------| +| Compile a package | `go build ./path/to/pkg` | +| Vet (static analysis) | `go vet ./...` | +| Compile tests without running | `go test -count=1 -run=^$ ./path/to/pkg` | +| Whole module | `go build ./...` | + +`go build ./...` is the closest thing to a "does it compile" gate. It does not exercise test files — use `go test -run=^$` to type-check tests as well. + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests in a package | `go test ./path/to/pkg` | +| All tests in module | `go test ./...` | +| Single test | `go test -run '^TestName$' ./path/to/pkg` | +| Subtest | `go test -run '^TestName$/^subname$' ./path/to/pkg` | +| Verbose | `go test -v ./path/to/pkg` | +| Race detector | `go test -race ./...` | +| Disable cache | `go test -count=1 ./...` | +| Short mode | `go test -short ./...` | + +- `-run` arguments are **regular expressions anchored** with `^...$`; without anchors the pattern matches as a substring +- `go test -count=1` is the canonical way to bypass the test result cache; never use a fake `-count=2` or environment hacks +- `-race` significantly slows tests and requires CGO — only enable if the repo's CI does + +## Lint Command + +Use the repo's lint script first (`make lint`, `task lint`). Otherwise detect from `.golangci.yml`/`.golangci.yaml`: + +- `.golangci.yml` present → `golangci-lint run ./...` +- No config → `gofmt -w .` and `go vet ./...` +- `goimports` config / pre-commit hook → `goimports -w path/to/file.go` + +Never disable existing linters in the test files you generate. + +## Project Layout and Imports + +Go uses package paths derived from the module path in `go.mod`. + +| Scenario | Test placement | Package declaration | +|----------|----------------|----------------------| +| Internal-only test (white-box) | `foo_test.go` next to `foo.go` | `package foo` (same as production) | +| External-only test (black-box) | `foo_test.go` next to `foo.go` | `package foo_test` (forces use of public API) | +| Integration / build-tag gated | `foo_integration_test.go` | Add `//go:build integration` at top | + +- Test files **must** end with `_test.go` — the toolchain ignores other names +- A package directory may contain both `package foo` and `package foo_test` test files simultaneously +- Helpers shared across tests in one package go in `helpers_test.go` — do not export them; put them in the `_test` package only if integration tests in another package need them +- Imports use the full module path: `import "github.com/org/module/pkg"` — copy the exact module path from `go.mod` + +## Test Function Signatures + +| Kind | Signature | +|------|-----------| +| Standard test | `func TestThing(t *testing.T)` | +| Subtests | `t.Run("name", func(t *testing.T) { ... })` | +| Benchmark | `func BenchmarkThing(b *testing.B)` | +| Example (godoc) | `func ExampleThing()` with `// Output:` comment | +| Fuzz (Go 1.18+) | `func FuzzThing(f *testing.F)` | +| Per-package setup | `func TestMain(m *testing.M)` — call `m.Run()` and `os.Exit` with its code | + +Use **table-driven tests** when generating multiple cases for the same behavior — this is idiomatic Go and matches what most repos already use: + +```go +func TestAdd(t *testing.T) { + tests := []struct { + name string + a, b int + want int + }{ + {"positives", 2, 3, 5}, + {"negatives", -1, -1, -2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Add(tt.a, tt.b); got != tt.want { + t.Errorf("Add(%d,%d) = %d, want %d", tt.a, tt.b, got, tt.want) + } + }) + } +} +``` + +When iterating with `t.Run` over a loop variable on Go < 1.22, capture it with `tt := tt` to avoid closure-over-loop-variable bugs. + +## Common Errors + +| Error | Fix | +|-------|-----| +| `package X is not in std` / `cannot find module providing package X` | Add the import to `go.mod`: `go get path/to/module@version`, then `go mod tidy` | +| `import cycle not allowed in test` | Move shared helpers to a separate package, or switch to a `_test` package for black-box tests | +| `undefined: X` in `_test` package | The symbol is unexported; either use `package foo` (white-box) or export it intentionally | +| `t.Parallel called multiple times` | Each subtest can call `t.Parallel()` once; do not call it twice in the same test | +| `panic: test executed panic(nil) or runtime.Goexit` | A goroutine called `t.Fatal` outside the test goroutine; only the main test goroutine may call `Fatal`/`FailNow` | +| `flag provided but not defined: -X` | Flags registered in `init()` of test files must use `flag.NewFlagSet` carefully; place test-only flags in `TestMain` | +| `go: cannot find main module` | Run inside the module directory (where `go.mod` lives), or use `-C path` (Go 1.20+) | +| `build constraints exclude all Go files in...` | Build tags filtered out every file — match the repo's tag with `-tags=integration` etc. | +| `missing go.sum entry for module` | Run `go mod download` or `go mod tidy` | +| Race detector reports data race | Fix the race; do not silence it. CGO must be enabled | + +## Mocking Rules + +Go has no reflection-based mocking framework that's universally adopted. Pick what the repo already uses: + +- **Interfaces + hand-written fakes** (most idiomatic) — define a small interface in the consumer package and pass a struct that implements it +- **`gomock` / `mockgen`** — if the repo has `//go:generate mockgen ...` directives or `mocks/` directories, regenerate via `go generate ./...` rather than editing generated files +- **`testify/mock`** — used in many repos; instantiate with `new(MockX)` and chain `.On("Method", ...).Return(...)` +- **`httptest`** — for HTTP clients/servers; spin up `httptest.NewServer` instead of mocking `http.Client` + +Always prefer dependency injection over global function patching. If a test needs more than 3 mocks, flag it as a design smell. + +## Concurrency and Cleanup + +- Use `t.Cleanup(func() { ... })` instead of deferring in test bodies — runs even if `t.FailNow` fires +- Use `t.TempDir()` for temp files — auto-cleaned at test end +- Use `t.Context()` (Go 1.24+) or pass an explicit `context.Background()` — never call real network or filesystem APIs without one in long-running tests + +## Dependency Installation (Last Resort) + +Only install packages after investigation confirms they are missing: + +``` +go get github.com/stretchr/testify@latest +go mod tidy +``` + +Run `go mod tidy` after any `go get` to keep `go.sum` consistent. Never edit `go.sum` by hand. + +## Skip Coverage Tools + +Do not configure or run coverage tools (`-cover`, `-coverprofile`, `go tool cover`). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/java-examples.md b/.github/skills/code-testing-extensions/extensions/java-examples.md new file mode 100644 index 0000000..e920a4c --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/java-examples.md @@ -0,0 +1,344 @@ +# Java Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Java codebase using JUnit 5 + Mockito. These show what each pipeline phase produces for a small project. + +## Source Under Test + +A simple `InvoiceService` in a Maven project using JUnit 5: + +```text +pom.xml +src/main/java/com/contoso/billing/ + InvoiceService.java + Invoice.java (mutable POJO with status, taxRate, lineItems and setStatus / setPaidDate mutators) + InvoiceStatus.java (enum: PENDING, PAID) + InvoiceRepository.java (interface) +src/test/java/com/contoso/billing/ (exists, empty) +``` + +```java +// src/main/java/com/contoso/billing/InvoiceService.java +package com.contoso.billing; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.Clock; +import java.time.LocalDateTime; +import java.util.Optional; + +public class InvoiceService { + + private final InvoiceRepository repository; + private final Clock clock; + + public InvoiceService(InvoiceRepository repository) { + this(repository, Clock.systemUTC()); + } + + public InvoiceService(InvoiceRepository repository, Clock clock) { + this.repository = repository; + this.clock = clock; + } + + public BigDecimal calculateTotal(Invoice invoice) { + if (invoice == null) { + throw new IllegalArgumentException("invoice must not be null"); + } + if (invoice.lineItems().isEmpty()) { + throw new IllegalStateException("Invoice has no line items."); + } + BigDecimal subtotal = invoice.lineItems().stream() + .map(li -> li.unitPrice().multiply(BigDecimal.valueOf(li.quantity()))) + .reduce(BigDecimal.ZERO, BigDecimal::add); + BigDecimal tax = subtotal.multiply(invoice.taxRate()); + return subtotal.add(tax).setScale(2, RoundingMode.HALF_UP); + } + + public Invoice getById(int id) { + Optional invoice = repository.find(id); + return invoice.orElseThrow( + () -> new IllegalArgumentException("Invoice " + id + " not found.")); + } + + public void markAsPaid(int id) { + Invoice invoice = repository.find(id) + .orElseThrow(() -> new IllegalArgumentException("Invoice " + id + " not found.")); + if (invoice.status() == InvoiceStatus.PAID) { + throw new IllegalStateException("Invoice is already paid."); + } + invoice.setStatus(InvoiceStatus.PAID); + invoice.setPaidDate(LocalDateTime.now(clock)); + repository.update(invoice); + } +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/billing +- **Language**: Java 21 (`21`) +- **Build Tool**: Maven (wrapper `./mvnw` present) +- **Test Framework**: JUnit 5 (Jupiter 5.10) + Mockito 5.x (detected in pom.xml) +- **Assertion library**: built-in `Assertions` (no AssertJ/Hamcrest in deps) + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Compile**: `./mvnw -q test-compile` +- **Test**: `./mvnw -q test` +- **Single class**: `./mvnw -q test -Dtest=InvoiceServiceTest` +- **Single method**: `./mvnw -q test -Dtest=InvoiceServiceTest#calculateTotal_validLineItems_returnsExpectedTotal` + +## Project Structure +- Source: `src/main/java/com/contoso/billing/` +- Tests: `src/test/java/com/contoso/billing/` (exists, empty) + +## Files to Test + +### High Priority +| File | Classes/Methods | Testability | Notes | +|------|-----------------|-------------|-------| +| InvoiceService.java | calculateTotal, getById, markAsPaid | High | Repository dependency mockable via Mockito; clock injection available for time-dependent test | + +## Testing Patterns +- No existing patterns; recommend JUnit 5 + Mockito with `@ExtendWith(MockitoExtension.class)`, `@Mock` / `@InjectMocks` fields, `@ParameterizedTest` + `@CsvSource` for table-driven `calculateTotal`, and `Clock.fixed(...)` for `markAsPaid` timestamp. + +## Recommendations +- Test class lives in the same package (`com.contoso.billing`) for package-private access if needed +- Inject `Clock.fixed(...)` rather than mocking `LocalDateTime.now(...)` — the service already accepts a Clock +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate JUnit 5 + Mockito tests for InvoiceService, covering all three public +methods across happy path, edge case, and error scenarios. Single phase since +there is only one source file. + +## Commands +- **Compile**: `./mvnw -q test-compile` +- **Test**: `./mvnw -q test -Dtest=InvoiceServiceTest` + +## Phase 1: InvoiceService + +### Files to Test + +#### 1. InvoiceService.java +- **Source**: `src/main/java/com/contoso/billing/InvoiceService.java` +- **Test File**: `src/test/java/com/contoso/billing/InvoiceServiceTest.java` + +**Methods to Test**: +1. `calculateTotal` — pure logic (parameterized) + - Happy paths: single item w/ tax, multi-quantity zero tax, rounding-half-up + - Error cases: null invoice → IllegalArgumentException; empty line items → IllegalStateException +2. `getById` — happy + missing +3. `markAsPaid` — happy (verify status + paid date via fixed clock + verify update) + already-paid + missing +``` + +## Sample Generated Test File + +```java +// src/test/java/com/contoso/billing/InvoiceServiceTest.java +package com.contoso.billing; + +import java.math.BigDecimal; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class InvoiceServiceTest { + + @Mock + InvoiceRepository repository; + + @InjectMocks + InvoiceService sut; + + // --- calculateTotal --- + + @ParameterizedTest(name = "qty={0} unitPrice={1} taxRate={2} -> {3}") + @CsvSource({ + "1, 100.00, 0.10, 110.00", + "3, 25.00, 0.00, 75.00", + "2, 9.99, 0.07, 21.38" + }) + void calculateTotal_validLineItems_returnsExpectedTotal( + int quantity, BigDecimal unitPrice, BigDecimal taxRate, BigDecimal expected + ) { + Invoice invoice = new Invoice(1, InvoiceStatus.PENDING, taxRate, + List.of(new LineItem(quantity, unitPrice))); + + BigDecimal total = sut.calculateTotal(invoice); + + assertEquals(0, total.compareTo(expected), + () -> "expected " + expected + " but got " + total); + } + + @Test + @DisplayName("null invoice throws IllegalArgumentException") + void calculateTotal_nullInvoice_throws() { + assertThrows(IllegalArgumentException.class, () -> sut.calculateTotal(null)); + } + + @Test + void calculateTotal_emptyLineItems_throws() { + Invoice invoice = new Invoice(1, InvoiceStatus.PENDING, BigDecimal.ZERO, List.of()); + + IllegalStateException ex = assertThrows(IllegalStateException.class, + () -> sut.calculateTotal(invoice)); + assertEquals("Invoice has no line items.", ex.getMessage()); + } + + // --- getById --- + + @Test + void getById_existingId_returnsInvoice() { + Invoice expected = new Invoice(42, InvoiceStatus.PENDING, BigDecimal.ZERO, List.of()); + when(repository.find(42)).thenReturn(Optional.of(expected)); + + assertSame(expected, sut.getById(42)); + } + + @Test + void getById_missingId_throws() { + when(repository.find(999)).thenReturn(Optional.empty()); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> sut.getById(999)); + assertEquals("Invoice 999 not found.", ex.getMessage()); + } + + // --- markAsPaid (uses an injected fixed Clock instead of @InjectMocks) --- + + @Test + void markAsPaid_pendingInvoice_transitionsToPaidAndPersists() { + Clock fixed = Clock.fixed(Instant.parse("2025-01-01T12:00:00Z"), ZoneOffset.UTC); + InvoiceService service = new InvoiceService(repository, fixed); + Invoice invoice = new Invoice(1, InvoiceStatus.PENDING, BigDecimal.ZERO, List.of()); + when(repository.find(1)).thenReturn(Optional.of(invoice)); + + service.markAsPaid(1); + + assertEquals(InvoiceStatus.PAID, invoice.status()); + assertEquals(LocalDateTime.ofInstant(fixed.instant(), ZoneOffset.UTC), invoice.paidDate()); + verify(repository).update(invoice); + } + + @Test + void markAsPaid_alreadyPaid_throwsAndDoesNotUpdate() { + Invoice invoice = new Invoice(1, InvoiceStatus.PAID, BigDecimal.ZERO, List.of()); + when(repository.find(1)).thenReturn(Optional.of(invoice)); + + assertThrows(IllegalStateException.class, () -> sut.markAsPaid(1)); + verify(repository, never()).update(any()); + } + + @Test + void markAsPaid_missingId_throws() { + when(repository.find(999)).thenReturn(Optional.empty()); + + assertThrows(IllegalArgumentException.class, () -> sut.markAsPaid(999)); + } +} +``` + +## Sample Fix Cycle + +When the implementer hits a compile or runtime error, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +[ERROR] No tests found for given includes: [com.contoso.billing.InvoiceServiceTest] +``` + +**Fixer diagnosis:** Surefire only includes `**/*Test.class` (default). The class is `InvoiceServiceTest` (correct) but it was created under `src/test/java/com/contoso/billing/` with **no** package declaration. Maven compiles it into the default package, so `-Dtest=com.contoso.billing.InvoiceServiceTest` doesn't match. + +**Fix applied:** Add `package com.contoso.billing;` at the top of the test file so it lands in the expected package. + +**Rebuild + rerun:** `./mvnw -q test -Dtest=InvoiceServiceTest` → SUCCESS + +--- + +**Another common cycle — wrong Mockito setup:** + +**Test output:** + +```text +org.mockito.exceptions.misusing.UnnecessaryStubbingException: +Unnecessary stubbings detected. + 1. -> at InvoiceServiceTest.calculateTotal_nullInvoice_throws(InvoiceServiceTest.java:55) +``` + +**Fixer diagnosis:** `@MockitoExtension` runs in strict mode by default — stubbed calls (`when(repository.find(...)).thenReturn(...)`) must be used. The test stubbed `repository` in a `@BeforeEach` for every test, but `calculateTotal_nullInvoice_throws` never touches the repository. + +**Fix applied:** Move stubs into the tests that actually need them (as shown in the generated file above), rather than a single shared `@BeforeEach`. + +**Rebuild + rerun:** SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: billing (Java / Maven) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 8 | +| Tests passing | 8 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `src/test/java/com/contoso/billing/InvoiceServiceTest.java` (8 tests, 3 parameterized cases via @CsvSource) + +### Coverage +- InvoiceService.calculateTotal — 3 happy path, 2 error cases +- InvoiceService.getById — happy + missing +- InvoiceService.markAsPaid — happy (fixed Clock) + already-paid + missing + +### Build / Test Validation +- `./mvnw -q test-compile`: ✅ +- `./mvnw -q test`: ✅ Tests run: 8, Failures: 0, Errors: 0 + +### Next Steps +- Add AssertJ if the team standardises on it (more expressive assertions) +- Consider Testcontainers for true repository integration tests +``` diff --git a/.github/skills/code-testing-extensions/extensions/java.md b/.github/skills/code-testing-extensions/extensions/java.md new file mode 100644 index 0000000..2a3e74d --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/java.md @@ -0,0 +1,198 @@ +# Java Extension + +Language-specific guidance for Java test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `*Test.java` / `*Tests.java` / `*IT.java` (integration) files and copy their style (JUnit version, assertion library, mock library, lifecycle methods) +2. **Build file** — `pom.xml` (Maven), `build.gradle` / `build.gradle.kts` (Gradle), `BUILD` / `BUILD.bazel` (Bazel) +3. **Java version** — ``, `sourceCompatibility`, or `toolchains` block +4. **Wrapper scripts** — always prefer `./mvnw` or `./gradlew` over a system-installed Maven/Gradle so you match the project's pinned version + +Use whatever framework the repo already uses (JUnit 4, JUnit 5/Jupiter, TestNG). Do not migrate to a different framework as a side effect of writing tests. + +## Build Tool Detection + +| Indicator | Build tool | Default test command | +|-----------|------------|----------------------| +| `pom.xml` | Maven | `./mvnw test` | +| `build.gradle` / `build.gradle.kts` | Gradle | `./gradlew test` | +| `settings.gradle*` with `include 'subproject'` | Gradle multi-project | `./gradlew :subproject:test` | +| `BUILD` / `BUILD.bazel` | Bazel | `bazel test //path/to:test` | + +If both `pom.xml` and `build.gradle` exist, pick the one used by CI. + +## Build Commands + +| Scope | Maven | Gradle | +|-------|-------|--------| +| Compile main + test | `./mvnw test-compile` | `./gradlew testClasses` | +| Compile only | `./mvnw compile` | `./gradlew classes` | +| Full build | `./mvnw verify` | `./gradlew build` | +| Skip tests during build | `./mvnw -DskipTests package` | `./gradlew assemble` | + +- Use `-q` (Maven) / `--console=plain` (Gradle) to reduce output noise +- For Gradle, prefer `--no-daemon` only in CI; locally the daemon makes incremental builds far faster + +## Test Commands + +| Scope | Maven | Gradle | +|-------|-------|--------| +| All unit tests | `./mvnw test` | `./gradlew test` | +| Single class | `./mvnw test -Dtest=MyClassTest` | `./gradlew test --tests MyClassTest` | +| Single method | `./mvnw test -Dtest=MyClassTest#myMethod` | `./gradlew test --tests MyClassTest.myMethod` | +| Tag filter (JUnit 5) | `./mvnw test -Dgroups=fast` | `./gradlew test -PincludeTags=fast` (if configured) or `--tests` | +| Integration tests | `./mvnw verify -DskipUnitTests` (with failsafe-plugin) | `./gradlew integrationTest` (if registered) | + +- `Surefire` runs unit tests (`*Test.java`); `Failsafe` runs integration tests (`*IT.java`) — do not put long integration tests under Surefire +- Gradle's `--tests` accepts wildcards: `--tests "*MyMethod*"` +- Use `--rerun-tasks` (Gradle) or `-DforkCount=...` (Surefire) only when troubleshooting cache issues + +## Lint Command + +Use the repo's existing lint task first. Otherwise check for: + +- Checkstyle (`checkstyle.xml`, `checkstyle`) → `./mvnw checkstyle:check` or `./gradlew checkstyleMain` +- Spotless (`spotless` block / plugin) → `./mvnw spotless:apply` or `./gradlew spotlessApply` +- ErrorProne / NullAway → integrated into compilation; run a normal build +- google-java-format / palantir-java-format → use the repo's configured formatter + +Never disable existing checks in the test files you generate. + +## Project Layout and Imports + +Maven/Gradle conventional layout: + +``` +src/ +├── main/java/com/example/foo/Bar.java +├── main/resources/ +├── test/java/com/example/foo/BarTest.java +└── test/resources/ +``` + +| Layout | Test placement | +|--------|----------------| +| Standard | `src/test/java//Test.java` | +| Integration tests separated | `src/integrationTest/java/...` (Gradle) or `src/it/java/...` (Maven w/ failsafe) | +| Multi-module Maven | Tests live in the same module as the code under test | + +- Test classes must mirror the production class's **package** to access package-private members +- Avoid wildcard imports unless the repo already uses them — match the explicit imports shown in the templates below +- For JUnit 5: import `org.junit.jupiter.api.Test` (and other annotations as needed) and `org.junit.jupiter.api.Assertions.assertEquals` etc. as static imports +- For JUnit 4: import `org.junit.Test`, `org.junit.Before`, etc., and `org.junit.Assert.assertEquals` etc. as static imports + +## Test Framework Detection + +| Indicator | Framework | Annotations | Assertion style | +|-----------|-----------|-------------|------------------| +| `junit-jupiter-*` deps | JUnit 5 | `@Test`, `@ParameterizedTest`, `@BeforeEach`, `@DisplayName` | `Assertions.assertEquals(expected, actual)` | +| `junit:junit:4.x` | JUnit 4 | `@Test`, `@Before`, `@RunWith` | `Assert.assertEquals(expected, actual)` | +| `org.testng:testng` | TestNG | `@Test(groups=...)`, `@BeforeMethod` | `Assert.assertEquals(actual, expected)` (note **reversed** order) | +| `org.assertj:assertj-core` | AssertJ (assertions only) | n/a | `assertThat(actual).isEqualTo(expected)` | +| `org.hamcrest:hamcrest` | Hamcrest matchers | n/a | `assertThat(actual, is(equalTo(expected)))` | + +**Argument order matters**: JUnit/AssertJ use `(expected, actual)`; TestNG uses `(actual, expected)`. Reversing them produces confusing failure messages. + +## JUnit 5 Template + +```java +package com.example.foo; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class CalculatorTest { + + @Test + @DisplayName("add returns sum of two positive numbers") + void add_positiveNumbers_returnsSum() { + Calculator sut = new Calculator(); + assertEquals(5, sut.add(2, 3)); + } + + @ParameterizedTest + @CsvSource({ + "2, 3, 5", + "-1, 1, 0" + }) + void add_validInputs_returnsSum(int a, int b, int expected) { + assertEquals(expected, new Calculator().add(a, b)); + } + + @Test + void divide_byZero_throws() { + Calculator sut = new Calculator(); + assertThrows(ArithmeticException.class, () -> sut.divide(1, 0)); + } +} +``` + +## Common Errors + +| Error | Fix | +|-------|-----| +| `package X does not exist` | Add the dependency to `pom.xml` / `build.gradle`; run `./mvnw dependency:resolve` or `./gradlew --refresh-dependencies` | +| `cannot find symbol` | Verify class name and import path; check that the test source set sees the production source set | +| `No tests found for given includes` (Gradle) | `--tests` pattern doesn't match; verify the class/method names, that test methods are annotated with `@Test`, and that the class name matches the test task's `include` pattern (default `**/*Test*.class`). For JUnit 4 only, the class must also be `public` with a public no-arg constructor — JUnit 5 allows package-private classes and methods | +| `Test class should have exactly one public zero-argument constructor` (JUnit 4) | Remove constructors with parameters; use `@Before` for setup | +| `org.junit.runners.model.InvalidTestClassError` (JUnit 4) | Class is missing `public`, has wrong constructor, or method signature is wrong | +| Mixing `org.junit.Test` (4) and `org.junit.jupiter.api.Test` (5) | Pick one framework per test class — imports must match the framework annotation | +| `java.lang.NoClassDefFoundError` at runtime | Test runtime classpath is missing a transitive dep; add it to `testRuntimeOnly` (Gradle) or `test` (Maven) | +| `UnsupportedClassVersionError` | JDK used to run tests is older than the JDK used to compile; align toolchains | +| `Mockito cannot mock final class` | Use Mockito's inline mock maker — Mockito 5+ uses it by default; for Mockito 3.x/4.x add the `mockito-inline` artifact (replaces `mockito-core`). Or switch to MockK for Kotlin. `mockito-subclass` does **not** mock final classes | +| `WrongTypeOfReturnValue` (Mockito) | The stubbed method returns a different type than the mock was set up for — check return type signatures | + +## Mocking Rules + +- Use whatever the repo already uses: **Mockito** (most common), **EasyMock**, **JMockit**, or hand-written fakes +- For JUnit 5 + Mockito, use `@ExtendWith(MockitoExtension.class)` with `@Mock` / `@InjectMocks` fields +- For JUnit 4 + Mockito, use `@RunWith(MockitoJUnitRunner.class)` or `MockitoAnnotations.openMocks(this)` in `@Before` +- Use `when(mock.method(...)).thenReturn(...)` for stubs and `verify(mock).method(...)` for interactions +- Use `ArgumentCaptor` to assert on complex argument values rather than over-specifying matchers +- Prefer constructor injection so production code stays testable without `@InjectMocks` +- If a test needs more than 3 mocks, flag it as a design smell + +## Spring Boot + +If the repo uses Spring Boot: + +- `@SpringBootTest` loads the full context — slow; use only when needed +- Slice tests are faster: `@WebMvcTest`, `@DataJpaTest`, `@JsonTest` +- Use `@MockBean` (Spring) only inside Spring tests; in plain unit tests use `@Mock` +- Use `@Testcontainers` for real-DB integration tests if the repo already has it on the classpath + +## Dependency Installation (Last Resort) + +Only add dependencies after investigation confirms they are missing. + +Maven (`pom.xml`): + +```xml + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + +``` + +Gradle (`build.gradle.kts`): + +```kotlin +testImplementation("org.junit.jupiter:junit-jupiter:5.10.2") +testRuntimeOnly("org.junit.platform:junit-platform-launcher") +``` + +If the repo uses BOMs (`` or Gradle platforms), reuse them — don't pin a different version than the BOM publishes. + +## Skip Coverage Tools + +Do not configure or run coverage tools (JaCoCo, Cobertura, OpenClover). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/kotlin-examples.md b/.github/skills/code-testing-extensions/extensions/kotlin-examples.md new file mode 100644 index 0000000..a496c13 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/kotlin-examples.md @@ -0,0 +1,300 @@ +# Kotlin Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Kotlin JVM codebase using Gradle + JUnit 5. These show what each pipeline phase produces for a small project. + +> `kotlin.test` follows the same shape for multiplatform projects. Replace JUnit Jupiter parameterization with `@Test` methods or the repo's established KMP data pattern, and place tests under `src/commonTest/kotlin` or the matching target source set. + +## Source Under Test + +A simple `InvoiceService` in a Gradle Kotlin JVM project: + +```text +settings.gradle.kts +build.gradle.kts +src/main/kotlin/com/contoso/billing/ + Invoice.kt + InvoiceRepository.kt + InvoiceService.kt +src/test/kotlin/com/contoso/billing/ (exists, empty) +``` + +```kotlin +// src/main/kotlin/com/contoso/billing/InvoiceService.kt +package com.contoso.billing + +import java.math.BigDecimal +import java.math.RoundingMode +import java.time.Clock +import java.time.LocalDateTime + +class InvoiceService( + private val repository: InvoiceRepository, + private val clock: Clock = Clock.systemUTC(), +) { + fun calculateTotal(invoice: Invoice): BigDecimal { + require(invoice.lineItems.isNotEmpty()) { "Invoice has no line items." } + + val subtotal = invoice.lineItems + .map { it.unitPrice.multiply(BigDecimal.valueOf(it.quantity.toLong())) } + .fold(BigDecimal.ZERO, BigDecimal::add) + val tax = subtotal.multiply(invoice.taxRate) + return subtotal.add(tax).setScale(2, RoundingMode.HALF_UP) + } + + fun getById(id: Int): Invoice = repository.find(id) + ?: throw NoSuchElementException("Invoice $id not found.") + + fun markAsPaid(id: Int) { + val invoice = getById(id) + check(invoice.status != InvoiceStatus.PAID) { "Invoice is already paid." } + + invoice.status = InvoiceStatus.PAID + invoice.paidAt = LocalDateTime.now(clock) + repository.update(invoice) + } +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: Kotlin 2.0 JVM +- **Build Tool**: Gradle wrapper present (`./gradlew`) +- **Test Framework**: JUnit 5 + kotlin.test assertions (`useJUnitPlatform()` and `junit-jupiter-params` detected) +- **Mocking**: MockK is not present; repository is an interface and can be faked directly + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Compile tests**: `./gradlew compileTestKotlin --console=plain` +- **Single class**: `./gradlew test --tests "com.contoso.billing.InvoiceServiceTest" --console=plain` +- **All tests**: `./gradlew test --console=plain` + +## Files to Test + +### High Priority +| File | Classes/Methods | Testability | Notes | +|------|-----------------|-------------|-------| +| src/main/kotlin/com/contoso/billing/InvoiceService.kt | calculateTotal, getById, markAsPaid | High | Repository interface is fakeable; Clock is injectable | + +## Testing Patterns +- No existing patterns; recommend JUnit 5 `@Test`, `@ParameterizedTest` + `@CsvSource`, backticked test names, `kotlin.test` assertions, and a hand-written fake repository. +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate JUnit 5 tests for InvoiceService covering calculation, lookup, and +paid-state transition behavior. + +## Commands +- **Compile tests**: `./gradlew compileTestKotlin --console=plain` +- **Test**: `./gradlew test --tests "com.contoso.billing.InvoiceServiceTest" --console=plain` + +## Phase 1: InvoiceService + +### Files to Test +- **Source**: `src/main/kotlin/com/contoso/billing/InvoiceService.kt` +- **Test File**: `src/test/kotlin/com/contoso/billing/InvoiceServiceTest.kt` + +**Methods to Test**: +1. `calculateTotal` — parameterized tax, zero tax, rounding, empty-line-items error +2. `getById` — existing invoice and missing invoice +3. `markAsPaid` — success with fixed clock, already-paid, missing +``` + +## Sample Generated Test File + +```kotlin +// src/test/kotlin/com/contoso/billing/InvoiceServiceTest.kt +package com.contoso.billing + +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource +import java.math.BigDecimal +import java.time.Clock +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneOffset +import kotlin.test.assertEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class InvoiceServiceTest { + + private class FakeRepository : InvoiceRepository { + val invoices = mutableMapOf() + var updated: Invoice? = null + + override fun find(id: Int): Invoice? = invoices[id] + + override fun update(invoice: Invoice) { + updated = invoice + invoices[invoice.id] = invoice + } + } + + @ParameterizedTest(name = "qty={0}, unitPrice={1}, taxRate={2} -> {3}") + @CsvSource( + "1, 100.00, 0.10, 110.00", + "3, 25.00, 0.00, 75.00", + "2, 9.99, 0.07, 21.38", + ) + fun `calculateTotal returns expected total for valid line items`( + quantity: Int, + unitPrice: BigDecimal, + taxRate: BigDecimal, + expected: BigDecimal, + ) { + val service = InvoiceService(FakeRepository()) + val invoice = invoice( + taxRate = taxRate, + lineItems = mutableListOf(LineItem(quantity = quantity, unitPrice = unitPrice)), + ) + + val total = service.calculateTotal(invoice) + + assertEquals(0, total.compareTo(expected), "expected $expected but got $total") + } + + @Test + fun `calculateTotal throws for empty line items`() { + val service = InvoiceService(FakeRepository()) + val invoice = invoice(lineItems = mutableListOf()) + + val exception = assertThrows { service.calculateTotal(invoice) } + + assertTrue(exception.message!!.contains("no line items", ignoreCase = true)) + } + + @Test + fun `getById returns existing invoice`() { + val repository = FakeRepository() + val expected = invoice(id = 42) + repository.invoices[42] = expected + val service = InvoiceService(repository) + + val result = service.getById(42) + + assertSame(expected, result) + } + + @Test + fun `getById throws for missing invoice`() { + val service = InvoiceService(FakeRepository()) + + val exception = assertThrows { service.getById(999) } + + assertTrue(exception.message!!.contains("999")) + } + + @Test + fun `markAsPaid updates status date and repository`() { + val repository = FakeRepository() + val invoice = invoice(id = 1) + repository.invoices[1] = invoice + val fixedClock = Clock.fixed(Instant.parse("2025-01-01T12:00:00Z"), ZoneOffset.UTC) + val service = InvoiceService(repository, fixedClock) + + service.markAsPaid(1) + + assertEquals(InvoiceStatus.PAID, invoice.status) + assertEquals(LocalDateTime.ofInstant(fixedClock.instant(), ZoneOffset.UTC), invoice.paidAt) + assertSame(invoice, repository.updated) + } + + @Test + fun `markAsPaid throws and does not update already paid invoice`() { + val repository = FakeRepository() + repository.invoices[1] = invoice(id = 1, status = InvoiceStatus.PAID) + val service = InvoiceService(repository) + + val exception = assertThrows { service.markAsPaid(1) } + + assertTrue(exception.message!!.contains("already paid", ignoreCase = true)) + assertEquals(null, repository.updated) + } + + private fun invoice( + id: Int = 1, + status: InvoiceStatus = InvoiceStatus.PENDING, + taxRate: BigDecimal = BigDecimal("0.10"), + lineItems: MutableList = mutableListOf(LineItem(quantity = 1, unitPrice = BigDecimal("100.00"))), + ): Invoice = Invoice(id = id, status = status, taxRate = taxRate, lineItems = lineItems, paidAt = null) +} +``` + +## Sample Fix Cycle + +When the implementer hits a Gradle or Kotlin compile issue, the fixer agent diagnoses and resolves it. + +**Build output:** + +```text +No tests found for given includes: [com.contoso.billing.InvoiceServiceTest] +``` + +**Fixer diagnosis:** The test file was created under `src/test/java` with Kotlin source, so the Kotlin JVM source set did not compile it into the expected package. + +**Fix applied:** Move the file to `src/test/kotlin/com/contoso/billing/InvoiceServiceTest.kt` and keep `package com.contoso.billing` at the top. + +**Rebuild + rerun:** `./gradlew test --tests "com.contoso.billing.InvoiceServiceTest" --console=plain` → SUCCESS + +--- + +**Another common cycle — JUnit Platform not enabled:** + +**Test output:** + +```text +0 tests completed +``` + +**Fixer diagnosis:** The project has JUnit Jupiter dependencies but the Gradle `test` task is not configured with `useJUnitPlatform()`. + +**Fix applied:** Match the repo's build convention and add `tasks.test { useJUnitPlatform() }` if it is missing. + +**Rerun:** SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: contoso-billing (Kotlin / Gradle) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 8 | +| Tests passing | 8 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `src/test/kotlin/com/contoso/billing/InvoiceServiceTest.kt` (8 JUnit 5 tests, 3 parameterized cases) + +### Coverage +- InvoiceService.calculateTotal — 3 happy path, 1 error case +- InvoiceService.getById — found and missing branches +- InvoiceService.markAsPaid — success and already-paid branches + +### Build / Test Validation +- Compile tests: ✅ `./gradlew compileTestKotlin --console=plain` +- Test run: ✅ `./gradlew test --tests "com.contoso.billing.InvoiceServiceTest" --console=plain` +``` diff --git a/.github/skills/code-testing-extensions/extensions/kotlin.md b/.github/skills/code-testing-extensions/extensions/kotlin.md new file mode 100644 index 0000000..1bebd0a --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/kotlin.md @@ -0,0 +1,227 @@ +# Kotlin Extension + +Language-specific guidance for Kotlin test generation. For pure-Java codebases, use `java.md` instead. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find files in `src/test/kotlin/`, `src/commonTest/kotlin/`, `src/jvmTest/kotlin/`, etc., and copy their style (framework, assertion library, mock library, coroutine helpers) +2. **Build file** — `build.gradle.kts` / `build.gradle` — note Kotlin version, plugins (`kotlin("jvm")`, `kotlin("multiplatform")`, `kotlin("android")`), and `dependencies { testImplementation(...) }` +3. **`gradle/libs.versions.toml`** — the version catalog if the repo uses one; reference aliases instead of hard-coded versions +4. **Wrapper script** — always invoke `./gradlew` (Unix) or `.\gradlew.bat` (Windows), never a system-installed Gradle +5. **Multiplatform layout** — `src//kotlin/` indicates KMP; tests live in matching `*Test` source sets (`commonTest`, `jvmTest`, `nativeTest`) + +Use whatever framework the repo already uses (JUnit Jupiter, JUnit 4, Kotest, kotlin.test). Do not switch. + +## Project Type Detection + +| Indicator | Project type | +|-----------|--------------| +| `kotlin("jvm")` plugin | Plain JVM Kotlin | +| `kotlin("multiplatform")` plugin with `kotlin { jvm(); js(); ... }` | Kotlin Multiplatform | +| `com.android.application` / `com.android.library` plugin | Android | +| `org.springframework.boot` plugin | Spring Boot Kotlin | +| `kotlin("jvm")` + `application` plugin | Kotlin CLI / server | + +For **Android**, see also platform-specific test types: `src/test/` for unit tests on the JVM, `src/androidTest/` for instrumented tests on a device/emulator. They use different runners and gradle tasks. + +## Build Commands + +| Scope | Command | +|-------|---------| +| Compile main + test (JVM) | `./gradlew compileTestKotlin` | +| Full build | `./gradlew build` | +| Skip tests | `./gradlew assemble` | +| Single module | `./gradlew :module-name:build` | +| KMP target only | `./gradlew :module:jvmTest` (or `linuxX64Test`, etc.) | + +- Use `--console=plain` to suppress Gradle's animated output +- Use `--build-cache` (often default in CI) to reuse outputs +- For Android: `./gradlew assembleDebug` (build APK) and `./gradlew testDebugUnitTest` (run unit tests) + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests (JVM) | `./gradlew test` | +| Single class | `./gradlew test --tests "com.example.WidgetTest"` | +| Single method | `./gradlew test --tests "com.example.WidgetTest.add returns sum"` | +| KMP all targets | `./gradlew allTests` | +| KMP one target | `./gradlew jvmTest`, `./gradlew jsTest`, `./gradlew linuxX64Test` | +| Android unit tests | `./gradlew testDebugUnitTest` | +| Android instrumented | `./gradlew connectedDebugAndroidTest` (requires device/emulator) | + +- `--tests` accepts wildcards: `--tests "*Widget*"`. Method names with spaces or backticks must be quoted: `--tests "com.example.WidgetTest.creates a widget"` +- Use `--rerun-tasks` only when troubleshooting cache issues +- For Kotest, the runner is registered with JUnit Platform — the standard `./gradlew test` and `--tests` flags work the same way + +## Lint Command + +Use the repo's lint tooling first: + +- `./gradlew ktlintCheck` (autoformat: `./gradlew ktlintFormat`) when ktlint is configured +- `./gradlew detekt` when detekt is configured +- `./gradlew spotlessCheck` / `spotlessApply` for the Spotless plugin +- Android Studio's IDE inspections; `./gradlew lint` (Android-only) for the Android Lint task + +## Project Layout + +``` +src/ +├── main/kotlin/com/example/foo/Bar.kt +├── main/resources/ +├── test/kotlin/com/example/foo/BarTest.kt # mirrors production package +└── test/resources/ +``` + +KMP layout: + +``` +src/ +├── commonMain/kotlin/... # shared +├── commonTest/kotlin/... # shared tests using kotlin.test +├── jvmMain/kotlin/... +├── jvmTest/kotlin/... +├── jsMain/kotlin/... +└── jsTest/kotlin/... +``` + +- Test classes mirror the production class's package so they can access `internal` members (Kotlin's `internal` is module-scoped — within the same Gradle module, including the test source set) +- For KMP common tests, you can only import from `kotlin.test` and other multiplatform-aware libraries (e.g. `kotlinx.coroutines.test`, Kotest multiplatform, MockK on JVM only) + +## Test Framework Detection + +| Dependency | Framework | Annotations / DSL | +|------------|-----------|--------------------| +| `org.jetbrains.kotlin:kotlin-test` | kotlin.test (multiplatform) | `@Test`, `@BeforeTest`, `assertEquals`, `assertFailsWith` | +| `junit-jupiter-*` | JUnit 5 | `@Test`, `@ParameterizedTest`, `@BeforeEach`, `@DisplayName` | +| `junit:junit:4.x` | JUnit 4 | `@Test`, `@Before`, `@RunWith(JUnitPlatform::class)` rare | +| `io.kotest:kotest-runner-junit5` | Kotest | `class FooSpec : FunSpec({ test("...") { ... } })` (DSL — many styles: `StringSpec`, `BehaviorSpec`, etc.) | +| `org.spekframework.spek2:spek-dsl-jvm` | Spek 2 | `object FooSpec : Spek({ describe(...) { it(...) {} } })` (legacy) | + +For Kotest, **stick to the spec style the repo already uses** — mixing styles is confusing. + +## Test Templates + +### JUnit 5 + +```kotlin +package com.example.foo + +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import kotlin.test.assertEquals + +class CalculatorTest { + + @Test + @DisplayName("add returns sum of two positive numbers") + fun `add returns sum of two positives`() { + val sut = Calculator() + assertEquals(5, sut.add(2, 3)) + } + + @Test + fun `divide by zero throws`() { + val sut = Calculator() + assertThrows { sut.divide(1, 0) } + } +} +``` + +Backticked method names (`` `like this` ``) are idiomatic for Kotlin tests because they read better in failure messages. + +### Kotest (StringSpec) + +```kotlin +package com.example.foo + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe +import io.kotest.assertions.throwables.shouldThrow + +class CalculatorSpec : StringSpec({ + "add returns sum of two positive numbers" { + Calculator().add(2, 3) shouldBe 5 + } + + "divide by zero throws" { + shouldThrow { Calculator().divide(1, 0) } + } +}) +``` + +## Coroutines + +- Use `kotlinx-coroutines-test` when it's already on the classpath; otherwise add it as a `testImplementation` only after confirming it is missing (see Dependency Installation) +- Use `runTest { ... }` (replaces the older `runBlockingTest`) for `suspend` test bodies +- For virtual time advance, use a `TestDispatcher` built from `testScheduler` — e.g. `StandardTestDispatcher(testScheduler)` or `UnconfinedTestDispatcher(testScheduler)` — rather than calling `delay` and waiting in real time +- Inject a `CoroutineDispatcher` into production code instead of using `Dispatchers.Main/IO` directly — then swap it in tests via `Dispatchers.setMain(testDispatcher)` + +```kotlin +@Test +fun `loads data eventually`() = runTest { + val repo = FakeRepo() + val dispatcher = StandardTestDispatcher(testScheduler) + val sut = Loader(repo, dispatcher) + sut.start() + advanceUntilIdle() + assertEquals(LoadState.Done, sut.state.value) +} +``` + +## Common Errors + +| Error | Fix | +|-------|-----| +| `Unresolved reference: X` | Add the import; verify the test source set sees the production source set; for KMP, the dep may be declared only in `jvmTest` | +| `Cannot access 'X': it is internal in module Y` | `internal` is module-scoped, so a test in another Gradle module cannot see it. Move the test into the same module, expose a public seam (e.g. a `*-testing` artifact, or change visibility deliberately), or add the consuming module to the source module's `friend modules` via the Kotlin compiler `-Xfriend-paths` option. `@VisibleForTesting` does **not** widen Kotlin visibility | +| `Class 'XTest' is not abstract and does not implement abstract member` (Kotest spec) | The spec class needs a no-arg constructor and a primary-constructor block — match the existing spec style | +| `No tests found for given includes` (Gradle) | `--tests` pattern doesn't match; verify class name and that the framework's runner is registered on the test task (`useJUnitPlatform()`) | +| `kotlin.UninitializedPropertyAccessException: lateinit property X has not been initialized` | The `@BeforeEach` (or `BeforeTest`) didn't run, or the field was reset; use `lateinit` only after confirming the lifecycle hook fires | +| `IllegalStateException: Module with the Main dispatcher had failed to initialize` | Coroutines test needs `Dispatchers.setMain(...)` before launching anything that touches `Dispatchers.Main`; reset with `Dispatchers.resetMain()` in teardown | +| `Mockito cannot mock final class` | Kotlin classes are `final` by default — either use **MockK** (works with final classes) or apply the `kotlin-allopen` plugin scoped to a marker annotation | +| `MissingMockKException` | The mock wasn't initialized; call `MockKAnnotations.init(this)` or use `@MockK` with `@MockKExtension` (JUnit 5) | +| KMP common test references a JVM-only API | Move the test to `jvmTest`, or use `expect/actual` declarations | +| Android: `Method ... not mocked` | The unit test runs on the JVM and the SDK class is just a stub — either use Robolectric, move the test to instrumented (`androidTest`), or refactor to inject the dependency | + +## Mocking Rules + +- **MockK** is the de-facto standard for Kotlin (final classes, coroutine support): `every { mock.foo() } returns 1`, `coEvery { mock.suspendFn() } returns 1`, `verify { mock.foo() }`, `coVerify { ... }` +- Mockito works on Kotlin too with `mockito-kotlin` extensions, but Kotlin classes are `final` by default — use Mockito's inline mock maker (default in Mockito 5+; the `mockito-inline` artifact for Mockito 3.x/4.x). `mockito-subclass` cannot mock final classes +- Avoid `mockkStatic`/`mockkObject` for production code you control — refactor to a wrapper instead +- Prefer constructor injection so you don't need framework annotations (`@InjectMocks`) at all +- If a test needs more than 3 mocks, flag it as a design smell + +## Android Specifics + +- Robolectric tests live under `src/test/` and emulate the Android framework on the JVM — fast but imperfect +- Instrumented tests live under `src/androidTest/`, require a connected device/emulator, and are slow — use sparingly +- Compose UI tests use `createComposeRule()` and `composeTestRule.onNodeWithText(...).performClick()` — match the existing test setup if Compose is in the project +- Hilt: use `@HiltAndroidTest` and `HiltAndroidRule` for instrumented tests; for unit tests pass fakes directly to ViewModels + +## Dependency Installation (Last Resort) + +Only add dependencies after investigation confirms they are missing. + +`build.gradle.kts`: + +```kotlin +dependencies { + testImplementation("org.junit.jupiter:junit-jupiter:5.10.2") + testImplementation("io.mockk:mockk:1.13.10") + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0") +} + +tasks.test { + useJUnitPlatform() +} +``` + +If the repo uses a version catalog, add to `gradle/libs.versions.toml` and reference via `libs.junit.jupiter` etc. Match the major versions already in use. + +## Skip Coverage Tools + +Do not configure or run coverage tools (JaCoCo, Kover). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/powershell-examples.md b/.github/skills/code-testing-extensions/extensions/powershell-examples.md new file mode 100644 index 0000000..119db68 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/powershell-examples.md @@ -0,0 +1,267 @@ +# PowerShell Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a PowerShell module using Pester v5. These show what each pipeline phase produces for a small module. + +## Source Under Test + +A simple `InvoiceService` module: + +```text +src/ + Contoso.Billing.psd1 + Contoso.Billing.psm1 +Tests/ (empty) +``` + +```powershell +# src/Contoso.Billing.psm1 +enum InvoiceStatus { + Pending + Paid +} + +function Get-InvoiceTotal { + [CmdletBinding()] + param([Parameter(Mandatory)][pscustomobject]$Invoice) + + if (-not $Invoice.LineItems -or $Invoice.LineItems.Count -eq 0) { + throw 'Invoice has no line items.' + } + + $subtotal = 0 + foreach ($lineItem in $Invoice.LineItems) { + $subtotal += $lineItem.Quantity * $lineItem.UnitPrice + } + + [math]::Round($subtotal + ($subtotal * $Invoice.TaxRate), 2) +} + +function Get-InvoiceById { + [CmdletBinding()] + param([Parameter(Mandatory)][int]$Id, [Parameter(Mandatory)][scriptblock]$FindInvoice) + + $invoice = & $FindInvoice $Id + if ($null -eq $invoice) { + throw "Invoice $Id not found." + } + + $invoice +} + +function Set-InvoicePaid { + [CmdletBinding()] + param( + [Parameter(Mandatory)][int]$Id, + [Parameter(Mandatory)][scriptblock]$FindInvoice, + [Parameter(Mandatory)][scriptblock]$UpdateInvoice, + [scriptblock]$GetNow = { Get-Date } + ) + + $invoice = Get-InvoiceById -Id $Id -FindInvoice $FindInvoice + if ($invoice.Status -eq [InvoiceStatus]::Paid) { + throw 'Invoice is already paid.' + } + + $invoice.Status = [InvoiceStatus]::Paid + $invoice.PaidDate = & $GetNow + & $UpdateInvoice $invoice +} + +Export-ModuleMember -Function Get-InvoiceTotal, Get-InvoiceById, Set-InvoicePaid +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: C:\work\contoso-billing +- **Language**: PowerShell 7.4 +- **Module**: `src/Contoso.Billing.psd1` imports `Contoso.Billing.psm1` +- **Test Framework**: Pester v5 + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Module load**: `Import-Module ./src/Contoso.Billing.psd1 -Force -ErrorAction Stop` +- **Discovery**: `Invoke-Pester -Configuration @{ Run = @{ Path = './Tests'; PassThru = $true; SkipRun = $true } }` +- **Test**: `Invoke-Pester -Path ./Tests -Output Detailed` + +## Files to Test + +### High Priority +| File | Functions | Testability | Notes | +|------|-----------|-------------|-------| +| src/Contoso.Billing.psm1 | Get-InvoiceTotal, Get-InvoiceById, Set-InvoicePaid | High | Dependencies are scriptblocks, easy to fake; clock is injectable | + +## Testing Patterns +- No existing patterns; recommend Pester v5 `Describe` / `Context` / `It`, `BeforeAll` module import, `-TestCases` for total calculations, and scriptblock fakes for repository operations. +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate Pester v5 tests for total calculation, repository lookup, and the +paid-state transition. Single phase since there is one module file. + +## Commands +- **Import**: `Import-Module ./src/Contoso.Billing.psd1 -Force -ErrorAction Stop` +- **Test**: `Invoke-Pester -Path ./Tests/Contoso.Billing.Tests.ps1 -Output Detailed` + +## Phase 1: Contoso.Billing + +### Files to Test +- **Source**: `src/Contoso.Billing.psm1` +- **Test File**: `Tests/Contoso.Billing.Tests.ps1` + +**Functions to Test**: +1. `Get-InvoiceTotal` — table-driven happy paths and empty-line-items error +2. `Get-InvoiceById` — existing invoice and missing invoice +3. `Set-InvoicePaid` — status/date update and persistence; already-paid error; missing invoice error +``` + +## Sample Generated Test File + +```powershell +# Tests/Contoso.Billing.Tests.ps1 +BeforeAll { + Import-Module (Join-Path $PSScriptRoot '..' 'src' 'Contoso.Billing.psd1') -Force -ErrorAction Stop + + function New-TestInvoice { + param( + [int]$Id = 1, + [InvoiceStatus]$Status = [InvoiceStatus]::Pending, + [double]$TaxRate = 0.10, + [object[]]$LineItems = @(@{ Quantity = 1; UnitPrice = 100.00 }) + ) + + [pscustomobject]@{ + Id = $Id + Status = $Status + TaxRate = $TaxRate + LineItems = $LineItems + PaidDate = $null + } + } +} + +Describe 'Contoso.Billing invoice functions' { + Context 'Get-InvoiceTotal' { + It 'returns for ' -TestCases @( + @{ Name = 'single item with tax'; LineItems = @(@{ Quantity = 1; UnitPrice = 100.00 }); TaxRate = 0.10; Expected = 110.00 } + @{ Name = 'multi quantity zero tax'; LineItems = @(@{ Quantity = 3; UnitPrice = 25.00 }); TaxRate = 0.00; Expected = 75.00 } + @{ Name = 'rounds to two decimals'; LineItems = @(@{ Quantity = 2; UnitPrice = 9.99 }); TaxRate = 0.07; Expected = 21.38 } + ) { + param($LineItems, $TaxRate, $Expected) + + $invoice = New-TestInvoice -LineItems $LineItems -TaxRate $TaxRate + + Get-InvoiceTotal -Invoice $invoice | Should -BeExactly $Expected + } + + It 'throws when the invoice has no line items' { + $invoice = New-TestInvoice -LineItems @() + + { Get-InvoiceTotal -Invoice $invoice } | Should -Throw '*no line items*' + } + } + + Context 'Get-InvoiceById' { + It 'returns an existing invoice' { + $expected = New-TestInvoice -Id 42 + $findInvoice = { param($Id) if ($Id -eq 42) { $expected } } + + $result = Get-InvoiceById -Id 42 -FindInvoice $findInvoice + + $result | Should -BeSame $expected + } + + It 'throws when the invoice is missing' { + $findInvoice = { $null } + + { Get-InvoiceById -Id 999 -FindInvoice $findInvoice } | Should -Throw '*999*' + } + } + + Context 'Set-InvoicePaid' { + It 'marks a pending invoice as paid and persists it' { + $invoice = New-TestInvoice -Id 1 + $script:updatedInvoice = $null + $fixedNow = [datetime]'2025-01-01T12:00:00Z' + $findInvoice = { param($Id) if ($Id -eq 1) { $invoice } } + $updateInvoice = { param($Invoice) $script:updatedInvoice = $Invoice } + + Set-InvoicePaid -Id 1 -FindInvoice $findInvoice -UpdateInvoice $updateInvoice -GetNow { $fixedNow } + + $invoice.Status | Should -Be ([InvoiceStatus]::Paid) + $invoice.PaidDate | Should -Be $fixedNow + $script:updatedInvoice | Should -BeSame $invoice + } + + It 'throws and does not update an already-paid invoice' { + $invoice = New-TestInvoice -Status ([InvoiceStatus]::Paid) + $script:updatedInvoice = $null + $findInvoice = { $invoice } + $updateInvoice = { param($Invoice) $script:updatedInvoice = $Invoice } + + { Set-InvoicePaid -Id 1 -FindInvoice $findInvoice -UpdateInvoice $updateInvoice } | Should -Throw '*already paid*' + $script:updatedInvoice | Should -BeNullOrEmpty + } + } +} +``` + +## Sample Fix Cycle + +When the implementer hits a Pester discovery or run issue, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +CommandNotFoundException: The term 'Get-InvoiceTotal' is not recognized +``` + +**Fixer diagnosis:** The module import was placed at script top level. Import the module in `BeforeAll` so the Pester run phase sees the exported functions. + +**Fix applied:** Move `Import-Module ... -Force` into `BeforeAll` (as shown above). + +**Rerun:** `Invoke-Pester -Path ./Tests/Contoso.Billing.Tests.ps1 -Output Detailed` → SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: contoso-billing (PowerShell) +**Strategy**: Direct (single module in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 8 | +| Tests passing | 8 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `Tests/Contoso.Billing.Tests.ps1` (8 Pester examples, 3 data-driven total cases) + +### Coverage +- Get-InvoiceTotal — 3 happy path, 1 error case +- Get-InvoiceById — found and missing branches +- Set-InvoicePaid — success and already-paid branches + +### Build / Test Validation +- Module load: ✅ `Import-Module ./src/Contoso.Billing.psd1 -Force -ErrorAction Stop` +- Discovery: ✅ Pester found 8 tests +- Test run: ✅ `Invoke-Pester -Path ./Tests -Output Detailed` +``` diff --git a/.github/skills/code-testing-extensions/extensions/powershell.md b/.github/skills/code-testing-extensions/extensions/powershell.md new file mode 100644 index 0000000..f8014f9 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/powershell.md @@ -0,0 +1,166 @@ +# PowerShell Extension + +Language-specific guidance for PowerShell test generation using Pester v5. + +## Rule #0: Confirm the Test Target + +If the prompt does not name a specific file (e.g. "test the repository", "cover one core module", "comprehensive suite"), do **not** assume the largest or top-level upstream code is the intended target. In real workflows the user usually wants to test code they have just added, and large upstream repos contain hundreds of scripts already covered by existing `*.Tests.ps1` files. + +Run these **read-only** discovery commands first — they are the deliberate exception to Rule #1's "before writing any test or running any command" rule, and their output is the ground truth Rule #1's reading is meant to interpret. Do **not** write or execute any tests until Rule #0 and Rule #1 are both complete. + +| Goal | Command | +|------|---------| +| List uncommitted edits + untracked files | `git status -s` | +| Untracked files only (typical for newly-added modules) | `git ls-files --others --exclude-standard` | +| Recently added scripts/modules | `git log --diff-filter=A --name-only -5 -- '*.ps1' '*.psm1' '*.psd1'` | +| Modules with no matching `*.Tests.ps1` | compare `Get-ChildItem -Recurse -Include *.psm1,*.ps1` against `*.Tests.ps1` files | + +Prefer targets that match **all** of: + +1. Untracked or recently added (`git status` / `git log --diff-filter=A`). +2. Small and pure (a few hundred lines, no external state, no `Invoke-WebRequest`/registry/filesystem side effects). +3. Located under a conventional source root (`tools/`, `src/`, `Public/`, `Private/`, or the module root next to a `.psd1`). +4. Have **no** existing matching `*.Tests.ps1` file. + +If a `.psd1` manifest's `RootModule` (or `ModuleToProcess`) points at a specific `.psm1`, that module is almost certainly the target — start there. + +### Test Placement Contract + +Pester only discovers tests under the path passed to `Invoke-Pester -Path` (or the current directory when no path is given). Verification harnesses (CI, msbench, coverage tools) typically scope discovery to a single directory such as `tools/` or `tests/`. Place every test file there, matching the existing convention in the repo: + +| Layout used by the repo | Test placement | +|-------------------------|----------------| +| Co-located convention (`Module.psm1` + `Module.Tests.ps1` side-by-side) | Drop `.Tests.ps1` next to the source file (`tools/StringUtils.psm1` → `tools/StringUtils.Tests.ps1`). | +| Sibling `Tests/` directory | Mirror the source path (`src/Foo/Bar.psm1` → `Tests/Foo/Bar.Tests.ps1`). | +| Mixed / unknown | Co-locate next to the source — this is what Pester discovers by default and what most harnesses scope to. | + +A `*.Tests.ps1` file placed outside the discovery root will be invisible to both `Invoke-Pester` and the harness. + +### First-Test Sanity Loop + +After writing the **first** `*.Tests.ps1` file — before writing any others: + +1. Run `Invoke-Pester -Path -PassThru` and confirm the `TotalCount` is `> 0`. If it is `0`, Pester is not discovering your file; fix the location, filename, or `Describe`/`It` structure before continuing. +2. Run the test (`Invoke-Pester -Path -Output Detailed`); fix `Import-Module` / dot-source / `BeforeAll` errors before adding more tests. +3. Only then expand to cover the remaining functions. + +This catches placement and discovery mistakes on turn 1 instead of after dozens of failed-test iterations. + +### Harness Discovery Check + +Before reporting success, run the **harness-equivalent** discovery command from the repo root and confirm the test count went up by at least the number of tests you generated. CI/msbench/coverage harnesses do not know which directory you targeted with `-Path`; they invoke Pester from the repo root with default discovery, so a test that passes via `Invoke-Pester -Path ./tools/Foo.Tests.ps1` is still worthless if `Invoke-Pester` from the repo root does not enumerate it. + +```powershell +# From repo root — mirrors what a generic harness sees +$result = Invoke-Pester -Configuration @{ Run = @{ PassThru = $true; SkipRun = $true } } +"$($result.TotalCount) tests discovered" +``` + +If the count did not increase, your `*.Tests.ps1` file is outside the harness discovery root. Move it to the convention the repo's existing tests use (or, if there are no existing tests, prefer the repo root's `tests/`, `Tests/`, `tst/`, `test/`, or co-locate next to the source). Do **not** report success until the harness-equivalent command sees your new tests. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `*.Tests.ps1` files and copy their style (structure, assertions, mock approach, import method) +2. **Module structure** — look for `.psd1` (manifest), `.psm1` (root module), `Public/`/`Private/` organization +3. **Build/test scripts** — check for `build.ps1`, `Invoke-Build` (`*.build.ps1`), `psake`, or CI scripts +4. **Shell target** — check `.psd1` for `PowerShellVersion`/`CompatiblePSEditions`, CI matrix for `pwsh` vs `powershell.exe` + +Use the repo's existing test conventions. Only add Pester if the repo has no tests at all. + +## Build Commands + +PowerShell is interpreted — no build step. If the repo has a build script, use it. Otherwise validate with: + +- **Module loads**: `Import-Module ./MyModule.psd1 -Force -ErrorAction Stop` +- **Script analyzer**: `Invoke-ScriptAnalyzer -Path ./src -Recurse` (if PSScriptAnalyzer is available) +- **Lint**: `Invoke-ScriptAnalyzer -Path path/to/file.ps1 -Fix` + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests | `Invoke-Pester` | +| Specific file | `Invoke-Pester -Path ./Tests/Get-Widget.Tests.ps1` | +| Filter by name | `Invoke-Pester -FullNameFilter '*Get-Widget*'` | +| Filter by tag | `Invoke-Pester -TagFilter 'Unit'` | +| Non-interactive (CI) | `Invoke-Pester -CI` | +| Detailed output | `Invoke-Pester -Output Detailed` | + +- Prefer the repo's build/test script over raw `Invoke-Pester` +- Use `-Output Detailed` during fix cycles, `-Output Minimal` for final validation + +## Project Layout and Imports + +| Layout | Import in `BeforeAll` | +|--------|-----------------------| +| Module (`.psd1`) | `Import-Module "$PSScriptRoot/../MyModule.psd1" -Force` | +| Library script (defines functions) | `. $PSScriptRoot/Get-Widget.ps1` | +| Co-located test | `. $PSCommandPath.Replace('.Tests.ps1', '.ps1')` | +| Executable script (has `param()`) | Do **not** dot-source — invoke with `& $PSScriptRoot/script.ps1 -Param value` and assert on output/errors | + +- **All imports go in `BeforeAll`** — never at script top level +- **Use `$PSScriptRoot` or `$PSCommandPath`** — never `$MyInvocation.MyCommand.Path` (returns empty in `BeforeAll`) +- Use `-Force` on `Import-Module` to pick up changes between runs + +## Test File Naming + +- Files: `*.Tests.ps1` — match existing convention (co-located vs `Tests/` directory) + +## Pester v5 Discovery vs Run (Critical) + +Pester v5 runs in **two phases**: Discovery (collects test metadata) then Run (executes tests). This is the #1 source of agent errors. + +**Rules:** +- All setup code goes in `BeforeAll` or `BeforeEach` — never at script top level or loose inside `Describe`/`Context` +- Code directly inside `Describe`/`Context` (but outside `It`/`Before*`/`After*`) runs during **Discovery** — do not put setup, imports, or variable assignments there +- Data for `-ForEach` / `-TestCases` must be set in `BeforeDiscovery`, not `BeforeAll` (BeforeAll runs after discovery) +- `-Skip:$condition` evaluates at Discovery time — conditions from `BeforeAll` will be `$null` +- Use `foreach` loops for dynamic test generation only with `BeforeDiscovery` data +- Use `TestDrive:` for file-based tests instead of touching repo files — Pester cleans it up automatically + +## Common Errors + +| Error | Fix | +|-------|-----| +| Variable is `$null` in `It` block | Move assignment into `BeforeAll` — variables set there are visible to child `It` blocks without `$script:` | +| `-ForEach` data is empty | Move data setup from `BeforeAll` to `BeforeDiscovery` | +| `CommandNotFoundException` for Mock target | The function must exist before mocking — import the module in `BeforeAll` first | +| `$MyInvocation.MyCommand.Path` returns empty | Use `$PSCommandPath` or `$PSScriptRoot` instead | +| `Should Be` (no dash) fails | Use v5 syntax: `Should -Be` (with dash prefix) | +| `Assert-MockCalled` not recognized | Use v5 syntax: `Should -Invoke` | +| Mock has no effect | Check scope — mocks in `It` only apply to that `It`; use `BeforeAll`/`BeforeEach` for broader scope | +| `Should -Throw` doesn't catch cmdlet errors | Most cmdlet errors are non-terminating — wrap with `{ cmd -ErrorAction Stop }` or set `$ErrorActionPreference = 'Stop'` in `BeforeEach` | +| Tests pass on Windows but fail on Linux | Use `Join-Path` not string concatenation; match exact file casing; avoid Windows-only cmdlets (Registry, EventLog) | + +## Mocking Rules + +- Place mocks in `BeforeAll` (shared) or `BeforeEach` (reset per test) +- Mock where the command is **called from** — use `-ModuleName` to mock inside a module's scope +- Use `-ParameterFilter` for selective mocking (no `param()` block needed in v5) +- Verify calls with `Should -Invoke` — default scope inside `It` counts only that test's calls +- Use `InModuleScope` sparingly and as narrowly as possible — prefer `Mock -ModuleName` for testing via public API +- Inside mock bodies, use `$PesterBoundParameters` not `$PSBoundParameters` +- If a test needs more than 3 mocks, flag it as a design smell + +## Non-Obvious Assertions + +Most `Should` operators are self-explanatory. These are the ones agents get wrong: + +- `Should -Throw` requires a **scriptblock**: `{ risky-op } | Should -Throw` — not a direct call +- `Should -Contain` is for **collections** — use `Should -Be` for scalar equality +- `Should -HaveParameter` validates cmdlet signatures: `Get-Command X | Should -HaveParameter 'Name' -Mandatory` +- `Should -Invoke` verifies mock calls: `Should -Invoke Get-Item -Times 1 -Exactly` + +## Cross-Platform + +- Prefer `pwsh` (PowerShell 7+) unless the repo explicitly targets Windows PowerShell 5.1 +- Use `Join-Path` for paths — never string concatenation with `\` +- Linux/macOS file systems are **case-sensitive** — match exact casing in imports and paths +- Windows ships Pester 3.4.0 — if v5 is needed: `Install-Module Pester -Force -SkipPublisherCheck` +- Check `$PSVersionTable.PSEdition` to detect Core vs Desktop + +## Skip Coverage Tools + +Do not configure or run coverage tools (Pester CodeCoverage, JaCoCo export). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/python-examples.md b/.github/skills/code-testing-extensions/extensions/python-examples.md new file mode 100644 index 0000000..6c27601 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/python-examples.md @@ -0,0 +1,411 @@ +# Python Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Python codebase using pytest. These show what each pipeline phase produces for a small project. + +## Source Under Test + +A simple `InvoiceService` in a Python package using pytest: + +```text +src/ + contoso_billing/ + __init__.py + invoice_service.py + invoice.py + invoice_repository.py +tests/ + __init__.py + conftest.py (empty, just marks tests/ as a package root) +pyproject.toml +``` + +```python +# src/contoso_billing/invoice_service.py +from decimal import Decimal, ROUND_HALF_UP +from .invoice import Invoice, InvoiceStatus +from .invoice_repository import InvoiceRepository + + +class InvoiceService: + def __init__(self, repository: InvoiceRepository) -> None: + self._repository = repository + + def calculate_total(self, invoice: Invoice) -> Decimal: + if invoice is None: + raise ValueError("invoice must not be None") + if not invoice.line_items: + raise ValueError("Invoice has no line items.") + + subtotal = sum( + (li.quantity * li.unit_price for li in invoice.line_items), + start=Decimal("0"), + ) + tax = subtotal * invoice.tax_rate + return (subtotal + tax).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + + def get_by_id(self, invoice_id: int) -> Invoice: + invoice = self._repository.find(invoice_id) + if invoice is None: + raise KeyError(f"Invoice {invoice_id} not found.") + return invoice + + def mark_as_paid(self, invoice_id: int) -> None: + invoice = self._repository.find(invoice_id) + if invoice is None: + raise KeyError(f"Invoice {invoice_id} not found.") + if invoice.status == InvoiceStatus.PAID: + raise ValueError("Invoice is already paid.") + invoice.status = InvoiceStatus.PAID + invoice.paid_date = _utcnow() + self._repository.update(invoice) + + +def _utcnow(): + from datetime import datetime, timezone + return datetime.now(timezone.utc) +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: Python 3.11 +- **Framework**: pure library (no Flask/Django) +- **Test Framework**: pytest 8.x (declared in pyproject.toml [project.optional-dependencies].test) +- **Package Layout**: `src/` layout — production package imports as `contoso_billing` + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Install (editable)**: `python -m pip install -e ".[test]"` +- **Build/Type-check**: none configured +- **Test**: `python -m pytest` +- **Lint**: none configured + +## Project Structure +- Source: `src/contoso_billing/` +- Tests: `tests/` (exists, empty besides `conftest.py`) + +## Files to Test + +### High Priority +| File | Classes/Functions | Testability | Notes | +|------|-------------------|-------------|-------| +| src/contoso_billing/invoice_service.py | InvoiceService: calculate_total, get_by_id, mark_as_paid | High | Core business logic, repository dependency needs mocking | + +### Low Priority / Skip +| File | Reason | +|------|--------| +| src/contoso_billing/invoice.py | Dataclass, no logic | +| src/contoso_billing/invoice_repository.py | Interface/protocol, no implementation | + +## Existing Tests +- No existing tests found + +## Testing Patterns +- No existing patterns; recommend pytest function-style tests in `tests/test_invoice_service.py`, `unittest.mock.Mock(spec=InvoiceRepository)` for repository fakes, and `@pytest.mark.parametrize` for table-driven cases. + +## Recommendations +- Start with `calculate_total` (pure logic, easy to parametrize) +- Then `get_by_id` and `mark_as_paid` (require mocking the repository) +- Use `unittest.mock.patch("contoso_billing.invoice_service._utcnow")` to control the timestamp in `mark_as_paid` +``` + +## Sample Plan Output + +What `code-testing-planner` produces in `.testagent/plan.md`: + +```markdown +# Test Implementation Plan + +## Overview +Generate pytest tests for the Contoso Billing InvoiceService, covering all three +public methods across happy path, edge case, and error scenarios. Single phase +since there is only one source file. + +## Commands +- **Install**: `python -m pip install -e ".[test]"` +- **Test**: `python -m pytest tests/test_invoice_service.py -q` +- **Test (file-scoped during dev)**: `python -m pytest tests/test_invoice_service.py::test_calculate_total_valid_line_items_returns_expected_total -q` + +## Phase Summary +| Phase | Focus | Files | Est. Tests | +|-------|-------|-------|------------| +| 1 | InvoiceService | 1 | 9-12 | + +--- + +## Phase 1: InvoiceService + +### Overview +Cover all public methods of InvoiceService. `calculate_total` is pure logic tested +with `@pytest.mark.parametrize`. The async-looking methods are synchronous but +require a mocked InvoiceRepository. + +### Files to Test + +#### 1. invoice_service.py +- **Source**: `src/contoso_billing/invoice_service.py` +- **Test File**: `tests/test_invoice_service.py` + +**Methods to Test**: +1. `calculate_total` — Pure calculation logic + - Happy path: single line item returns quantity × price + tax + - Happy path: multiple line items summed correctly + - Edge case: zero tax rate returns subtotal only + - Error case: None invoice raises ValueError + - Error case: empty line items raises ValueError + +2. `get_by_id` — Repository lookup + - Happy path: existing ID returns invoice + - Error case: missing ID raises KeyError + +3. `mark_as_paid` — State transition + - Happy path: pending invoice transitions to PAID with `paid_date` set + - Error case: already-paid raises ValueError + - Error case: missing ID raises KeyError + +### Success Criteria +- [ ] Test file created at `tests/test_invoice_service.py` +- [ ] `python -m pytest` reports all tests passed +- [ ] No real network/IO; repository is mocked with `Mock(spec=InvoiceRepository)` +``` + +## Sample Generated Test File + +What `code-testing-implementer` produces: + +```python +# tests/test_invoice_service.py +from datetime import datetime, timezone +from decimal import Decimal +from unittest.mock import Mock, patch + +import pytest + +from contoso_billing.invoice import Invoice, InvoiceStatus, LineItem +from contoso_billing.invoice_repository import InvoiceRepository +from contoso_billing.invoice_service import InvoiceService + + +@pytest.fixture +def repository() -> Mock: + return Mock(spec=InvoiceRepository) + + +@pytest.fixture +def sut(repository: Mock) -> InvoiceService: + return InvoiceService(repository) + + +# --- calculate_total --- + +@pytest.mark.parametrize( + "quantity, unit_price, tax_rate, expected", + [ + (1, "100.00", "0.10", "110.00"), + (3, "25.00", "0.00", "75.00"), + (2, "9.99", "0.07", "21.38"), + ], + ids=["single-item-10pct-tax", "multi-quantity-zero-tax", "rounds-half-up"], +) +def test_calculate_total_valid_line_items_returns_expected_total( + sut: InvoiceService, quantity: int, unit_price: str, tax_rate: str, expected: str +) -> None: + invoice = Invoice( + tax_rate=Decimal(tax_rate), + line_items=[LineItem(quantity=quantity, unit_price=Decimal(unit_price))], + ) + + total = sut.calculate_total(invoice) + + assert total == Decimal(expected) + + +def test_calculate_total_none_invoice_raises_value_error(sut: InvoiceService) -> None: + with pytest.raises(ValueError, match="invoice must not be None"): + sut.calculate_total(None) + + +def test_calculate_total_empty_line_items_raises_value_error(sut: InvoiceService) -> None: + invoice = Invoice(tax_rate=Decimal("0"), line_items=[]) + + with pytest.raises(ValueError, match="no line items"): + sut.calculate_total(invoice) + + +# --- get_by_id --- + +def test_get_by_id_existing_id_returns_invoice( + sut: InvoiceService, repository: Mock +) -> None: + expected = Invoice(id=42, tax_rate=Decimal("0"), line_items=[]) + repository.find.return_value = expected + + result = sut.get_by_id(42) + + assert result is expected + repository.find.assert_called_once_with(42) + + +def test_get_by_id_missing_id_raises_key_error( + sut: InvoiceService, repository: Mock +) -> None: + repository.find.return_value = None + + with pytest.raises(KeyError, match="999"): + sut.get_by_id(999) + + +# --- mark_as_paid --- + +def test_mark_as_paid_pending_invoice_sets_status_and_date( + sut: InvoiceService, repository: Mock +) -> None: + invoice = Invoice(id=1, status=InvoiceStatus.PENDING, tax_rate=Decimal("0"), line_items=[]) + repository.find.return_value = invoice + fixed_now = datetime(2025, 1, 1, 12, 0, tzinfo=timezone.utc) + + with patch("contoso_billing.invoice_service._utcnow", return_value=fixed_now): + sut.mark_as_paid(1) + + assert invoice.status == InvoiceStatus.PAID + assert invoice.paid_date == fixed_now + repository.update.assert_called_once_with(invoice) + + +def test_mark_as_paid_already_paid_raises_value_error( + sut: InvoiceService, repository: Mock +) -> None: + invoice = Invoice(id=1, status=InvoiceStatus.PAID, tax_rate=Decimal("0"), line_items=[]) + repository.find.return_value = invoice + + with pytest.raises(ValueError, match="already paid"): + sut.mark_as_paid(1) + + repository.update.assert_not_called() + + +def test_mark_as_paid_missing_id_raises_key_error( + sut: InvoiceService, repository: Mock +) -> None: + repository.find.return_value = None + + with pytest.raises(KeyError, match="999"): + sut.mark_as_paid(999) +``` + +## Sample Fix Cycle + +When the implementer encounters an import or attribute error, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +ModuleNotFoundError: No module named 'contoso_billing' +``` + +**Fixer diagnosis:** The package is not installed in editable mode, so the `src/` layout's package is not on `sys.path`. + +**Fix applied:** + +```bash +python -m pip install -e ".[test]" +``` + +**Rerun:** `python -m pytest tests/test_invoice_service.py -q` → SUCCESS + +--- + +**Another common cycle — patch target wrong:** + +**Test output:** + +```text +AttributeError: does not have the attribute '_utcnow' +``` + +**Fixer diagnosis:** The test patched `datetime._utcnow` but the production code defines its own `_utcnow` helper inside `contoso_billing.invoice_service`. Patches must target the lookup site, not the definition site. + +**Fix applied:** + +```python +# Before (wrong) +with patch("datetime._utcnow", return_value=fixed_now): + +# After (fixed) — patch where the name is looked up +with patch("contoso_billing.invoice_service._utcnow", return_value=fixed_now): +``` + +**Rerun:** SUCCESS + +--- + +**Another common cycle — Mock without spec:** + +**Test output:** + +```text +AttributeError: Mock object has no attribute 'find_by_id' +``` + +(but the actual repository method is `find`, not `find_by_id`) + +**Fixer diagnosis:** `Mock()` happily creates any attribute on access, so a typo in the test went undetected until the production code called `repository.find(...)`. Using `Mock(spec=InvoiceRepository)` would have failed at setup time. + +**Fix applied:** + +```python +# Before +repository = Mock() +repository.find_by_id.return_value = expected # typo, silently accepted + +# After +repository = Mock(spec=InvoiceRepository) +repository.find.return_value = expected # typos now raise AttributeError +``` + +**Rerun:** SUCCESS + +## Sample Final Report + +What `code-testing-generator` produces at Step 9: + +```markdown +## Test Generation Report + +**Project**: contoso-billing +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 9 | +| Tests passing | 9 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `tests/test_invoice_service.py` (9 tests, 3 parametrized) + +### Coverage +- InvoiceService.calculate_total — 3 happy path, 2 error cases +- InvoiceService.get_by_id — 1 happy path, 1 error case +- InvoiceService.mark_as_paid — 1 happy path, 2 error cases + +### Build / Install Validation +- Editable install: ✅ `python -m pip install -e ".[test]"` +- Test run: ✅ `python -m pytest` — 9 passed in 0.12s + +### Next Steps +- Add tests for repository implementations if any exist +- Consider snapshot/property-based testing (`hypothesis`) for `calculate_total` rounding behaviour +``` diff --git a/.github/skills/code-testing-extensions/extensions/python.md b/.github/skills/code-testing-extensions/extensions/python.md new file mode 100644 index 0000000..7abd6cc --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/python.md @@ -0,0 +1,195 @@ +# Python Extension + +Language-specific guidance for Python test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, discover what the repo already does: + +1. **Find ALL existing test files** — search broadly: `test_*.py`, `*_test.py`, `*.uts`, `test/*.sh`, or any other test format. Do not assume pytest. +2. **Identify the test framework and runner** — check, in order: + - `pyproject.toml` `[tool.pytest.ini_options]` `testpaths` + - `pytest.ini` + - `setup.cfg` `[tool:pytest]` + - `tox.ini` `[testenv]` `commands` + - `Makefile`, `noxfile.py`, `conftest.py` locations + - Project-specific runners such as Django `runtests.py`, `manage.py test`, or `DJANGO_SETTINGS_MODULE` +3. **Find the active test layout** — note the directory, working directory, and fixture scope used by existing tests. Some repos use non-standard layouts such as Ansible-style `test/units/`. +4. **Read existing tests thoroughly** — copy their exact style: file format, imports, fixtures, assertion patterns, helper utilities, setup/teardown conventions +5. **Package layout** — determine import paths from existing code, not guesswork + +**Use whatever framework and conventions the repo already uses.** If the repo uses a custom test framework (custom file formats, custom runners, domain-specific test utilities), adopt it fully — do not layer pytest on top. Only introduce pytest if the repo has no tests at all. + +**Never finish with a failing or erroring test.** Run the full new-test suite before finishing. If a test cannot be made to pass within a reasonable number of attempts, delete it. A smaller suite where every test passes is strictly better than a larger suite with any failure — a suite with one failing test can score zero. + +**Start simple to bank coverage.** Write high-certainty tests for pure functions, validation branches, serializers, small helpers, and deterministic error paths before attempting async views, templates, sessions, network paths, or integration-heavy code. + +## Environment Detection + +Detect the runner from lockfiles/config and prefix all commands accordingly: + +| Indicator | Prefix | +|-----------|--------| +| `poetry.lock` / `[tool.poetry]` in `pyproject.toml` | `poetry run` | +| `pdm.lock` / `[tool.pdm]` in `pyproject.toml` | `pdm run` | +| `uv.lock` / `[tool.uv]` in `pyproject.toml` | `uv run` | +| `Pipfile.lock` | `pipenv run` | +| `hatch.toml` / `[tool.hatch]` in `pyproject.toml` | `hatch run` | +| None of the above | `python -m` | + +`` applies to **module execution** only. With the default `python -m` prefix, ` pytest` expands to `python -m pytest`, but a script entry point or inline probe must not be double-prefixed — `python -m python manage.py …` / `python -m python -c …` is invalid. Run script entry points (`manage.py`, `runtests.py`) and `python -c` probes with `python` directly, wrapping with the env tool when one is detected (e.g. `poetry run python manage.py test …`, `uv run python -c "…"`) instead of `python -m`. + +If `Makefile`, `tox.ini`, or `nox` config exists, prefer those scripts over raw commands. + +## Build Commands + +Python has no separate build step. Validate with the type checker if one is configured: + +| Scope | Command | +|-------|---------| +| Syntax check | ` py_compile path/to/file.py` | +| Type check | ` mypy path/to/file.py` or ` pyright path/to/file.py` | + +## Test Commands + +Run new tests the same way the repo runs existing tests: same working directory, same command wrapper, same `conftest.py` scope, and same settings environment variables. + +Before choosing a command, inspect runner configuration with copy-pasteable probes: + +```powershell +Get-ChildItem -Recurse -File -Include pyproject.toml,pytest.ini,setup.cfg,tox.ini,Makefile,noxfile.py,conftest.py,runtests.py,manage.py +Select-String -Path pyproject.toml,pytest.ini,setup.cfg,tox.ini -Pattern 'testpaths|\[tool.pytest|\[tool:pytest|commands|DJANGO_SETTINGS_MODULE' -ErrorAction SilentlyContinue +``` + +If the repo uses a **custom test framework** (custom file formats, custom runner), use its native commands — do not wrap them in pytest. Examples: + +| Framework | Command | +|-----------|---------| +| UTscapy (`.uts` files) | ` scapy.tools.UTscapy -f test/test_file.uts` | +| Django runner script | `python runtests.py app_label.tests.test_module` | +| Django project | `python manage.py test app_label.tests.test_module` | +| Custom runner script | `make test`, `./run_tests.sh`, `tox` | +| Repo-defined script | Whatever `scripts.test` in Makefile/tox/nox specifies | + +For **pytest** projects (the most common case), use the detected ``: + +| Scope | Command | +|-------|---------| +| All tests | ` pytest` | +| Specific file | ` pytest tests/test_module.py` | +| Specific test | ` pytest tests/test_module.py::TestClass::test_method` | +| Keyword filter | ` pytest -k "keyword"` | +| Stop on first failure | ` pytest -x --tb=short` | + +- Prefer `python -m pytest` over bare `pytest` to ensure the correct interpreter +- If the project uses `unittest` only (no pytest in deps), use `python -m unittest discover` +- If tests must run from a subdirectory, `Set-Location` there first and keep that working directory for verification + +## Frameworks Beyond Plain Pytest + +Mirror the existing tests' import style and invocation exactly. + +- **Django**: Prefer the repo's runner (`runtests.py`, `manage.py test`, or tox/make target). If the repo uses `pytest-django`, ensure `DJANGO_SETTINGS_MODULE` is set exactly as existing tests/config require. + - `python manage.py test app_label.tests.test_module` + - `$env:DJANGO_SETTINGS_MODULE='project.settings'; python -m pytest tests/app/test_module.py` +- **unittest-style suites**: Use `python -m unittest path.to.test_module` or the repo's discover command; do not force pytest unless existing tests already do. +- **Subdir runners**: Some repos expect commands from `tests/`, `test/units/`, or another subdir so relative imports and fixtures work. + +## Lint Command + +Use the repo's existing lint script first (`make lint`, `tox -e lint`). Otherwise detect tools from config: + +- `ruff.toml` or `[tool.ruff]` → ` ruff check --fix && ruff format` +- `[tool.black]` → ` black` +- `.flake8` → ` flake8` + +## Project Layout and Imports + +| Layout | Import Style | +|--------|-------------| +| `src/package/module.py` | `from package.module import X` | +| `package/module.py` at root | `from package.module import X` | +| `module.py` at root | `from module import X` | + +- **Match existing test imports exactly** — do not invent `src.` prefixes unless existing tests use them +- Place new tests where the existing suite lives so the same `conftest.py`, fixtures, helpers, and settings apply +- Check `pyproject.toml` `[tool.setuptools.package-dir]` for layout hints +- Default test placement: `tests/` mirroring source structure (`src/billing/service.py` → `tests/billing/test_service.py`) + +## Heavy or Native Dependencies + +Before writing tests for a target module, verify it imports cleanly in the same environment and working directory as tests. Run the probe under the **same env wrapper as the test command** (`poetry run`, `pdm run`, `uv run`, `pipenv run`, `hatch run`) so the check reflects the real test interpreter/venv — a bare `python` may resolve to a different environment and report a misleading `ok`: + +```powershell +# Wrap with the detected env tool, e.g. `poetry run python -c "..."` +python -c "import package.module; print('ok')" +python -c "from package import module; print('ok')" +``` + +If a heavy/native dependency such as NumPy, pandas, PyTorch, TensorFlow, cryptography, or a compiled extension cannot be imported or built in the environment: + +- Do not write tests that import the failing module +- Do not spend the budget fighting native build/import failures or installing unrelated packages +- Scope down to a pure-Python submodule that imports cleanly, or omit tests for that module rather than shipping ones that cannot run (see *Finalization: Green Suite or Remove*) + +## Test File Naming + +Match the repo's existing conventions. Common patterns: + +- **pytest**: Files `test_*.py` or `*_test.py`, functions `test_` prefix, classes `Test` prefix +- **Custom frameworks**: Use whatever format existing tests use (e.g. `.uts` for UTscapy, custom extensions) + +If writing new tests in a repo with no tests, default to pytest conventions. + +## Common Errors + +| Error | Fix | +|-------|-----| +| `ModuleNotFoundError: No module named 'src'` | Import from the package name used by the repo, not from `src` | +| `ModuleNotFoundError: No module named 'X'` | Check existing imports for the correct package name; if editable install needed: ` pip install -e .` | +| `ImportError: attempted relative import` | Convert to absolute imports matching existing test patterns | +| `fixture 'X' not found` | Check `conftest.py` for existing fixtures; reuse them instead of creating new ones | +| `TypeError: missing required argument` | Read the full `__init__`/function signature; pass all required parameters | +| `async def functions are not natively supported` | Use `@pytest.mark.asyncio` only if `pytest-asyncio` is already in deps; check for `asyncio_mode = "auto"` in config | +| `DJANGO_SETTINGS_MODULE is undefined` | Use the repo's Django runner or set the same settings module used by existing tests | +| `ImportError` from `torch`, `numpy`, or compiled extension | Avoid that module; choose a pure-Python target that imports cleanly | +| `SyntaxError` | Fix syntax at the indicated line | + +## Mocking Rules + +- Use `unittest.mock` (stdlib) — no extra dependency needed +- **Patch where the name is looked up**, not where it is defined: `@patch("mypackage.module.datetime")` not `@patch("datetime.datetime")` +- Use `Mock(spec=RealClass)` to catch attribute errors +- Use `AsyncMock` for async functions +- Prefer dependency injection over `@patch` +- If a test needs more than 3 mocks, flag it as a design smell + +## Dependency Installation (Last Resort) + +Only install packages after investigation confirms they are missing. Use the detected prefix: + +| Manager | Install command | +|---------|----------------| +| Poetry | `poetry add --group dev pytest` | +| PDM | `pdm add -dG test pytest` | +| uv | `uv add --dev pytest` | +| pip | `python -m pip install -e ".[dev]"` | + +Never run bare `pip install` in a Poetry/PDM/uv project — it bypasses the lockfile. + +## Finalization: Green Suite or Remove + +Before finishing, run the complete set of tests you added with the repo's native invocation, under the **same env wrapper as the repo's tests** (`poetry run`, `pdm run`, `uv run`, `pipenv run`, `hatch run`). Running the green-suite check in a different interpreter/venv can pass locally yet still fail under the repo's actual runner. + +```powershell +# Examples; choose the repo-native command/wrapper discovered above +poetry run python -m pytest tests/path/to/new_tests.py +uv run python -m unittest path.to.new_test_module +poetry run python manage.py test app_label.tests.test_module +``` + +If any new test fails or errors after a reasonable fix attempt, delete that test before finishing. Never leave skipped, xfailed, failing, or collection-error tests just to keep more lines. The final submitted suite must be green. + +## Skip Coverage Tools + +Do not configure or run coverage tools (coverage.py, pytest-cov). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/ruby-examples.md b/.github/skills/code-testing-extensions/extensions/ruby-examples.md new file mode 100644 index 0000000..b00c04a --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/ruby-examples.md @@ -0,0 +1,277 @@ +# Ruby Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Ruby codebase using RSpec. These show what each pipeline phase produces for a small gem-style project. + +> Minitest follows the same coverage shape. Replace `RSpec.describe` / `it` / `expect` with `Minitest::Test` methods and assertions, and run through `bundle exec rake test` or the repo's established Minitest command. + +## Source Under Test + +A simple `InvoiceService` in a Ruby gem: + +```text +Gemfile +lib/ + contoso_billing.rb + contoso_billing/invoice.rb + contoso_billing/invoice_repository.rb + contoso_billing/invoice_service.rb +spec/spec_helper.rb +``` + +```ruby +# lib/contoso_billing/invoice_service.rb +require 'bigdecimal' +require 'time' + +module ContosoBilling + class InvoiceService + def initialize(repository:, clock: -> { Time.now.utc }) + @repository = repository + @clock = clock + end + + def calculate_total(invoice) + raise ArgumentError, 'invoice must not be nil' if invoice.nil? + raise ArgumentError, 'Invoice has no line items.' if invoice.line_items.empty? + + subtotal = invoice.line_items.sum { |item| item.quantity * item.unit_price } + tax = subtotal * invoice.tax_rate + (subtotal + tax).round(2) + end + + def get_by_id(id) + invoice = @repository.find(id) + raise KeyError, "Invoice #{id} not found." if invoice.nil? + + invoice + end + + def mark_as_paid(id) + invoice = get_by_id(id) + raise StandardError, 'Invoice is already paid.' if invoice.status == :paid + + invoice.status = :paid + invoice.paid_at = @clock.call + @repository.update(invoice) + end + end +end +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: Ruby 3.3 (from `.ruby-version`) +- **Project Type**: Plain gem +- **Test Framework**: RSpec 3.x (detected in Gemfile.lock and `spec/spec_helper.rb`) +- **Run Prefix**: `bundle exec` required because Gemfile.lock is present + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 examples across 0 files + +## Build & Test Commands +- **Syntax**: `ruby -c lib/contoso_billing/invoice_service.rb` +- **Discovery**: `bundle exec rspec --dry-run` +- **Single file**: `bundle exec rspec spec/contoso_billing/invoice_service_spec.rb` +- **All specs**: `bundle exec rspec` + +## Files to Test + +### High Priority +| File | Classes/Methods | Testability | Notes | +|------|-----------------|-------------|-------| +| lib/contoso_billing/invoice_service.rb | InvoiceService: calculate_total, get_by_id, mark_as_paid | High | Repository dependency and clock are injected | + +## Testing Patterns +- Existing specs use `RSpec.describe`, `subject`, `let`, and `instance_double`. +- Recommend `instance_double('InvoiceRepository')` for the repository and a fixed clock lambda for time-dependent behavior. +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate RSpec tests for ContosoBilling::InvoiceService, covering calculation, +lookup, and paid-state transition behavior. + +## Commands +- **Syntax**: `ruby -c spec/contoso_billing/invoice_service_spec.rb` +- **Discovery**: `bundle exec rspec --dry-run` +- **Test**: `bundle exec rspec spec/contoso_billing/invoice_service_spec.rb` + +## Phase 1: InvoiceService + +### Files to Test +- **Source**: `lib/contoso_billing/invoice_service.rb` +- **Test File**: `spec/contoso_billing/invoice_service_spec.rb` + +**Methods to Test**: +1. `calculate_total` — tax, zero tax, rounding, nil, and empty line items +2. `get_by_id` — existing invoice and missing invoice +3. `mark_as_paid` — success with fixed clock, already-paid, missing +``` + +## Sample Generated Test File + +```ruby +# spec/contoso_billing/invoice_service_spec.rb +require 'spec_helper' +require 'bigdecimal' +require 'ostruct' +require 'time' +require 'contoso_billing/invoice_service' + +RSpec.describe ContosoBilling::InvoiceService do + subject(:service) { described_class.new(repository: repository, clock: clock) } + + let(:repository) { instance_double('InvoiceRepository') } + let(:fixed_time) { Time.utc(2025, 1, 1, 12, 0, 0) } + let(:clock) { -> { fixed_time } } + + def build_invoice(id: 1, status: :pending, tax_rate: BigDecimal('0.10'), line_items: [OpenStruct.new(quantity: 1, unit_price: BigDecimal('100.00'))]) + OpenStruct.new(id: id, status: status, tax_rate: tax_rate, line_items: line_items, paid_at: nil) + end + + describe '#calculate_total' do + it 'returns the total for a single item with tax' do + expect(service.calculate_total(build_invoice)).to eq(BigDecimal('110.00')) + end + + it 'returns the subtotal when tax is zero' do + invoice = build_invoice( + tax_rate: BigDecimal('0'), + line_items: [OpenStruct.new(quantity: 3, unit_price: BigDecimal('25.00'))] + ) + + expect(service.calculate_total(invoice)).to eq(BigDecimal('75.00')) + end + + it 'rounds to two decimals' do + invoice = build_invoice( + tax_rate: BigDecimal('0.07'), + line_items: [OpenStruct.new(quantity: 2, unit_price: BigDecimal('9.99'))] + ) + + expect(service.calculate_total(invoice)).to eq(BigDecimal('21.38')) + end + + it 'raises for a nil invoice' do + expect { service.calculate_total(nil) }.to raise_error(ArgumentError, /must not be nil/) + end + + it 'raises when there are no line items' do + expect { service.calculate_total(build_invoice(line_items: [])) }.to raise_error(ArgumentError, /no line items/) + end + end + + describe '#get_by_id' do + it 'returns an existing invoice' do + invoice = build_invoice(id: 42) + allow(repository).to receive(:find).with(42).and_return(invoice) + + expect(service.get_by_id(42)).to be(invoice) + end + + it 'raises KeyError for a missing invoice' do + allow(repository).to receive(:find).with(999).and_return(nil) + + expect { service.get_by_id(999) }.to raise_error(KeyError, /999/) + end + end + + describe '#mark_as_paid' do + it 'marks a pending invoice as paid and persists it' do + invoice = build_invoice(id: 1) + allow(repository).to receive(:find).with(1).and_return(invoice) + allow(repository).to receive(:update) + + service.mark_as_paid(1) + + expect(invoice.status).to eq(:paid) + expect(invoice.paid_at).to eq(fixed_time) + expect(repository).to have_received(:update).with(invoice) + end + + it 'raises and does not update an already-paid invoice' do + invoice = build_invoice(id: 1, status: :paid) + allow(repository).to receive(:find).with(1).and_return(invoice) + allow(repository).to receive(:update) + + expect { service.mark_as_paid(1) }.to raise_error(StandardError, /already paid/) + expect(repository).not_to have_received(:update) + end + end +end +``` + +## Sample Fix Cycle + +When the implementer encounters a load or mock issue, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +LoadError: cannot load such file -- contoso_billing/invoice_service +``` + +**Fixer diagnosis:** The generated spec omitted `require 'spec_helper'`, so the gem's load-path setup did not run. + +**Fix applied:** Add `require 'spec_helper'` as the first require and keep source requires consistent with existing specs. + +**Rerun:** `bundle exec rspec spec/contoso_billing/invoice_service_spec.rb` → SUCCESS + +--- + +**Another common cycle — verifying double mismatch:** + +**Test output:** + +```text +The InvoiceRepository class does not implement the instance method: find_by_id +``` + +**Fixer diagnosis:** `instance_double` caught a typo in the test setup. The production code calls `repository.find`, not `find_by_id`. + +**Fix applied:** Stub `find` with the expected id instead of `find_by_id`. + +**Rerun:** SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: contoso-billing (Ruby) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 9 | +| Tests passing | 9 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `spec/contoso_billing/invoice_service_spec.rb` (9 RSpec examples) + +### Coverage +- InvoiceService#calculate_total — 3 happy path, 2 error cases +- InvoiceService#get_by_id — found and missing branches +- InvoiceService#mark_as_paid — success and already-paid branches + +### Build / Test Validation +- Syntax: ✅ `ruby -c spec/contoso_billing/invoice_service_spec.rb` +- Discovery: ✅ `bundle exec rspec --dry-run` found the new examples +- Test run: ✅ `bundle exec rspec spec/contoso_billing/invoice_service_spec.rb` +``` diff --git a/.github/skills/code-testing-extensions/extensions/ruby.md b/.github/skills/code-testing-extensions/extensions/ruby.md new file mode 100644 index 0000000..f9a8483 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/ruby.md @@ -0,0 +1,261 @@ +# Ruby Extension + +Language-specific guidance for Ruby test generation. + +## Rule #0: Confirm the Test Target + +If the prompt does not name a specific file (e.g. "test the repository", "cover one core module", "comprehensive suite"), do **not** assume the largest or top-level upstream code is the intended target. In real workflows the user usually wants to test code they have just added, and large upstream repos contain hundreds of modules already covered by existing specs. + +Run these **read-only** discovery commands first — they are the deliberate exception to Rule #1's "before writing any test or running any command" rule, and their output is the ground truth Rule #1's reading is meant to interpret. Do **not** write or execute any tests until Rule #0 and Rule #1 are both complete. + +| Goal | Command | +|------|---------| +| List uncommitted edits + untracked files | `git status -s` | +| Untracked files only (typical for newly-added modules) | `git ls-files --others --exclude-standard` | +| Recently added files under `lib/` or `app/` | `git log --diff-filter=A --name-only -5 -- 'lib/**' 'app/**'` | +| Files referenced by `spec_helper.rb` / `rails_helper.rb` | `grep -nE "^\s*require(_relative)?\s" spec/spec_helper.rb spec/rails_helper.rb 2>/dev/null` | +| Modules with no matching spec | compare `lib/**/*.rb` against `spec/**/*_spec.rb` paths | + +Prefer targets that match **all** of: + +1. Untracked or recently added (`git status` / `git log --diff-filter=A`). +2. Small and pure (a few hundred lines, no I/O, no global state). +3. Located under a conventional source root (`lib/`, `app/models/`, `app/services/`). +4. Have **no** existing matching `*_spec.rb` / `*_test.rb`. + +If `spec/spec_helper.rb` already `require`s one specific file (e.g. `require "string_utils"`), that file is almost certainly the target — start there. + +### Test Placement Contract + +RSpec only discovers specs under `spec/` by default, and verification harnesses (CI, msbench, coverage tools) typically scope discovery to `spec/` alone. Place every spec there, mirroring the source layout: + +| Source | Spec | +|--------|------| +| `lib/string_utils.rb` | `spec/string_utils_spec.rb` | +| `lib/foo/bar.rb` | `spec/foo/bar_spec.rb` | +| `app/models/user.rb` (Rails) | `spec/models/user_spec.rb` | + +A spec placed anywhere outside `spec/` (e.g. next to the source under `lib/`) will be invisible to `bundle exec rspec` and to the harness. The same applies to Minitest: place tests under `test/` and use `*_test.rb` naming. + +**Gem-monorepo trap (fastlane, ruby/ruby, large gems with sub-gems):** if the repo contains multiple `*/spec/` directories (each sub-gem with its own specs), `bundle exec rspec` from the repo root only loads `./spec/` by default — sub-gem specs are invisible to the harness. Either: + +- place the new spec inside the **root** `./spec/` (with a `require_relative` to the sub-gem's `lib/`), or +- run the sub-gem's `bundle exec rspec` from the sub-gem dir AND verify in the Harness Discovery Check below that the root command also enumerates it (often it won't — you'll need to extend `.rspec` with `--default-path` or the root `Rakefile`'s test task). + +For interpreter-build repos (ruby/ruby itself) the test runner requires `make test-all` after `make miniruby` — `ruby test/foo_test.rb` alone is not what the harness runs. + +### First-Test Sanity Loop + +After writing the **first** spec — before writing any others: + +1. Run `bundle exec rspec --dry-run` and confirm the example count is `> 0`. If it is `0`, RSpec is not seeing your file; fix the location, filename, or `$LOAD_PATH` before continuing. +2. Run the spec (`bundle exec rspec spec/.rb`); fix `LoadError`, missing `require`, or constant errors before adding more tests. +3. Only then expand to cover the remaining methods. + +This catches placement and load-path mistakes on turn 1 instead of after dozens of failed-test iterations. + +### Harness Discovery Check + +Before reporting success, run the **harness-equivalent** discovery command from the repo root and confirm the example count went up by at least the number of tests you generated. CI/msbench/coverage harnesses do not know which file or sub-gem dir you targeted; they run the framework's default discovery from the repo root, so a spec that passes via `bundle exec rspec fastlane_core/spec/foo_spec.rb` is still worthless if `bundle exec rspec --dry-run` from the repo root doesn't enumerate it. + +```bash +# RSpec — from repo root +bundle exec rspec --dry-run 2>&1 | grep -E '^[0-9]+ example' + +# Minitest (Rails) +{ bundle exec rake test --dry-run 2>/dev/null || bin/rails test --list-tests; } | wc -l + +# Custom runner (Homebrew, ruby/ruby, etc.) +# Use the repo's own runner — `./bin/brew tests --list`, `make test-all`, etc. +# If no `--list`/`--dry-run` mode exists, run a single matching test by name and confirm exit 0. +``` + +If the count did not increase, your spec is invisible to the harness. Move it into `./spec/`, extend `.rspec`/`Rakefile` so the harness picks up the sub-gem dir, or switch to a `require_relative` strategy from a root-level spec. Do **not** report success until the harness-equivalent command sees your new tests. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `spec/**/*_spec.rb` (RSpec) or `test/**/*_test.rb` (Minitest) and copy their style (matchers, helpers, factories, contexts) +2. **`Gemfile` / `Gemfile.lock`** — Ruby version, test framework, supporting gems (`rspec`, `minitest`, `factory_bot`, `webmock`, `vcr`, `rails`) +3. **`.ruby-version`** / `.tool-versions` — pinned Ruby version +4. **Test helpers** — `spec/spec_helper.rb`, `spec/rails_helper.rb`, `test/test_helper.rb` — these dictate the load path, requires, and global config +5. **Rake tasks** — `Rakefile` may define a `default` task that runs the full test suite + +Use the framework the repo already uses. Do not introduce RSpec into a Minitest project (or vice versa). + +## Toolchain Detection + +| Indicator | Manager | Run prefix | +|-----------|---------|------------| +| `Gemfile.lock` | Bundler | `bundle exec ` | +| `.ruby-version` + `rbenv` | rbenv | combine with `bundle exec` | +| `mise.toml` / `asdf` `.tool-versions` | mise/asdf | the wrapper handles version selection; still use `bundle exec` | +| Plain Ruby, no Bundler | system Ruby | `ruby ` (rare in real projects) | + +Always run inside `bundle exec` if a `Gemfile.lock` is present — otherwise you may pick up a system gem version that disagrees with the lockfile. + +## Build Commands + +Ruby is interpreted — there is no compile step. The closest validations: + +| Scope | Command | +|-------|---------| +| Syntax check | `ruby -c path/to/file.rb` | +| Lint (RuboCop) | `bundle exec rubocop path/to/file.rb` | +| Type check (Sorbet) | `bundle exec srb tc` (only if `sorbet/` dir exists) | +| Type check (RBS/Steep) | `bundle exec steep check` | + +For Rails: load all classes once with `bundle exec rails zeitwerk:check` to catch missing constants before running tests. + +## Test Commands + +### RSpec + +| Scope | Command | +|-------|---------| +| All specs | `bundle exec rspec` | +| Single file | `bundle exec rspec spec/models/widget_spec.rb` | +| Single line | `bundle exec rspec spec/models/widget_spec.rb:42` | +| By name | `bundle exec rspec -e "creates a widget"` | +| Tagged | `bundle exec rspec --tag focus` | +| Fail fast | `bundle exec rspec --fail-fast` | +| Documentation format | `bundle exec rspec --format documentation` | + +### Minitest + +| Scope | Command | +|-------|---------| +| All tests | `bundle exec rake test` (Rails) or `bundle exec ruby -Ilib -Itest -e 'Dir.glob("./test/**/*_test.rb").each { |f| require f }'` | +| Single file | `bundle exec ruby -Itest test/models/widget_test.rb` | +| Single test | `bundle exec ruby -Itest test/models/widget_test.rb -n test_creates_widget` | +| By name pattern | `... -n /pattern/` | + +### Rails (any framework) + +| Scope | Command | +|-------|---------| +| Default suite | `bin/rails test` (Minitest) or `bundle exec rspec` | +| Single Rails test file | `bin/rails test test/models/widget_test.rb:42` | +| System tests | `bin/rails test:system` | + +Always prefer the wrapper script (`bin/rails`, `bin/rspec`) when present — they enforce the project's loader/setup. + +## Lint Command + +- `bundle exec rubocop` — autocorrect with `bundle exec rubocop -A` (only if existing tests already conform; do not autocorrect unrelated files) +- `bundle exec standardrb --fix` if `standard` is in the Gemfile +- Some Rails projects add `rubocop-rails`, `rubocop-rspec`, `rubocop-performance` — they enforce extra rules + +## Project Layout and Loading + +| Layout | Test placement | +|--------|----------------| +| Plain gem (RSpec) | `spec/` mirrors `lib/` (e.g. `lib/foo/bar.rb` → `spec/foo/bar_spec.rb`) | +| Plain gem (Minitest) | `test/` mirrors `lib/` (e.g. `test/foo/bar_test.rb`) | +| Rails (RSpec) | `spec/models`, `spec/controllers`, `spec/requests`, `spec/system`, etc. | +| Rails (Minitest) | `test/models`, `test/controllers`, `test/integration`, `test/system` | + +**Loading source code:** + +- RSpec: `spec/spec_helper.rb` typically does `require 'my_gem'` or sets `$LOAD_PATH`. Match its pattern in new specs by `require 'spec_helper'` (or `require 'rails_helper'` in Rails) +- Minitest: each `_test.rb` typically `require 'test_helper'` +- Rails uses Zeitwerk autoloading — do **not** add `require_relative '../../app/models/widget'`; just `require 'rails_helper'` and reference the constant + +## Test File Naming + +| Framework | File suffix | Class/example | +|-----------|-------------|---------------| +| RSpec | `_spec.rb` | `RSpec.describe Widget do ... end`, `it "..." do ... end` | +| Minitest (classic) | `_test.rb` | `class WidgetTest < Minitest::Test`, methods `def test_...` | +| Minitest (spec) | `_test.rb` | `describe Widget do ... it "..." do ... end end` | +| Rails Minitest | `_test.rb` | `class WidgetTest < ActiveSupport::TestCase` | + +## RSpec Template + +```ruby +require 'spec_helper' +require 'calculator' + +RSpec.describe Calculator do + subject(:calculator) { described_class.new } + + describe '#add' do + it 'returns the sum of two positive numbers' do + expect(calculator.add(2, 3)).to eq(5) + end + + context 'with negative numbers' do + it 'returns the correct sum' do + expect(calculator.add(-1, 1)).to eq(0) + end + end + + it 'raises when given non-numeric input' do + expect { calculator.add('a', 1) }.to raise_error(TypeError) + end + end +end +``` + +## Common Errors + +| Error | Fix | +|-------|-----| +| `LoadError: cannot load such file -- foo` | Missing `require` or load path; check `spec_helper.rb` for the established pattern instead of patching `$LOAD_PATH` ad hoc | +| `NameError: uninitialized constant X` | Constant isn't loaded — in Rails, ensure you require `rails_helper`; in plain Ruby, add the appropriate `require` | +| `ArgumentError: wrong number of arguments (given X, expected Y)` | Read the method signature; pass keyword vs positional args correctly | +| `NoMethodError: undefined method 'foo' for nil:NilClass` | Test setup left a value `nil`; check `let`/`before` ordering and factory data | +| `Failure/Error: ... received :foo with unexpected arguments` (RSpec) | Tighten the matcher: `with(hash_including(...))` or relax to `with(any_args)` deliberately | +| `expected #<...> to receive :foo (1 time) but received it 0 times` | Either the code path didn't call the stub, or you stubbed the wrong receiver | +| `DEPRECATION WARNING` (Rails) | Address the deprecation rather than silencing it; tests that warn today break tomorrow | +| `ActiveRecord::PendingMigrationError` | Run `bin/rails db:migrate RAILS_ENV=test` before tests | +| `Mysql2::Error / PG::ConnectionBad` in CI | Tests need a database — check `config/database.yml` and CI service containers | +| `Capybara::ElementNotFound` (system tests) | Use `find` with explicit waits; do not add `sleep` | + +## Mocking Rules (RSpec) + +- Use `instance_double(Klass)` and `class_double(Klass)` — they verify that the method actually exists, unlike `double` +- `allow(obj).to receive(:method).and_return(value)` for stubs; `expect(obj).to receive(:method)` for interaction expectations +- Prefer `instance_double` over plain `double`; prefer dependency injection over `allow_any_instance_of` +- Use `let` for memoized helpers; use `let!` only when the side effect must run before each example +- Avoid global state mutation in tests — wrap in `around` blocks or use `ClimateControl` for env vars +- For HTTP, use `webmock` (`stub_request(:get, ...)`) or `vcr` cassettes if the project already uses them +- If a test needs more than 3 mocks, flag it as a design smell + +## Mocking Rules (Minitest) + +- Use `Minitest::Mock` for simple cases: `mock = Minitest::Mock.new; mock.expect(:method, return_value, [arg])` +- For richer mocking, projects commonly add `mocha`: `obj.expects(:method).returns(value)` (in `test_helper.rb`: `require 'mocha/minitest'`) +- Always verify mocks at end of test (`mock.verify` for `Minitest::Mock`); Mocha verifies automatically + +## Rails Specifics + +- Use the **smallest** spec type that covers the behavior: model spec for pure logic, request spec for HTTP, system spec only when JS/UI matters +- `rails-controller-testing` gem must be present for `assigns(:foo)` and `assert_template` +- `ActiveJob::TestHelper` and `ActiveSupport::Testing::TimeHelpers` (`travel_to`) come with Rails — use them instead of `Timecop` if Rails ≥ 5 +- Use fixtures only if the project already uses them; `factory_bot` is more common in modern Rails apps +- Database transactions wrap each test by default — for system tests with browser drivers, use `DatabaseCleaner` strategies the project already configures + +## Dependency Installation (Last Resort) + +Only add gems after investigation confirms they are missing. Edit `Gemfile`: + +```ruby +group :test do + gem 'rspec' + gem 'webmock' +end +``` + +Then run: + +``` +bundle install +``` + +Never `gem install` outside Bundler — it bypasses the lockfile and changes the global Ruby environment. + +## Skip Coverage Tools + +Do not configure or run coverage tools (SimpleCov). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/rust-examples.md b/.github/skills/code-testing-extensions/extensions/rust-examples.md new file mode 100644 index 0000000..e34e297 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/rust-examples.md @@ -0,0 +1,327 @@ +# Rust Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a Rust crate using the built-in test harness. These show what each pipeline phase produces for a small library crate. + +## Source Under Test + +A simple `InvoiceService` in a Rust crate: + +```text +Cargo.toml +src/ + lib.rs + invoice.rs + invoice_repository.rs + invoice_service.rs +``` + +```rust +// src/invoice_service.rs +use crate::invoice::{Invoice, InvoiceStatus}; +use crate::invoice_repository::InvoiceRepository; +use std::time::SystemTime; + +#[derive(Debug, PartialEq, Eq)] +pub enum InvoiceError { + EmptyLineItems, + NotFound(i32), + AlreadyPaid, + Repository(String), +} + +pub struct InvoiceService +where + R: InvoiceRepository, + C: Fn() -> SystemTime, +{ + repository: R, + clock: C, +} + +impl InvoiceService +where + R: InvoiceRepository, + C: Fn() -> SystemTime, +{ + pub fn new(repository: R, clock: C) -> Self { + Self { repository, clock } + } + + pub fn calculate_total(&self, invoice: &Invoice) -> Result { + if invoice.line_items.is_empty() { + return Err(InvoiceError::EmptyLineItems); + } + + let subtotal: f64 = invoice.line_items.iter().map(|item| item.quantity as f64 * item.unit_price).sum(); + Ok(((subtotal + subtotal * invoice.tax_rate) * 100.0).round() / 100.0) + } + + pub fn get_by_id(&self, id: i32) -> Result { + self.repository.find(id).map_err(InvoiceError::Repository)?.ok_or(InvoiceError::NotFound(id)) + } + + pub fn mark_as_paid(&mut self, id: i32) -> Result<(), InvoiceError> { + let mut invoice = self.get_by_id(id)?; + if invoice.status == InvoiceStatus::Paid { + return Err(InvoiceError::AlreadyPaid); + } + + invoice.status = InvoiceStatus::Paid; + invoice.paid_at = Some((self.clock)()); + self.repository.update(invoice).map_err(InvoiceError::Repository) + } +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: Rust 1.78 (edition 2021, from Cargo.toml) +- **Crate Type**: library +- **Test Framework**: built-in Rust test harness (`#[test]`), no `mockall`/`rstest` dev-dependencies detected + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Check**: `cargo check --all-targets` +- **Compile tests**: `cargo test --no-run` +- **Test**: `cargo test` +- **Single module**: `cargo test invoice_service::tests` + +## Files to Test + +### High Priority +| File | Types/Methods | Testability | Notes | +|------|---------------|-------------|-------| +| src/invoice_service.rs | InvoiceService: calculate_total, get_by_id, mark_as_paid | High | Generic repository trait is easy to fake; clock closure is injectable | + +## Testing Patterns +- No existing patterns; recommend unit tests in `#[cfg(test)] mod tests` at the bottom of `invoice_service.rs` and a hand-written fake repository. +``` + +## Sample Plan Output + +```markdown +# Test Implementation Plan + +## Overview +Generate built-in Rust unit tests for InvoiceService covering calculation, +lookup, and paid-state transition behavior. + +## Commands +- **Check**: `cargo check --all-targets` +- **Compile tests**: `cargo test --no-run` +- **Test**: `cargo test invoice_service::tests` + +## Phase 1: InvoiceService + +### Files to Test +- **Source**: `src/invoice_service.rs` +- **Test Location**: `#[cfg(test)] mod tests` appended to `src/invoice_service.rs` + +**Methods to Test**: +1. `calculate_total` — tax, zero tax, rounding, empty-line-items error +2. `get_by_id` — existing invoice, missing invoice, repository error +3. `mark_as_paid` — success, already-paid, missing invoice +``` + +## Sample Generated Test File + +```rust +// Appended to src/invoice_service.rs +#[cfg(test)] +mod tests { + use super::*; + use crate::invoice::{Invoice, InvoiceStatus, LineItem}; + use std::collections::HashMap; + use std::time::{Duration, UNIX_EPOCH}; + + #[derive(Default)] + struct FakeRepository { + invoices: HashMap, + updated: Option, + find_error: Option, + } + + impl InvoiceRepository for FakeRepository { + fn find(&self, id: i32) -> Result, String> { + if let Some(error) = &self.find_error { + return Err(error.clone()); + } + + Ok(self.invoices.get(&id).cloned()) + } + + fn update(&mut self, invoice: Invoice) -> Result<(), String> { + self.updated = Some(invoice.clone()); + self.invoices.insert(invoice.id, invoice); + Ok(()) + } + } + + fn make_invoice(id: i32) -> Invoice { + Invoice { + id, + status: InvoiceStatus::Pending, + tax_rate: 0.10, + line_items: vec![LineItem { quantity: 1, unit_price: 100.0 }], + paid_at: None, + } + } + + fn service_with(repository: FakeRepository) -> InvoiceService std::time::SystemTime> { + InvoiceService::new(repository, || UNIX_EPOCH + Duration::from_secs(123)) + } + + #[test] + fn calculate_total_valid_line_items_returns_expected_total() { + let cases = [ + ("single item with tax", vec![LineItem { quantity: 1, unit_price: 100.0 }], 0.10, 110.0), + ("multi quantity zero tax", vec![LineItem { quantity: 3, unit_price: 25.0 }], 0.0, 75.0), + ("rounds to two decimals", vec![LineItem { quantity: 2, unit_price: 9.99 }], 0.07, 21.38), + ]; + let service = service_with(FakeRepository::default()); + + for (name, line_items, tax_rate, expected) in cases { + let mut invoice = make_invoice(1); + invoice.line_items = line_items; + invoice.tax_rate = tax_rate; + + let total = service.calculate_total(&invoice).unwrap_or_else(|err| panic!("{name}: unexpected error: {err:?}")); + + assert!((total - expected).abs() < 0.001, "{name}: got {total}, expected {expected}"); + } + } + + #[test] + fn calculate_total_empty_line_items_returns_error() { + let service = service_with(FakeRepository::default()); + let mut invoice = make_invoice(1); + invoice.line_items.clear(); + + assert_eq!(Err(InvoiceError::EmptyLineItems), service.calculate_total(&invoice)); + } + + #[test] + fn get_by_id_existing_invoice_returns_invoice() { + let mut repository = FakeRepository::default(); + repository.invoices.insert(42, make_invoice(42)); + let service = service_with(repository); + + let invoice = service.get_by_id(42).expect("invoice should exist"); + + assert_eq!(42, invoice.id); + } + + #[test] + fn get_by_id_missing_invoice_returns_not_found() { + let service = service_with(FakeRepository::default()); + + assert_eq!(Err(InvoiceError::NotFound(999)), service.get_by_id(999)); + } + + #[test] + fn get_by_id_repository_error_is_preserved() { + let repository = FakeRepository { find_error: Some("boom".to_owned()), ..FakeRepository::default() }; + let service = service_with(repository); + + assert_eq!(Err(InvoiceError::Repository("boom".to_owned())), service.get_by_id(1)); + } + + #[test] + fn mark_as_paid_pending_invoice_updates_status_date_and_repository() { + let mut repository = FakeRepository::default(); + repository.invoices.insert(1, make_invoice(1)); + let mut service = service_with(repository); + + service.mark_as_paid(1).expect("mark_as_paid should succeed"); + + let updated = service.repository.updated.as_ref().expect("repository should be updated"); + assert_eq!(InvoiceStatus::Paid, updated.status); + assert_eq!(Some(UNIX_EPOCH + Duration::from_secs(123)), updated.paid_at); + } + + #[test] + fn mark_as_paid_already_paid_returns_error_without_update() { + let mut invoice = make_invoice(1); + invoice.status = InvoiceStatus::Paid; + let mut repository = FakeRepository::default(); + repository.invoices.insert(1, invoice); + let mut service = service_with(repository); + + assert_eq!(Err(InvoiceError::AlreadyPaid), service.mark_as_paid(1)); + assert!(service.repository.updated.is_none()); + } +} +``` + +## Sample Fix Cycle + +When the implementer hits a compiler or test-runner issue, the fixer agent diagnoses and resolves it. + +**Build output:** + +```text +error[E0596]: cannot borrow `self.repository` as mutable, as it is behind a `&` reference +``` + +**Fixer diagnosis:** `mark_as_paid` calls `repository.update(...)`, which requires mutable repository access. The production method must take `&mut self`, and tests must bind the service as `let mut service`. + +**Fix applied:** Change the method receiver to `&mut self` and update tests to use mutable bindings for `mark_as_paid` cases. + +**Rebuild + rerun:** `cargo test --no-run && cargo test invoice_service::tests` → SUCCESS + +--- + +**Another common cycle — integration test imports:** + +**Build output:** + +```text +error[E0432]: unresolved import `crate::invoice_service` +``` + +**Fixer diagnosis:** The test was created under `tests/invoice_service.rs`, which is an integration test crate. Integration tests import the library by crate name, not `crate::`. + +**Fix applied:** Move the tests into `#[cfg(test)] mod tests` in `src/invoice_service.rs` and use `use super::*;`. + +**Rerun:** SUCCESS + +## Sample Final Report + +```markdown +## Test Generation Report + +**Project**: contoso-billing (Rust) +**Strategy**: Direct (single module in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 7 | +| Tests passing | 7 | +| Tests failing | 0 | +| Files created | 0 (tests appended to source module) | + +### Files Modified +- `src/invoice_service.rs` (7 unit tests in `#[cfg(test)] mod tests`) + +### Coverage +- InvoiceService::calculate_total — 3 happy path, 1 error case +- InvoiceService::get_by_id — found, missing, repository error +- InvoiceService::mark_as_paid — success and already-paid branches + +### Build / Test Validation +- Check: ✅ `cargo check --all-targets` +- Compile tests: ✅ `cargo test --no-run` +- Test run: ✅ `cargo test invoice_service::tests` +``` diff --git a/.github/skills/code-testing-extensions/extensions/rust.md b/.github/skills/code-testing-extensions/extensions/rust.md new file mode 100644 index 0000000..b383435 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/rust.md @@ -0,0 +1,180 @@ +# Rust Extension + +Language-specific guidance for Rust test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — look at `#[cfg(test)] mod tests` blocks inside `src/`, integration tests in `tests/`, doc tests in source comments, and any `examples/` that double as smoke tests +2. **`Cargo.toml`** — workspace layout (`[workspace]`), edition, `dev-dependencies`, feature flags, `[[bench]]` / `[[test]]` declarations +3. **`Cargo.lock`** — if checked in, you must not break it without intent +4. **Toolchain** — `rust-toolchain.toml` pins the channel (stable / nightly / specific version) +5. **`build.rs`** — custom build scripts may set `cfg` flags or generate code that tests rely on + +Match the repo's existing conventions — assertion macros, mock approach, feature-gating — exactly. Do not introduce `tokio::test` if the repo uses `async-std`, etc. + +## Toolchain Detection + +| Indicator | Meaning | +|-----------|---------| +| `rust-toolchain.toml` with `channel = "..."` | Use rustup to install/select that channel — `rustup show active-toolchain` | +| `rust-version = "1.x"` in `Cargo.toml` | Minimum supported Rust version (MSRV); do not use newer language features | +| `[workspace]` in root `Cargo.toml` | Multi-crate workspace; commands accept `-p ` to target one member | +| `nightly` channel | Tests may use `#![feature(...)]` flags; do not remove them | + +## Build Commands + +| Scope | Command | +|-------|---------| +| Type-check fast | `cargo check` | +| Type-check whole workspace | `cargo check --workspace --all-targets` | +| Build (debug) | `cargo build` | +| Build with all features | `cargo build --all-features` | +| Build a single crate | `cargo build -p crate-name` | +| Build tests without running | `cargo test --no-run` | + +`cargo check` is far faster than `cargo build` and catches almost the same errors. Prefer it during the fix loop; use `cargo build --tests` (or `cargo test --no-run`) before declaring tests compilable. + +## Test Commands + +| Scope | Command | +|-------|---------| +| All tests | `cargo test` | +| Workspace | `cargo test --workspace` | +| Single crate | `cargo test -p crate-name` | +| Filter by name | `cargo test substring_of_test_name` | +| Exact name | `cargo test -- --exact path::to::test_fn` | +| Single integration file | `cargo test --test file_stem` (no `.rs`) | +| Doc tests only | `cargo test --doc` | +| Show stdout | `cargo test -- --nocapture` | +| Single-threaded | `cargo test -- --test-threads=1` | +| Ignored tests | `cargo test -- --ignored` | +| With features | `cargo test --features "feat1 feat2"` | +| All features | `cargo test --all-features` | + +- Arguments before `--` are for cargo; arguments after `--` go to the test binary +- `cargo test foo` runs every test with `foo` in its full path (`module::tests::foo_does_a_thing`) — to avoid surprise matches use `--exact` +- `cargo nextest run` is significantly faster if the repo already uses it (`Cargo.toml` `[profile.nextest...]` or `.config/nextest.toml`) — match the repo's choice + +## Lint Command + +Use the repo's lint script first. Otherwise: + +- `cargo fmt --all -- --check` (CI), `cargo fmt` (apply) +- `cargo clippy --all-targets --all-features -- -D warnings` +- If `clippy.toml` / `rustfmt.toml` exists, the project has opinions — never override them in your tests + +## Project Layout + +``` +my_crate/ +├── Cargo.toml +├── src/ +│ ├── lib.rs # library crate root +│ ├── main.rs # binary crate root (mutually OK with lib.rs) +│ └── module.rs # private/public module +├── tests/ # integration tests — each .rs is a separate crate +│ └── widget.rs +├── benches/ # cargo bench targets +└── examples/ # cargo run --example name +``` + +| Test type | Where | Sees | +|-----------|-------|------| +| Unit test | `#[cfg(test)] mod tests` inside the source file | Private items in the surrounding module | +| Integration test | `tests/.rs` | Only the public API of the crate | +| Doc test | `///` doctests in source comments | Only the public API; runs via `cargo test --doc` | + +- **Unit tests** at the bottom of `module.rs`: + + ```rust + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn name_scenario_expected() { + // ... + } + } + ``` + +- **Integration tests** import the crate by name: `use my_crate::PublicType;` +- Helpers shared between integration tests must live in `tests/common/mod.rs` (the `mod.rs` form prevents cargo from treating them as a top-level test crate) + +## Test Function Patterns + +| Kind | Attribute | +|------|-----------| +| Sync test | `#[test]` | +| Should panic | `#[test] #[should_panic(expected = "message substring")]` | +| Ignored (long/manual) | `#[test] #[ignore = "reason"]` | +| Async test (Tokio) | `#[tokio::test]` (or `#[tokio::test(flavor = "multi_thread")]`) | +| Async test (async-std) | `#[async_std::test]` | +| Returning `Result` | `fn name() -> Result<(), Box>` — use `?` instead of `.unwrap()` | + +Pick the async harness the repo already uses. Do not mix `tokio` and `async-std` in tests. + +## Common Errors + +| Error | Fix | +|-------|-----| +| `cannot find type X in this scope` | Add `use crate::module::X;` or `use super::*;` inside the test module | +| `function or associated item not found in 'X'` | Verify the method exists on the exact type; check trait imports (e.g. `use std::io::Read`) | +| `the trait bound 'X: Y' is not satisfied` | Either implement the trait, add a `where` bound, or change the test to use a type that already implements it | +| `borrow of moved value` | Add `.clone()`, borrow with `&`, or restructure ownership — do not use `mem::transmute` to dodge it | +| `cannot borrow as mutable` | Make the binding `let mut x` or restructure to avoid simultaneous mutable + immutable borrows | +| `lifetime may not live long enough` | Add explicit lifetime annotations or use owned types (`String` instead of `&str`) in the test | +| `mismatched types` between `i32` and `usize` | Use `as` casts deliberately or change the literal type with a suffix (`5usize`, `5u32`) | +| `unresolved import 'crate::...'` in `tests/foo.rs` | Integration tests must import via the **crate name** (as listed in `Cargo.toml`), not `crate::` | +| `error: no test target found` for `cargo test --test foo` | The file must live directly in `tests/`, not `tests/subdir/foo.rs` (subdirs are treated as helpers) | +| `attempt to subtract with overflow` (debug) | Underflow on unsigned types; use `checked_sub`/`saturating_sub` or compare before subtracting | +| Doctest fails to compile | Use a leading "# " on hidden setup lines; mark code blocks `ignore`/`no_run`/`should_panic` if needed | +| `the following imports are unused` (warning treated as error) | Remove unused `use` statements; do not silence with `#[allow(unused_imports)]` | + +## Mocking Rules + +Rust has no single dominant mocking framework. Match the repo: + +- **Trait + struct fakes** (most idiomatic): define a trait, pass `Arc` or generic `T: Trait`, implement a fake struct in tests +- **`mockall`** crate: `#[automock]` on a trait generates `MockTrait` for use in tests +- **`mockito`** / **`wiremock`**: HTTP server mocks for client tests +- **`tempfile`**: scoped temp directories that auto-clean (`tempfile::tempdir()`) + +Avoid `unsafe` patches to "mock" free functions. Refactor to inject a trait instead. If a test needs more than 3 mocks, flag it as a design smell. + +## Features and `cfg` + +- Tests behind a feature flag run only when that feature is enabled — use `#[cfg(feature = "foo")]` on the `mod tests` or individual `#[test]` functions +- `--all-features` exercises everything but may pull conflicting features in some workspaces; check `cargo test --all-features` is part of CI before relying on it +- Use `#[cfg(test)]` to gate test-only helpers in production source files — not `#[cfg(feature = "test")]` + +## Concurrency, IO, and `unsafe` + +- Tests run in parallel by default. If your tests share global state (env vars, current dir, statics), serialize them with the `serial_test` crate (if present) or move state into the test +- Never write to `/tmp` or the repo dir directly — use `tempfile::tempdir()` so cleanup is automatic +- Tests in `unsafe` code should also run under Miri (`cargo +nightly miri test`) if the repo's CI does + +## Dependency Installation (Last Resort) + +Only add dependencies after investigation confirms they are missing: + +```toml +[dev-dependencies] +mockall = "0.12" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +``` + +Or via cargo: + +``` +cargo add --dev mockall +cargo add --dev tokio --features macros,rt-multi-thread +``` + +Match the major version of any tokio/serde/etc. already pinned by the workspace. + +## Skip Coverage Tools + +Do not configure or run coverage tools (`cargo tarpaulin`, `cargo llvm-cov`, `grcov`). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/swift.md b/.github/skills/code-testing-extensions/extensions/swift.md new file mode 100644 index 0000000..e4bf7b9 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/swift.md @@ -0,0 +1,227 @@ +# Swift Extension + +Language-specific guidance for Swift test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find files in `Tests/` (SPM) or `*Tests/` groups (Xcode) and copy their style. Distinguish **XCTest** (`import XCTest`, classes inheriting `XCTestCase`) from **Swift Testing** (`import Testing`, free functions tagged `@Test`) +2. **Project file** — `Package.swift` (SPM), `*.xcodeproj`, `*.xcworkspace`, or `Project.swift` (Tuist) +3. **Swift toolchain** — `.swift-version`, `swift-tools-version` line in `Package.swift`, `IPHONEOS_DEPLOYMENT_TARGET` and `SWIFT_VERSION` build settings in Xcode +4. **CI scripts** — `.github/workflows/*.yml`, `Fastfile`, `Makefile` — these reveal the canonical build/test invocation + +Use the testing framework the repo already uses. Both XCTest and Swift Testing can coexist in one target — match what the file you're adding tests next to uses. + +## Project Type Detection + +| Indicator | Project type | Build tool | +|-----------|--------------|------------| +| `Package.swift` only | Swift Package Manager | `swift build` / `swift test` | +| `*.xcodeproj` or `*.xcworkspace` | Xcode project (often app/iOS) | `xcodebuild` | +| Both | SPM library + Xcode app shell | Use SPM for library targets, Xcode for app targets | +| `Project.swift` (Tuist) | Tuist-generated Xcode project | Run `tuist generate` first, then xcodebuild | +| `project.yml` (XcodeGen) | XcodeGen-generated project | Run `xcodegen generate` first | + +If both an `.xcodeproj` and `.xcworkspace` exist (e.g. CocoaPods), **always pass `-workspace` not `-project`** to xcodebuild. + +## Build Commands + +### Swift Package Manager + +| Scope | Command | +|-------|---------| +| Build all | `swift build` | +| Build a target | `swift build --target MyLibrary` | +| Build for release | `swift build -c release` | + +### Xcode (`xcodebuild`) + +``` +xcodebuild build \ + -workspace MyApp.xcworkspace \ + -scheme MyAppScheme \ + -destination 'platform=iOS Simulator,name=iPhone 15' \ + -configuration Debug +``` + +- Always specify `-destination` for iOS/tvOS/watchOS — the default may not exist on the build machine +- Use `-quiet` to suppress xcodebuild's chatty output, and pipe to `xcbeautify`/`xcpretty` if installed +- For deterministic CI builds add `-derivedDataPath ./DerivedData` + +## Test Commands + +### Swift Package Manager + +| Scope | Command | +|-------|---------| +| All tests | `swift test` | +| Filter by test name (XCTest) | `swift test --filter MyClassTests/testFooBar` | +| Filter by test name (Swift Testing) | `swift test --filter MyTestSuite.fooBar` | +| Parallel | `swift test --parallel` | +| Single platform | `swift test --triple x86_64-apple-macosx` (rare; usually skip) | + +### Xcode + +``` +xcodebuild test \ + -workspace MyApp.xcworkspace \ + -scheme MyAppScheme \ + -destination 'platform=iOS Simulator,name=iPhone 15' \ + -only-testing:MyAppTests/MyClassTests/testFooBar +``` + +- `-only-testing:` and `-skip-testing:` accept `Bundle/Class/Method` paths and may be repeated +- `xcodebuild test-without-building` skips compilation if you've already built +- For Swift Testing in Xcode 16+, use the same `-only-testing:` syntax — the runner handles both frameworks + +## Lint Command + +Use the repo's lint tooling first: + +- `swiftlint lint --quiet` (autocorrect: `swiftlint --fix`) when `.swiftlint.yml` is present +- `swiftformat .` when `.swiftformat` is present +- Some projects gate format on a build phase — running `xcodebuild` may already invoke it + +## Project Layout + +### SPM + +``` +Package.swift +Sources/ +└── MyLibrary/ + ├── Foo.swift + └── Bar.swift +Tests/ +└── MyLibraryTests/ + └── FooTests.swift +``` + +- Test target name conventionally is `Tests` and lives in `Tests/Tests/` +- Test target must list its production target as a dependency in `Package.swift`: + + ```swift + .testTarget( + name: "MyLibraryTests", + dependencies: ["MyLibrary"]), + ``` + +### Xcode + +- Tests live in a separate target (e.g. `MyAppTests`) added to the scheme's "Test" action +- The test target's "Host Application" determines whether tests run on the simulator with the app loaded (unit tests) or as a UI test runner + +## Imports + +- XCTest: `import XCTest` plus `@testable import MyLibrary` to access `internal` symbols +- Swift Testing: `import Testing` plus `@testable import MyLibrary` +- `@testable` works only when the production target is built with `-enable-testing` (the SPM test target and Xcode "Debug" config do this by default) +- Never mark production code `public` solely to make it visible to tests — use `@testable import` instead + +## Test File Templates + +### Swift Testing (Xcode 16 / Swift 6) + +```swift +import Testing +@testable import MyLibrary + +@Suite("Calculator") +struct CalculatorTests { + @Test("add returns the sum of two integers") + func addReturnsSum() { + let calc = Calculator() + #expect(calc.add(2, 3) == 5) + } + + @Test("add throws on overflow", arguments: [ + (Int.max, 1), + (Int.min, -1), + ]) + func addThrowsOnOverflow(a: Int, b: Int) { + #expect(throws: ArithmeticError.self) { + try Calculator().add(a, b) + } + } +} +``` + +### XCTest + +```swift +import XCTest +@testable import MyLibrary + +final class CalculatorTests: XCTestCase { + func testAddReturnsSum() { + let calc = Calculator() + XCTAssertEqual(calc.add(2, 3), 5) + } + + func testAddThrowsOnOverflow() { + XCTAssertThrowsError(try Calculator().add(.max, 1)) { error in + XCTAssertEqual(error as? ArithmeticError, .overflow) + } + } +} +``` + +- XCTest requires test methods to start with `test` and take no arguments +- Mark XCTest classes `final` to silence warnings and prevent unintended subclassing +- Use `XCTUnwrap` instead of force-unwrapping (`!`) inside tests so the failure is reported rather than crashing the runner + +## Async, Throws, and Concurrency + +- Test methods may be `async` and/or `throws` in both frameworks +- For asynchronous expectations under XCTest, use `XCTestExpectation` + `wait(for:timeout:)` only when you cannot refactor to `async` +- For Swift Testing, use `await confirmation { ... }` to assert that a callback fires +- Cancel tasks deliberately with `Task.cancel()` instead of relying on test timeout + +## Common Errors + +| Error | Fix | +|-------|-----| +| `cannot find 'X' in scope` from a test | Add `@testable import MyLibrary` (and ensure the test target depends on it) | +| `module 'MyLibrary' was not compiled for testing` | Build the production target with `-enable-testing`; SPM test targets do this automatically — Xcode Debug configs need "Enable Testability" = YES | +| `failed to launch test runner` (Xcode) | Simulator destination may be invalid; list with `xcrun simctl list devices` and pick an existing one | +| `No such module 'XCTest'` outside a test target | XCTest is only available in test targets — do not import it from production code | +| `Static method 'expect(_:_:sourceLocation:)' is unavailable` / `No such module 'Testing'` | Swift Testing requires Swift 6 / Xcode 16+. On older toolchains, fall back to XCTest | +| `Symbol not found: _OBJC_CLASS_$_...` | Linker missing a framework; add it to the test target's "Link Binary With Libraries" | +| `signal SIGABRT` in tests | Often a force-unwrap on `nil`; replace `!` with `XCTUnwrap` to localize the failure | +| `MainActor-isolated property cannot be referenced from a non-isolated context` | Mark the test method `@MainActor` or move setup into a `MainActor` task | +| `Sandbox: ... deny file-write-create` | Use `FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)` instead of writing to fixed paths | +| Test discovery shows zero tests on Linux | XCTest on Linux needs `XCTMain([testCase(MyTests.allTests), ...])` in `Tests/LinuxMain.swift` (legacy SwiftPM only); for Swift 5.4+ this is auto-generated | + +## Mocking Rules + +Swift has no Mockito-equivalent — favor protocol-oriented design: + +- Define a **protocol** for the dependency, pass it via initializer, and implement a fake/stub struct in the test target +- For URL/HTTP, use `URLProtocol` subclasses to intercept `URLSession` requests, or use `MockingbirdSwift` / `Cuckoo` if the repo already adopts them +- For dates/clocks, inject a `Clock` (`ContinuousClock`, `SuspendingClock`, or a custom `Clock`-conforming type) — do not call `Date()` directly in business logic +- Avoid `swizzling` and runtime hacks — they break under Swift's optimizer + +If a test needs more than 3 mocks, flag it as a design smell. + +## Cross-Platform Considerations + +- Swift on Linux supports XCTest but **not** all of Foundation — guard with `#if canImport(Darwin)` or `#if os(macOS)` only when necessary +- Use `String(decoding:as:)` rather than `String(contentsOf:encoding:)` for cross-platform reads +- Be careful with `Bundle.main` in tests — on macOS unit tests it points to `xctest`, not your bundle; use `Bundle(for: type(of: self))` (XCTest) or a resource-bundle helper + +## Dependency Installation (Last Resort) + +Only add dependencies after investigation confirms they are missing. + +`Package.swift`: + +```swift +.package(url: "https://github.com/apple/swift-collections.git", from: "1.1.0"), +``` + +Then add to the test target's `dependencies:`. For CocoaPods/Carthage, edit `Podfile`/`Cartfile` and run `pod install` / `carthage update --use-xcframeworks`. + +## Skip Coverage Tools + +Do not configure or run coverage tools (`-enableCodeCoverage YES`, `xccov`, `slather`). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/extensions/typescript-examples.md b/.github/skills/code-testing-extensions/extensions/typescript-examples.md new file mode 100644 index 0000000..85cd710 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/typescript-examples.md @@ -0,0 +1,423 @@ +# TypeScript Pipeline Examples + +Concrete input→output examples for the test generation pipeline targeting a TypeScript codebase using Vitest. These show what each pipeline phase produces for a small project. + +> Jest, Mocha, and node:test follow the same shape. Replace `vi.fn()` / `vi.mock()` with `jest.fn()` / `jest.mock()` (Jest) or hand-written stubs (node:test/Mocha) and adjust the runner command accordingly. + +## Source Under Test + +A simple `InvoiceService` in a TypeScript library using Vitest: + +```text +src/ + invoiceService.ts + invoice.ts + invoiceRepository.ts + index.ts (re-exports public API) +package.json +tsconfig.json +vitest.config.ts +package-lock.json (committed for reproducible installs) +``` + +```typescript +// src/invoiceService.ts +import { Invoice, InvoiceStatus } from "./invoice"; +import { InvoiceRepository } from "./invoiceRepository"; + +export class InvoiceService { + constructor(private readonly repository: InvoiceRepository) {} + + calculateTotal(invoice: Invoice): number { + if (invoice == null) throw new TypeError("invoice must not be null"); + if (invoice.lineItems.length === 0) { + throw new Error("Invoice has no line items."); + } + + const subtotal = invoice.lineItems.reduce( + (acc, li) => acc + li.quantity * li.unitPrice, + 0, + ); + const tax = subtotal * invoice.taxRate; + return roundTo2(subtotal + tax); + } + + async getById(id: number): Promise { + const invoice = await this.repository.find(id); + if (invoice == null) { + throw new Error(`Invoice ${id} not found.`); + } + return invoice; + } + + async markAsPaid(id: number): Promise { + const invoice = await this.repository.find(id); + if (invoice == null) { + throw new Error(`Invoice ${id} not found.`); + } + if (invoice.status === InvoiceStatus.Paid) { + throw new Error("Invoice is already paid."); + } + invoice.status = InvoiceStatus.Paid; + invoice.paidDate = new Date(); + await this.repository.update(invoice); + } +} + +function roundTo2(n: number): number { + return Math.round((n + Number.EPSILON) * 100) / 100; +} +``` + +## Sample Research Output + +What `code-testing-researcher` produces in `.testagent/research.md`: + +```markdown +# Test Generation Research + +## Project Overview +- **Path**: /work/contoso-billing +- **Language**: TypeScript 5.4 +- **Module system**: ESM (`"type": "module"` in package.json) +- **Test Framework**: Vitest 1.x (detected via `vitest.config.ts` and `devDependencies.vitest`) +- **Package Manager**: npm (lockfile = `package-lock.json`) + +## Coverage Baseline +- **Initial Line Coverage**: unknown +- **Strategy**: broad +- **Existing Test Count**: 0 tests across 0 files + +## Build & Test Commands +- **Install**: `npm ci` +- **Type-check**: `npx tsc --noEmit` +- **Test**: `npx vitest run` (NEVER bare `vitest` — that starts watch mode) +- **Lint**: none configured + +## Project Structure +- Source: `src/` +- Tests: none (will colocate as `src/invoiceService.test.ts` to match Vitest defaults) + +## Files to Test + +### High Priority +| File | Classes/Functions | Testability | Notes | +|------|-------------------|-------------|-------| +| src/invoiceService.ts | InvoiceService: calculateTotal, getById, markAsPaid | High | Core business logic, repository dependency needs mocking | + +### Low Priority / Skip +| File | Reason | +|------|--------| +| src/invoice.ts | Type definitions and enum | +| src/invoiceRepository.ts | Interface only | +| src/index.ts | Re-export barrel | + +## Existing Tests +- No existing tests found + +## Testing Patterns +- No existing patterns; recommend `describe`/`it` blocks, `vi.fn()` stubs for the repository interface, and `it.each` for table-driven cases. + +## Recommendations +- Co-locate test next to source (`src/invoiceService.test.ts`) — matches Vitest defaults and avoids reaching into `../src/` +- Use a fake-timers helper (`vi.useFakeTimers()`) to control `new Date()` in `markAsPaid` +- Use a type-narrowed mock object (`{ find: vi.fn(), update: vi.fn() } satisfies InvoiceRepository`) rather than full module mocking +``` + +## Sample Plan Output + +What `code-testing-planner` produces in `.testagent/plan.md`: + +```markdown +# Test Implementation Plan + +## Overview +Generate Vitest tests for InvoiceService, covering all three public methods +across happy path, edge case, and error scenarios. Single phase since there is +only one source file. + +## Commands +- **Install**: `npm ci` +- **Type-check**: `npx tsc --noEmit` +- **Test (file-scoped during dev)**: `npx vitest run src/invoiceService.test.ts` +- **Test (full)**: `npx vitest run` + +## Phase Summary +| Phase | Focus | Files | Est. Tests | +|-------|-------|-------|------------| +| 1 | InvoiceService | 1 | 9-12 | + +--- + +## Phase 1: InvoiceService + +### Overview +Cover all public methods of InvoiceService. `calculateTotal` is pure logic tested +with `it.each`. Async methods require a fake repository. + +### Files to Test + +#### 1. invoiceService.ts +- **Source**: `src/invoiceService.ts` +- **Test File**: `src/invoiceService.test.ts` + +**Methods to Test**: +1. `calculateTotal` — Pure calculation logic + - Happy path: single line item returns quantity × price + tax + - Happy path: multiple line items summed correctly + - Edge case: zero tax rate returns subtotal only + - Error case: null invoice throws TypeError + - Error case: empty line items throws Error + +2. `getById` — Repository lookup + - Happy path: existing ID returns invoice + - Error case: missing ID rejects with Error + +3. `markAsPaid` — State transition + - Happy path: pending invoice transitions to Paid with `paidDate` set + - Error case: already-paid rejects with Error + - Error case: missing ID rejects with Error + +### Success Criteria +- [ ] Test file created at `src/invoiceService.test.ts` +- [ ] `npx tsc --noEmit` succeeds +- [ ] `npx vitest run` reports all tests passed +- [ ] No real network/timers — repository is a `vi.fn()` fake, `new Date()` is controlled via fake timers +``` + +## Sample Generated Test File + +What `code-testing-implementer` produces: + +```typescript +// src/invoiceService.test.ts +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Invoice, InvoiceStatus } from "./invoice"; +import type { InvoiceRepository } from "./invoiceRepository"; +import { InvoiceService } from "./invoiceService"; + +function makeRepository(): InvoiceRepository & { find: ReturnType; update: ReturnType } { + return { + find: vi.fn(), + update: vi.fn(), + }; +} + +describe("InvoiceService", () => { + let repository: ReturnType; + let sut: InvoiceService; + + beforeEach(() => { + repository = makeRepository(); + sut = new InvoiceService(repository); + }); + + // --- calculateTotal --- + + describe("calculateTotal", () => { + it.each([ + { quantity: 1, unitPrice: 100, taxRate: 0.1, expected: 110 }, + { quantity: 3, unitPrice: 25, taxRate: 0, expected: 75 }, + { quantity: 2, unitPrice: 9.99, taxRate: 0.07, expected: 21.38 }, + ])( + "returns $expected for $quantity × $unitPrice with tax $taxRate", + ({ quantity, unitPrice, taxRate, expected }) => { + const invoice: Invoice = { + id: 1, + status: InvoiceStatus.Pending, + taxRate, + lineItems: [{ quantity, unitPrice }], + }; + + expect(sut.calculateTotal(invoice)).toBe(expected); + }, + ); + + it("throws TypeError when invoice is null", () => { + expect(() => sut.calculateTotal(null as unknown as Invoice)).toThrow(TypeError); + }); + + it("throws when line items are empty", () => { + const invoice: Invoice = { + id: 1, + status: InvoiceStatus.Pending, + taxRate: 0, + lineItems: [], + }; + + expect(() => sut.calculateTotal(invoice)).toThrow("no line items"); + }); + }); + + // --- getById --- + + describe("getById", () => { + it("returns the invoice for an existing id", async () => { + const expected: Invoice = { id: 42, status: InvoiceStatus.Pending, taxRate: 0, lineItems: [] }; + repository.find.mockResolvedValue(expected); + + await expect(sut.getById(42)).resolves.toBe(expected); + expect(repository.find).toHaveBeenCalledWith(42); + }); + + it("rejects with Error when the id is missing", async () => { + repository.find.mockResolvedValue(null); + + await expect(sut.getById(999)).rejects.toThrow(/999/); + }); + }); + + // --- markAsPaid --- + + describe("markAsPaid", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2025-01-01T12:00:00.000Z")); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it("transitions a pending invoice to Paid with paidDate set", async () => { + const invoice: Invoice = { + id: 1, + status: InvoiceStatus.Pending, + taxRate: 0, + lineItems: [], + }; + repository.find.mockResolvedValue(invoice); + repository.update.mockResolvedValue(undefined); + + await sut.markAsPaid(1); + + expect(invoice.status).toBe(InvoiceStatus.Paid); + expect(invoice.paidDate).toEqual(new Date("2025-01-01T12:00:00.000Z")); + expect(repository.update).toHaveBeenCalledWith(invoice); + }); + + it("rejects when the invoice is already paid", async () => { + const invoice: Invoice = { + id: 1, + status: InvoiceStatus.Paid, + taxRate: 0, + lineItems: [], + }; + repository.find.mockResolvedValue(invoice); + + await expect(sut.markAsPaid(1)).rejects.toThrow("already paid"); + expect(repository.update).not.toHaveBeenCalled(); + }); + + it("rejects when the id is missing", async () => { + repository.find.mockResolvedValue(null); + + await expect(sut.markAsPaid(999)).rejects.toThrow(/999/); + }); + }); +}); +``` + +## Sample Fix Cycle + +When the implementer encounters a runner or type error, the fixer agent diagnoses and resolves it. + +**Test output:** + +```text +Error: Vitest failed to access its internal state. +One of the following is possible: +- "vitest" is imported directly without running "vitest" command +``` + +**Fixer diagnosis:** The agent ran `node src/invoiceService.test.ts` (or bare `vitest`, which is watch-mode). The runner must be invoked via `npx vitest run`. + +**Fix applied:** + +```bash +# Wrong — bare vitest starts an interactive watcher in CI +npx vitest + +# Right — `run` is the one-shot command +npx vitest run +``` + +**Rerun:** SUCCESS + +--- + +**Another common cycle — ESM/CJS mismatch:** + +**Test output:** + +```text +SyntaxError: Cannot use import statement outside a module +``` + +**Fixer diagnosis:** The project's `tsconfig.json` emits ESM (`"module": "NodeNext"`) but `package.json` has no `"type": "module"`. Vitest happens to handle this natively; switching to Jest would require additional configuration. The fix here is to ensure Vitest is the runner being used (as already configured in `vitest.config.ts`) and avoid recompiling test files through a separate non-ESM-aware tool. + +**Fix applied:** Use `npx vitest run` (which uses esbuild internally and handles both ESM and CJS) instead of compiling with `tsc` and running the emitted `.js` directly. + +**Rerun:** SUCCESS + +--- + +**Another common cycle — wrong mock typing:** + +**Build output:** + +```text +src/invoiceService.test.ts:14:5 - error TS2322: Type '{ find: Mock; }' is not assignable to type 'InvoiceRepository'. + Property 'update' is missing in type '{ find: Mock; }' but required in type 'InvoiceRepository'. +``` + +**Fixer diagnosis:** The fake repository only stubbed `find`, not `update`. The `InvoiceRepository` interface requires both. TypeScript caught this at compile time. + +**Fix applied:** + +```typescript +// Before +const repository = { find: vi.fn() } as InvoiceRepository; + +// After — provide both methods, narrow the return type so the test code keeps autocomplete +function makeRepository(): InvoiceRepository & { find: ReturnType; update: ReturnType } { + return { find: vi.fn(), update: vi.fn() }; +} +``` + +**Rebuild + rerun:** SUCCESS + +## Sample Final Report + +What `code-testing-generator` produces at Step 9: + +```markdown +## Test Generation Report + +**Project**: contoso-billing (TypeScript) +**Strategy**: Direct (single source file in scope) + +### Results +| Metric | Value | +|----------------|-------| +| Tests created | 9 | +| Tests passing | 9 | +| Tests failing | 0 | +| Files created | 1 | + +### Files Created +- `src/invoiceService.test.ts` (9 tests, 3 parameterized via `it.each`) + +### Coverage +- InvoiceService.calculateTotal — 3 happy path, 2 error cases +- InvoiceService.getById — 1 happy path, 1 error case +- InvoiceService.markAsPaid — 1 happy path, 2 error cases + +### Build / Test Validation +- Install: ✅ `npm ci` +- Type-check: ✅ `npx tsc --noEmit` +- Test run: ✅ `npx vitest run` + +### Next Steps +- Add tests for any HTTP/Express adapters once they exist +- Consider property-based testing (`fast-check`) for `calculateTotal` rounding +``` diff --git a/.github/skills/code-testing-extensions/extensions/typescript.md b/.github/skills/code-testing-extensions/extensions/typescript.md new file mode 100644 index 0000000..de26a70 --- /dev/null +++ b/.github/skills/code-testing-extensions/extensions/typescript.md @@ -0,0 +1,136 @@ +# TypeScript Extension + +Language-specific guidance for TypeScript (and JavaScript) test generation. + +## Rule #1: Investigate the Repo First + +Before writing any test or running any command, read: + +1. **Existing tests** — find `*.test.ts` / `*.spec.ts` files and copy their style (imports, describe/it vs test, assertion patterns, mock approach) +2. **`package.json`** — `scripts.test`, `devDependencies`, `type` field +3. **Config files** — `tsconfig.json`, `jest.config.*`, `vitest.config.*`, `eslint.config.*` + +Use the repo's existing test runner and conventions — do not switch frameworks. If multiple runners are configured, follow whichever `scripts.test` invokes. Only introduce a framework if the repo has no tests at all. + +## Package Manager Detection + +Detect the package manager from lockfiles and use it consistently for **all** commands: + +| Indicator | Manager | Run script | Execute binary | +|-----------|---------|------------|----------------| +| `pnpm-lock.yaml` | pnpm | `pnpm test` | `pnpm exec ` | +| `yarn.lock` | Yarn | `yarn test` | `yarn ` | +| `bun.lockb` / `bun.lock` | Bun | `bun test` | `bunx ` | +| `package-lock.json` or none | npm | `npm test` | `npx ` | + +Use `` below as shorthand for the detected exec command. + +## Build Commands + +| Scope | Command | +|-------|---------| +| Type check | ` tsc --noEmit` or the repo's `typecheck` script | +| Build (if configured) | The repo's `build` script | + +Many projects don't need an explicit build step — the test runner handles transpilation. + +## Test Commands + +Detect the runner from `devDependencies` and `scripts.test`. Always prefer the repo's test script first. + +| Runner | Run once | Filter by file | Filter by name | +|--------|----------|----------------|----------------| +| **Jest** | ` jest` | ` jest path/to/file` | ` jest -t "name"` | +| **Vitest** | ` vitest run` | ` vitest run path/to/file` | ` vitest run -t "name"` | +| **Mocha** | ` mocha` | (use config or positional args) | ` mocha --grep "name"` | + +- **Always use `vitest run`** (not bare `vitest`) — bare `vitest` starts watch mode +- **Never use `--watch`** — the agent must not start interactive/watch mode +- For Jest: `--bail` to stop on first failure, `--verbose` for detail +- Mocha `--grep` filters by **test name**, not file path + +## Lint Command + +Use the repo's lint script first. Otherwise detect from `devDependencies` and config: + +- `eslint.config.*` or `.eslintrc.*` → ` eslint --fix path/to/file.ts` +- `prettier` → ` prettier --write path/to/file.ts` +- `biome.json` → ` biome check --write path/to/file.ts` + +## Project Layout and Imports + +| Layout | Import Style | +|--------|-------------| +| Colocated (`src/module.test.ts`) | `import { X } from './module'` | +| `__tests__/` dir | `import { X } from '../module'` | +| Top-level `tests/` | `import { X } from '../src/module'` | + +- **Match existing test imports** — copy path style from neighboring tests +- If `tsconfig.json` has `paths` aliases (e.g., `@/`), use them in tests too +- For monorepos: import from the package name, not relative cross-package paths +- For monorepo workspaces (Nx, Turborepo, Lerna): run tests via the workspace tool (`nx test `, `turbo test`), not from a random package directory + +## Test File Naming + +- Match existing convention — check for `.test.ts` vs `.spec.ts` +- Jest/Vitest default: `*.test.ts`, `*.spec.ts`, or files inside `__tests__/` +- Place test files to mirror the existing project pattern + +## Common Errors + +| Error | Fix | +|-------|-----| +| `Cannot find module 'X'` | Check existing imports for correct paths; verify `tsconfig.json` `paths`; check `moduleNameMapper` (Jest) or `resolve.alias` (Vitest) | +| `TS2305: has no exported member` | Verify the exact export name from the source file | +| `TS2345: type not assignable` | Match the expected type; use type assertion only for mock objects | +| `SyntaxError: Unexpected token` / `Jest encountered an unexpected token` | Verify TS transform config (`ts-jest`, `@swc/jest`, or Vitest handles natively) | +| `ReferenceError: describe is not defined` | Vitest: import from `vitest` or set `globals: true` in config; Jest: ensure tests run under Jest not bare `node` | +| `Cannot use import statement outside a module` / `ERR_REQUIRE_ESM` | ESM/CJS mismatch — align runner config with the project's module system (see ESM section); do **not** blindly set `"type": "module"` | +| `ReferenceError: document is not defined` | Set test environment: `testEnvironment: 'jsdom'` (Jest) or `environment: 'jsdom'` (Vitest) | +| `jest.mock() ... out-of-scope variables` | Keep `jest.mock()` at top level; don't reference variables declared after the mock call (Jest hoists mocks) | +| `Cannot find module '@/...'` | Mirror the project's alias config in the test runner's module resolution | +| `Warning: not wrapped in act(...)` | Await async UI updates using the repo's existing pattern (`waitFor`, `act`) | + +## ESM vs CommonJS + +Check these signals to determine the project's module system: + +- `"type": "module"` in `package.json` → ESM +- `"module": "ESNext"` or `"NodeNext"` in `tsconfig.json` → ESM output (but not sufficient alone) +- `.mjs`/`.mts` extensions → ESM files + +If the test runner fails with ESM errors, align the runner's config with the project's module system. **Do not change `package.json` `type` field** — align the test runner to match whatever the project uses: + +- **Jest**: `--experimental-vm-modules` + `ts-jest` with `useESM: true`, or `@swc/jest` +- **Vitest**: handles ESM natively +- **Mocha**: `--loader ts-node/esm` + +## Mocking Rules + +- Prefer dependency injection over module mocking +- Use typed mocks: `jest.Mocked`, `vi.mocked(obj)`, or `Partial` with `as T` +- Jest: `jest.mock()` is hoisted — keep at top level, don't close over local variables +- Vitest: `vi.mock()` follows the same hoisting rules +- If a test needs more than 3–4 mocks, flag it as a design smell +- Mock reset: rely on `clearMocks`/`restoreMocks` config if present; otherwise reset in `beforeEach` + +## Framework-Specific Notes + +- **React/Preact**: use `@testing-library/react`, wrap with necessary providers (router, query client, theme) matching existing test setup +- **Express/Koa**: use `supertest` for HTTP testing if the repo already uses it +- **NestJS**: build testing module with `Test.createTestingModule` — don't instantiate controllers directly + +## Dependency Installation (Last Resort) + +Only install packages after investigation confirms they are missing. Use the detected package manager: + +``` + add --save-dev jest ts-jest @types/jest + add --save-dev vitest +``` + +Never install test infrastructure that conflicts with what the repo already uses. + +## Skip Coverage Tools + +Do not configure or run coverage tools (istanbul, c8, `vitest --coverage`). Coverage is measured separately by the evaluation harness. diff --git a/.github/skills/code-testing-extensions/skill.md b/.github/skills/code-testing-extensions/skill.md new file mode 100644 index 0000000..d1e96d6 --- /dev/null +++ b/.github/skills/code-testing-extensions/skill.md @@ -0,0 +1,40 @@ +--- +name: code-testing-extensions +description: >- + Provides file paths to language-specific extension files for the code-testing + pipeline. Call this skill to discover available extension guidance files + (e.g., dotnet.md for .NET, cpp.md for C++). Do not use directly — invoked + by code-testing agents and skills that need language-specific references. +user-invocable: false +disable-model-invocation: true +license: MIT +--- + +# Code Testing Extensions + +This skill provides access to language-specific guidance files used by the code-testing pipeline. Call this skill to get the file paths, then read the relevant file for your target language. + +## Available Extension Files + +| File | Language | Contents | +|------|----------|----------| +| [extensions/dotnet.md](extensions/dotnet.md) | .NET (C#/F#/VB) | Build commands, test commands, project reference validation, common CS error codes, MSTest template | +| [extensions/python.md](extensions/python.md) | Python | Framework-adaptive test commands (pytest, custom runners), project layout detection, mocking guidelines, common errors | +| [extensions/typescript.md](extensions/typescript.md) | TypeScript/JavaScript | Build/test commands (Jest/Vitest/Mocha), framework detection, mocking, TS-specific considerations | +| [extensions/powershell.md](extensions/powershell.md) | PowerShell | Test commands (Pester v5), module import patterns, discovery/run pitfalls, mocking, common errors | +| [extensions/cpp.md](extensions/cpp.md) | C++ | Testing internals with friend declarations | +| [extensions/go.md](extensions/go.md) | Go | `go test` commands, table-driven tests, integration vs unit layout, mocking via interfaces, common errors | +| [extensions/java.md](extensions/java.md) | Java | Maven/Gradle commands, JUnit 4/5 and TestNG detection, Mockito, Spring Boot slices, common errors | +| [extensions/rust.md](extensions/rust.md) | Rust | `cargo test` commands, unit vs integration vs doc tests, features, async test harnesses, common errors | +| [extensions/ruby.md](extensions/ruby.md) | Ruby | RSpec and Minitest commands, Bundler usage, Rails specifics, mocking patterns, common errors | +| [extensions/swift.md](extensions/swift.md) | Swift | SPM and Xcode test commands, XCTest vs Swift Testing, `@testable import`, async/throws tests, common errors | +| [extensions/kotlin.md](extensions/kotlin.md) | Kotlin | Gradle commands, JUnit/Kotest detection, MockK, coroutines test, KMP and Android specifics, common errors | +| [extensions/dotnet-examples.md](extensions/dotnet-examples.md) | .NET (C#/F#/VB) | Concrete pipeline examples: sample research output, plan, generated tests, fix cycles, final report | +| [extensions/python-examples.md](extensions/python-examples.md) | Python | Concrete pipeline examples (pytest): research, plan, generated test file, fix cycles, final report | +| [extensions/typescript-examples.md](extensions/typescript-examples.md) | TypeScript/JavaScript | Concrete pipeline examples (Vitest, applicable to Jest): research, plan, generated test file, fix cycles, final report | +| [extensions/go-examples.md](extensions/go-examples.md) | Go | Concrete pipeline examples (standard `testing`): research, plan, table-driven test file, fix cycles, final report | +| [extensions/java-examples.md](extensions/java-examples.md) | Java | Concrete pipeline examples (JUnit 5 + Mockito on Maven): research, plan, generated test file, fix cycles, final report | + +## Usage + +Read the appropriate extension file for the target language before writing test code. When an `-examples.md` file exists for the target language, read it alongside the base extension to see a concrete end-to-end pipeline walkthrough (research output, plan, generated tests, fix cycles, final report). diff --git a/.github/skills/coverage-analysis/references/guidelines.md b/.github/skills/coverage-analysis/references/guidelines.md new file mode 100644 index 0000000..344f69e --- /dev/null +++ b/.github/skills/coverage-analysis/references/guidelines.md @@ -0,0 +1,59 @@ +# Guidelines + +**Don't modify source or production code.** The only permitted project file modifications are adding a coverage provider package to test projects that currently have no provider: `coverlet.collector` (coverlet/mixed modes) or `Microsoft.Testing.Extensions.CodeCoverage` (ms-codecoverage mode). Do not add a second provider to projects that already have one. Always log package additions and document revert commands in the report. Write all other output to `TestResults/coverage-analysis/` under the test project directory. + +**Always show and open the generated markdown report — but only after the assistant response with the CRAP/risk-hotspot summary has been delivered.** Saving and opening `TestResults/coverage-analysis/coverage-analysis.md` is a follow-up action; it must never delay the user-facing summary. + +**Don't generate new tests during the initial analysis run.** This skill surfaces where tests are needed. Test generation is a separate follow-up step outside the scope of this skill. + +**Use inline `dotnet test` arguments, not runsettings files.** Runsettings files require the developer to already know what they're doing — the whole point of this skill is that they shouldn't have to. Inline data collector args produce the same result with zero configuration. + +**Show the risk hotspots table even when all thresholds pass.** A project at 90% line coverage can still have a method with cyclomatic complexity 20 and 0% branch coverage. The thresholds measure averages; the hotspot table finds outliers. Don't hide it just because the summary looks green. + +**Always compute and surface CRAP scores.** The Risk Hotspots table is mandatory in every analysis output, whether analyzing pre-existing data, freshly collected data, or diagnosing a plateau. Never skip CRAP score computation — it is the primary differentiator between this skill and raw `dotnet test` coverage output. + +**Continue past test failures (exit code 1).** If some tests fail, coverage is still collected from the passing tests — partial data is better than no data. Note the failures in the summary and proceed. Aborting would leave the developer with nothing actionable. + +**Run `dotnet test` only once per entry point during normal flow.** When a solution is found, run it once against the solution. When no solution is found, run it once per test project. A single recovery rerun is allowed only if the first run produced no Cobertura XML and only `.coverage` binary output. + +**CRAP threshold of 30 is the default for a reason.** Scores above 30 are widely cited (by the original researchers) as "needs immediate attention." Scores between 15 and 30 are moderate — flag them in the table but don't make them sound catastrophic. Scores ≤ 5 are generally fine. + +**Priority assignment for coverage gaps:** + +- **HIGH** — file has both a CRAP score above threshold AND coverage below threshold (the double failure is what makes it urgent) +- **MED** — coverage below threshold OR CRAP score above threshold, but not both +- **LOW** — coverage below threshold with all methods having complexity ≤ 2 (trivial code — missing coverage here is unlikely to hide real bugs) + +--- + +## Coverage Intelligence — Going Beyond the Numbers + +**Prioritize uncovered code that is** complex (cyclomatic complexity > 5), on critical paths (auth, payment, data access, error handling), or changed frequently. **Deprioritize** trivial getters (complexity 1–2), generated files (EF migrations, `*.Designer.cs`, `*.g.cs`), and DI/configuration glue code. + +**Coverage plateau diagnosis** — if coverage has stopped increasing, check for: `[Exclude]` attributes hiding large code sections, tests that execute code but assert nothing (inflated coverage without verification), or integration code that needs external dependencies (databases, file system). + +**AI-generated test quality** — coverage delta alone is insufficient. Flag methods where CRAP score is still above threshold after coverage increased (tests may be happy-path only), and methods covered by a single test with no branch variation. + +--- + +## Style + +- **Keep risk hotspots prominent and immediately after the summary section** — developers should find the highest-risk methods quickly +- **Quantify recommendations** — "adding 3 tests for `ProcessOrder` would cut the CRAP score from 48 to ~6" +- **Be direct** — skip preamble, get to the table +- **Emoji for visual scanning in generated output** (defined in `references/output-format.md`): + + | Symbol | Meaning | + |--------|---------| + | 🔥 | hotspots | + | 📋 | gaps | + | 💡 | recommendations | + | 📁 | reports | + | ✅ | passing | + | ❌ | failing | + | ⚠️ | warning | + | 🔴 | HIGH priority | + | 🟡 | MED priority | + | 🟢 | LOW priority | + +- **Always use Unicode emoji in generated output** — never shortcodes like `:x:` or `:fire:` diff --git a/.github/skills/coverage-analysis/references/output-format.md b/.github/skills/coverage-analysis/references/output-format.md new file mode 100644 index 0000000..7e3c5b6 --- /dev/null +++ b/.github/skills/coverage-analysis/references/output-format.md @@ -0,0 +1,87 @@ +# Output Format + +Copy the template below **verbatim** for all fixed elements (headings, table headers, emoji, symbols). Only replace `` values with actual data. Do not substitute emoji with text equivalents, do not change `·` to `-`, do not change `×` to `x`, and do not drop section emoji prefixes. + +```markdown +# Coverage Analysis - + +| Metric | Value | +|--------|-------| +| **Date** | | +| **Line Coverage** | % | +| **Branch Coverage** | % | +| **Risk Hotspots** | (CRAP > ) | +| **Tests** | passed · failed | + +## Summary + +| Metric | Value | Threshold | Status | +|--------|-------|-----------|--------| +| **Line Coverage** | % | % | ✅ / ❌ | +| **Branch Coverage** | % | % | ✅ / ❌ | +| **Methods Analyzed** | | — | — | +| **Risk Hotspots** | | 0 | ✅ / ⚠️ | +| **Test Result** | | — | ✅ / ⚠️ | + +> Coverage collected from ** of test project(s)**. +> Outputs saved to: `/` (markdown summary + raw Cobertura XML). +> *If Phase 5 ran:* HTML/CSV reports also at `/reports/`. + +If any coverage provider package was added to test projects, include this note after the summary: + +> ℹ️ **Coverage provider package updates** +> - `coverlet.collector` added to `` project(s): ``, `` +> - `Microsoft.Testing.Extensions.CodeCoverage` added to `` project(s): `` +> +> To revert: `git checkout -- ` + +If all test projects already had a coverage provider, omit this note. + +--- + +## 🔥 Risk Hotspots (Top by CRAP Score) + +Methods flagged as high-risk: complex code with low test coverage that is dangerous to change. + +| Rank | Method | Class | File | Complexity | Coverage | CRAP Score | +|------|--------|-------|------|-----------|---------|-----------| +| 1 | `` | `` | `` | | % | **** | +| … | … | … | … | … | … | … | + +> **CRAP Score** = `Complexity² × (1 − Coverage)³ + Complexity`. +> Scores above are flagged. A score ≤ 5 is considered safe. + +--- + +## 📋 Coverage Gaps by File + +Files below the line or branch coverage threshold, ordered by uncovered lines descending: + +| File | Line Coverage | Branch Coverage | Uncovered Lines | Priority | +|------|--------------|----------------|----------------|---------| +| `` | % | % | | 🔴 HIGH / 🟡 MED / 🟢 LOW | +| … | … | … | … | … | + +--- + +## 💡 Recommendations + +1. **Write tests for the top risk hotspot first** — `` in `` has a CRAP score of (complexity , % coverage). Reducing it to 80% coverage would drop the score to ~. +2. **Focus on ``** — uncovered lines, below threshold. +3. **** + +--- + +## 📁 Reports + +| Report | Path | +|--------|------| +| Markdown summary (this file) | `/coverage-analysis.md` | +| Raw Cobertura XML | `` | +| HTML (browsable) | `/reports/index.html` *or* `Not generated (optional — request HTML reports to enable)` | +| Text summary | `/reports/Summary.txt` *or* `Not generated` | +| GitHub markdown | `/reports/SummaryGithub.md` *or* `Not generated` | +| CSV data | `/reports/Summary.csv` *or* `Not generated` | +``` + +If ReportGenerator (Phase 5) has not run, mark the HTML/Text/GitHub-markdown/CSV rows as `Not generated (optional — request HTML reports to enable)`. Do not invent paths for files that have not been produced. For **Raw Cobertura XML**, list the actual XML file path(s) used in analysis (for from-scratch runs this is typically under `/raw/`; for existing-data runs this may be under `TestResults/` or another user-supplied location). diff --git a/.github/skills/coverage-analysis/scripts/Compute-CrapScores.ps1 b/.github/skills/coverage-analysis/scripts/Compute-CrapScores.ps1 new file mode 100644 index 0000000..b0c8d9f --- /dev/null +++ b/.github/skills/coverage-analysis/scripts/Compute-CrapScores.ps1 @@ -0,0 +1,165 @@ +# Compute-CrapScores.ps1 +# +# Reads a Cobertura XML coverage file and calculates CRAP scores per method. +# Uses Alberto Savoia's original CRAP formula: +# CRAP(m) = comp(m)^2 * (1 - cov(m))^3 + comp(m) +# +# Usage: +# .\Compute-CrapScores.ps1 -CoberturaPath ,,... [-CrapThreshold ] [-TopN ] +# +# Outputs: +# - OVERALL_LINE_COVERAGE: (aggregate line coverage across input files, as percent) +# - OVERALL_BRANCH_COVERAGE: (aggregate branch coverage across input files, as percent) +# - TOTAL_METHODS: +# - FLAGGED_METHODS: +# - HOTSPOTS: (top N by CRAP score) + +param( + [Parameter(Mandatory)][string[]]$CoberturaPath, + [int]$CrapThreshold = 30, + [int]$TopN = 10 +) + +# Merge methods across all Cobertura files using a stable key (Class|Method|Signature|File). +# Line hits are accumulated so a line is counted as covered if any input coverage file covered it. +$methodMap = @{} +$overallLineRate = 0.0 +$overallBranchRate = 0.0 +$totalLinesCovered = 0 +$totalLinesValid = 0 +$totalBranchesCovered = 0 +$totalBranchesValid = 0 +$fallbackLineRates = [System.Collections.Generic.List[double]]::new() +$fallbackBranchRates = [System.Collections.Generic.List[double]]::new() + +foreach ($filePath in $CoberturaPath) { + if (-not (Test-Path $filePath)) { + Write-Error "Cobertura file not found: $filePath" + exit 2 + } + + try { + [xml]$cobertura = Get-Content $filePath -Encoding UTF8 -ErrorAction Stop + } catch { + Write-Error "Failed to parse Cobertura XML: $filePath. $_" + exit 2 + } + + # Prefer aggregate numerator/denominator attributes when present. + if ($null -ne $cobertura.coverage.'lines-covered' -and $null -ne $cobertura.coverage.'lines-valid') { + $totalLinesCovered += [double]$cobertura.coverage.'lines-covered' + $totalLinesValid += [double]$cobertura.coverage.'lines-valid' + } elseif ($cobertura.coverage.'line-rate') { + $fallbackLineRates.Add([double]$cobertura.coverage.'line-rate') + } + if ($null -ne $cobertura.coverage.'branches-covered' -and $null -ne $cobertura.coverage.'branches-valid') { + $totalBranchesCovered += [double]$cobertura.coverage.'branches-covered' + $totalBranchesValid += [double]$cobertura.coverage.'branches-valid' + } elseif ($cobertura.coverage.'branch-rate') { + $fallbackBranchRates.Add([double]$cobertura.coverage.'branch-rate') + } + + foreach ($package in $cobertura.coverage.packages.package) { + foreach ($class in $package.classes.class) { + $className = $class.name + $fileName = $class.filename + + foreach ($method in $class.methods.method) { + $key = "$className|$($method.name)|$($method.signature)|$fileName" + + # Cyclomatic complexity is stored as an XML attribute in Cobertura format + $complexity = if ($null -ne $method.complexity) { [int]$method.complexity } else { 1 } + if ($complexity -lt 1) { $complexity = 1 } + + if (-not $methodMap.ContainsKey($key)) { + $methodMap[$key] = @{ + Class = $className + Method = $method.name + Signature = $method.signature + File = $fileName + Complexity = $complexity + LineHits = @{} + } + } + + # Accumulate hit counts per line number across files + foreach ($line in $method.lines.line) { + $lineNo = $line.number + $hits = [int]$line.hits + if ($methodMap[$key].LineHits.ContainsKey($lineNo)) { + $methodMap[$key].LineHits[$lineNo] += $hits + } else { + $methodMap[$key].LineHits[$lineNo] = $hits + } + } + } + } + } +} + +$results = [System.Collections.Generic.List[PSCustomObject]]::new() + +foreach ($entry in $methodMap.Values) { + $totalLines = $entry.LineHits.Count + $coveredLines = ($entry.LineHits.Values | Where-Object { $_ -gt 0 } | Measure-Object).Count + $lineCoverage = if ($totalLines -gt 0) { $coveredLines / $totalLines } else { 0.0 } + + $complexity = $entry.Complexity + + # Alberto Savoia's CRAP formula: comp^2 * (1 - cov)^3 + comp + # The cubic exponent on (1-cov) sharply penalizes low coverage: + # at 0% coverage the risk multiplier is 1.0; at 50% it drops to 0.125. + # Higher scores = more complex AND less covered = riskier to change + $uncovered = 1.0 - $lineCoverage + $crapScore = [Math]::Round(($complexity * $complexity * [Math]::Pow($uncovered, 3)) + $complexity, 2) + + $results.Add([PSCustomObject]@{ + Class = $entry.Class + Method = $entry.Method + Signature = $entry.Signature + File = $entry.File + TotalLines = $totalLines + CoveredLines = $coveredLines + LineCoverage = [Math]::Round($lineCoverage * 100, 1) + Complexity = $complexity + CrapScore = $crapScore + }) +} + +$hotspots = $results | Sort-Object CrapScore -Descending | Select-Object -First $TopN +$flagged = $results | Where-Object { $_.CrapScore -gt $CrapThreshold } + +if ($totalLinesValid -gt 0) { + $overallLineRate = $totalLinesCovered / $totalLinesValid +} else { + # Fallback approximation when Cobertura aggregate counters and per-file rates are unavailable. + # This uses merged method line totals and may under/over-estimate if Cobertura + # includes executable lines outside method nodes. + $mergedTotalLines = ($results | Measure-Object -Property TotalLines -Sum).Sum + $mergedCoveredLines = ($results | Measure-Object -Property CoveredLines -Sum).Sum + if ($mergedTotalLines -gt 0) { + $overallLineRate = [double]$mergedCoveredLines / [double]$mergedTotalLines + } elseif ($fallbackLineRates.Count -gt 0) { + $overallLineRate = ($fallbackLineRates | Measure-Object -Average).Average + } else { + $overallLineRate = 0.0 + } +} + +if ($totalBranchesValid -gt 0) { + $overallBranchRate = $totalBranchesCovered / $totalBranchesValid +} elseif ($fallbackBranchRates.Count -gt 0) { + $overallBranchRate = ($fallbackBranchRates | Measure-Object -Average).Average +} else { + $overallBranchRate = 0.0 +} + +Write-Host "OVERALL_LINE_COVERAGE:$([Math]::Round($overallLineRate * 100, 1))" +Write-Host "OVERALL_BRANCH_COVERAGE:$([Math]::Round($overallBranchRate * 100, 1))" +Write-Host "TOTAL_METHODS:$($results.Count)" +Write-Host "FLAGGED_METHODS:$($flagged.Count)" +if ($hotspots) { + Write-Output "HOTSPOTS:$(@($hotspots) | ConvertTo-Json -Compress)" +} else { + Write-Output "HOTSPOTS:[]" +} diff --git a/.github/skills/coverage-analysis/scripts/Extract-MethodCoverage.ps1 b/.github/skills/coverage-analysis/scripts/Extract-MethodCoverage.ps1 new file mode 100644 index 0000000..999a827 --- /dev/null +++ b/.github/skills/coverage-analysis/scripts/Extract-MethodCoverage.ps1 @@ -0,0 +1,193 @@ +param( + [Parameter(Mandatory=$true)] + [string[]]$CoberturaPath, + + [Parameter(Mandatory=$false)] + [int]$CoverageThreshold = 80, + + [Parameter(Mandatory=$false)] + [int]$BranchThreshold = 70, + + [Parameter(Mandatory=$false)] + [ValidateSet('uncovered', 'below-threshold', 'all')] + [string]$Filter = 'all' +) + +<# +.SYNOPSIS +Extract method-level coverage from Cobertura XML and output as JSON. + +.DESCRIPTION +Parses one or more Cobertura code coverage XML files and extracts per-method coverage metrics: +- Method name and class +- Line coverage percentage +- Branch coverage percentage +- Lines covered / total +- Branches covered / total +- Complexity (if available) + +When multiple files are provided, line hits are merged across files so a line is counted +as covered if any test project covered it. + +Filters by coverage status (uncovered, below threshold, or all). +Output is JSON for easy post-processing into tables, CSV, or other formats. + +.PARAMETER CoberturaPath +Path(s) to Cobertura coverage.cobertura.xml file(s). Accepts multiple paths for multi-test-project merging. + +.PARAMETER CoverageThreshold +Minimum acceptable line coverage percentage. Methods below this threshold are flagged (default: 80). + +.PARAMETER BranchThreshold +Minimum acceptable branch coverage percentage for methods that contain branches (default: 70). + +.PARAMETER Filter +Which methods to include: + 'uncovered' - methods with 0% coverage only + 'below-threshold' - methods with line coverage < CoverageThreshold OR branch coverage < BranchThreshold (for methods with branches) + 'all' - all methods (default) + +.EXAMPLE +PS> & .\Extract-MethodCoverage.ps1 -CoberturaPath "coverage.cobertura.xml" -CoverageThreshold 80 -BranchThreshold 70 -Filter uncovered +Outputs a JSON array of uncovered methods. + +.EXAMPLE +PS> & .\Extract-MethodCoverage.ps1 -CoberturaPath @("tests1/coverage.cobertura.xml","tests2/coverage.cobertura.xml") +Merges coverage from multiple test projects and outputs combined method-level metrics. + +.OUTPUTS +Writes JSON array to stdout. +Sets exit code 0 on success, 2 on missing/invalid file. +#> + +foreach ($p in $CoberturaPath) { + if (-not (Test-Path $p)) { + Write-Error "Cobertura file not found: $p" + exit 2 + } +} + +# Merge methods across all Cobertura files using a stable key (Class|Method|Signature|File). +# Line hits and branch data are accumulated so coverage reflects all test projects. +$methodMap = @{} + +foreach ($p in $CoberturaPath) { + try { + [xml]$xml = Get-Content $p -Encoding UTF8 -ErrorAction Stop + } catch { + Write-Error "Failed to parse Cobertura XML: $_" + exit 2 + } + + foreach ($package in $xml.coverage.packages.package) { + foreach ($class in $package.classes.class) { + $className = $class.name + $classFilename = $class.filename + + foreach ($method in $class.methods.method) { + $key = "$className|$($method.name)|$($method.signature)|$classFilename" + + if (-not $methodMap.ContainsKey($key)) { + $complexity = if ($null -ne $method.complexity) { [int]$method.complexity } else { 1 } + if ($complexity -lt 1) { $complexity = 1 } + $methodMap[$key] = @{ + Class = $className + Method = $method.name + Signature = $method.signature + File = $classFilename + Complexity = $complexity + LineHits = @{} + BranchData = @{} + } + } + + # Accumulate line hits across files + foreach ($line in $method.lines.line) { + $lineNo = $line.number + $hits = [int]$line.hits + if ($methodMap[$key].LineHits.ContainsKey($lineNo)) { + $methodMap[$key].LineHits[$lineNo] += $hits + } else { + $methodMap[$key].LineHits[$lineNo] = $hits + } + + # Accumulate branch data + if ($line.branch -eq 'true' -and $line.'condition-coverage') { + if ($line.'condition-coverage' -match '\((\d+)/(\d+)\)') { + $covered = [int]$Matches[1] + $total = [int]$Matches[2] + if ($methodMap[$key].BranchData.ContainsKey($lineNo)) { + # Merge branch coverage across files by accumulating covered branches (capped at total) + $existingCovered = $methodMap[$key].BranchData[$lineNo].Covered + $existingTotal = $methodMap[$key].BranchData[$lineNo].Total + if ($existingTotal -ne $total) { + Write-Warning ("Branch total mismatch for {0} at line {1}: {2} vs {3}" -f $key, $lineNo, $existingTotal, $total) + } + $mergedTotal = [Math]::Max($existingTotal, $total) + $mergedCovered = [Math]::Min($existingCovered + $covered, $mergedTotal) + $methodMap[$key].BranchData[$lineNo] = @{ Covered = $mergedCovered; Total = $mergedTotal } + } else { + $methodMap[$key].BranchData[$lineNo] = @{ Covered = $covered; Total = $total } + } + } + } + } + } + } + } +} + +$methods = [System.Collections.Generic.List[PSCustomObject]]::new() + +foreach ($entry in $methodMap.Values) { + $totalLines = $entry.LineHits.Count + $coveredLineCount = ($entry.LineHits.Values | Where-Object { $_ -gt 0 } | Measure-Object).Count + $lineCoveragePercent = if ($totalLines -gt 0) { [math]::Round(($coveredLineCount / $totalLines) * 100, 1) } else { 0 } + + $branchesTotal = 0 + $branchesCovered = 0 + foreach ($bd in $entry.BranchData.Values) { + $branchesCovered += $bd.Covered + $branchesTotal += $bd.Total + } + $branchCoveragePercent = if ($branchesTotal -gt 0) { [math]::Round(($branchesCovered / $branchesTotal) * 100, 1) } else { 0 } + + # Apply filter + if ($Filter -eq 'uncovered' -and $lineCoveragePercent -gt 0) { continue } + if ($Filter -eq 'below-threshold') { + $lineOk = $lineCoveragePercent -ge $CoverageThreshold + $branchOk = ($branchesTotal -eq 0) -or ($branchCoveragePercent -ge $BranchThreshold) + if ($lineOk -and $branchOk) { continue } + } + + $methods.Add([PSCustomObject]@{ + Class = $entry.Class + Method = $entry.Method + Signature = $entry.Signature + File = $entry.File + Complexity = $entry.Complexity + LineCoverage = $lineCoveragePercent + BranchCoverage = $branchCoveragePercent + CoveredLines = $coveredLineCount + TotalLines = $totalLines + UncoveredLines = ($totalLines - $coveredLineCount) + CoveredBranches = $branchesCovered + TotalBranches = $branchesTotal + }) +} +# Sort by uncovered lines descending, then by line coverage ascending +$sorted = $methods | Sort-Object -Property @{Expression='UncoveredLines';Descending=$true}, @{Expression='LineCoverage';Descending=$false}, Class, Method + +# Output as JSON (empty array guard for zero results) +if ($sorted.Count -eq 0) { + Write-Output "[]" +} else { + $json = @($sorted) | ConvertTo-Json + Write-Output $json +} + +# Summary +Write-Host "METHODS_FILTERED:$($methods.Count)" -ForegroundColor Green +$uncovered = $methods | Where-Object { $_.LineCoverage -eq 0 } | Measure-Object | Select-Object -ExpandProperty Count +Write-Host "UNCOVERED_METHODS:$uncovered" -ForegroundColor $(if ($uncovered -gt 0) { 'Yellow' } else { 'Green' }) +exit 0 diff --git a/.github/skills/coverage-analysis/skill.md b/.github/skills/coverage-analysis/skill.md new file mode 100644 index 0000000..a6de1e0 --- /dev/null +++ b/.github/skills/coverage-analysis/skill.md @@ -0,0 +1,533 @@ +--- +name: coverage-analysis +description: > + Project-wide code coverage and CRAP (Change Risk Anti-Patterns) score + analysis for .NET projects. Calculates CRAP scores per method and surfaces + risk hotspots — complex code with low coverage that is dangerous to modify. + Use to diagnose why coverage is stuck or plateaued, identify what methods + block improvement, or get project-wide coverage analysis with risk ranking. + USE FOR: coverage stuck, coverage plateau, can't increase coverage, what's + blocking coverage, coverage gap, CRAP scores, risk hotspots, where to add + tests, coverage analysis, coverage report. + DO NOT USE FOR: targeted single-method CRAP analysis (use crap-score); + auditing test code for coverage-touching or other anti-patterns (use + test-anti-patterns); writing tests; running tests (use run-tests). Requires + or produces coverage (Cobertura) and CRAP metrics. +license: MIT +--- + +# Coverage Analysis + +## Purpose + +Raw coverage percentages answer "what code was executed?" — they don't answer what you actually need to know: + +- **What tests should I write next?** — ranked by risk and impact +- **Which uncovered code is risky vs. trivial?** — CRAP scores separate the two +- **Why has coverage plateaued?** — identify the files blocking further gains +- **Is this code safe to refactor?** — complex + uncovered = dangerous to change + +This skill bridges that gap: from a bare .NET solution to a prioritized risk hotspot list, with no manual tool configuration required. + +## When to Use + +Use this skill when the user mentions test coverage, coverage gaps, code risk, CRAP scores, where to add tests, why coverage plateaued, or wants to know which code is safest to refactor — even if they don't explicitly say "coverage analysis". + +## When Not to Use + +- **Targeted single-method CRAP analysis** — use the `crap-score` skill instead +- **Writing or generating tests** — this skill identifies where tests are needed, not write them +- **General test execution** unrelated to coverage or CRAP analysis +- **Coverage reporting without CRAP context** — use `dotnet test` with coverage collection directly + +## Inputs + +| Input | Required | Default | Description | +|-------|----------|---------|-------------| +| Project/solution path | No | Current directory | Path to the .NET solution or project | +| Line coverage threshold | No | 80% | Minimum acceptable line coverage | +| Branch coverage threshold | No | 70% | Minimum acceptable branch coverage | +| CRAP threshold | No | 30 | Maximum acceptable CRAP score before flagging | +| Top N hotspots | No | 10 | Number of risk hotspots to surface | + +### Prerequisites + +- .NET SDK installed (`dotnet` on PATH) +- At least one test project referencing the production code (xUnit, NUnit, or MSTest) — only required for the from-scratch path; not needed when the user supplies an existing Cobertura XML +- **Optional, only for the from-scratch path:** internet/NuGet access for `dotnet add package coverlet.collector` (or `Microsoft.Testing.Extensions.CodeCoverage`) when a test project has no coverage provider yet. Skip when the user supplies an existing Cobertura XML. +- **Optional, only for Phase 5:** internet access for `dotnet tool install` (ReportGenerator). Core CRAP/coverage analysis works from Cobertura XML alone — ReportGenerator only adds HTML/CSV reports as an optional post-summary extra. + +The skill auto-detects coverage provider state per test project and selects the least-invasive execution strategy: + +- unified Microsoft CodeCoverage when all projects use it, +- unified Coverlet when no project uses Microsoft CodeCoverage, +- per-project provider execution when the solution is truly mixed. + +No pre-existing runsettings files or manually installed tools required. + +## Workflow + +> **MANDATORY: deliver the final assistant response with the CRAP/risk-hotspot summary BEFORE any optional work.** As soon as `Compute-CrapScores.ps1` and `Extract-MethodCoverage.ps1` return data, your **next** assistant response must contain the user-facing analysis (CRAP table, blocking methods, recommendations). Do not run ReportGenerator (Phase 5), do not install global tools, and do not start any heavy parallel work before that response is delivered. The user is judged on the final assistant message, not on side-effect files. +> +> If a phase fails, times out, or budget is running low, skip remaining optional work and immediately return a partial summary containing: (1) what was found in the Cobertura XML, (2) any CRAP/risk-hotspot data already extracted, (3) which methods are blocking coverage, and (4) failures encountered. + +If the user provides a path to existing Cobertura XML (or coverage data is already present in `TestResults/`), **skip Phase 2 entirely** (no test execution) **and skip Phase 5 by default** (no ReportGenerator install or HTML report) — go directly from Phase 3 (analysis scripts) to Phase 4 (user-facing summary). Only run Phase 5 if the user explicitly asks for HTML/CSV reports. The Risk Hotspots table and CRAP scores are mandatory in every output — they are the skill's core value-add over raw coverage numbers. + +The workflow runs in five phases. Phases 1–4 are required; Phase 5 (ReportGenerator HTML/CSV reports) is strictly optional and runs **after** the user-facing summary has been delivered. Do not parallelize Phase 5 with earlier phases — the heavy `dotnet tool install` for ReportGenerator can crash the session before Phase 4 completes. + +### Phase 1 — Setup (sequential) + +#### Step 1: Locate the solution or project + +Given the user's path (default: current directory), find the entry point: + +```powershell +$root = "" + +# Prefer solution file; fall back to project file +$sln = Get-ChildItem -Path $root -Filter "*.sln" -Recurse -Depth 2 -ErrorAction SilentlyContinue | + Select-Object -First 1 +if ($sln) { + Write-Host "ENTRY_TYPE:Solution"; Write-Host "ENTRY:$($sln.FullName)" +} else { + $project = Get-ChildItem -Path $root -Filter "*.csproj" -Recurse -Depth 2 -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($project) { + Write-Host "ENTRY_TYPE:Project"; Write-Host "ENTRY:$($project.FullName)" + } else { + Write-Host "ENTRY_TYPE:NotFound" + } +} + +# Test projects: search path first, then git root, then parent +$searchRoots = @($root) +$gitRoot = (git -C $root rev-parse --show-toplevel 2>$null) +if ($gitRoot) { $gitRoot = [System.IO.Path]::GetFullPath($gitRoot) } +if ($gitRoot -and $gitRoot -ne $root) { $searchRoots += $gitRoot } +$parentPath = Split-Path $root -Parent +if ($parentPath -and $parentPath -ne $root -and $parentPath -ne $gitRoot) { $searchRoots += $parentPath } + +$testProjects = @() +foreach ($sr in $searchRoots) { + # Primary: match by .csproj content (test framework references) + $testProjects = @(Get-ChildItem -Path $sr -Filter "*.csproj" -Recurse -Depth 5 -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -notmatch '([/\\]obj[/\\]|[/\\]bin[/\\])' } | + Where-Object { (Select-String -Path $_.FullName -Pattern 'Microsoft\.NET\.Test\.Sdk|xunit|nunit|MSTest\.TestAdapter|"MSTest"|MSTest\.TestFramework|TUnit' -Quiet) }) + if ($testProjects.Count -gt 0) { + if ($sr -ne $root) { Write-Host "SEARCHED:$sr" } + break + } +} + +# Fallback: match by file name convention +if ($testProjects.Count -eq 0) { + foreach ($sr in $searchRoots) { + $testProjects = @(Get-ChildItem -Path $sr -Filter "*.csproj" -Recurse -Depth 5 -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match '(?i)(test|spec)' }) + if ($testProjects.Count -gt 0) { + if ($sr -ne $root) { Write-Host "SEARCHED:$sr" } + break + } + } +} +Write-Host "TEST_PROJECTS:$($testProjects.Count)" +$testProjects | ForEach-Object { Write-Host "TEST_PROJECT:$($_.FullName)" } + +# Resolve the test output root (where coverage-analysis artifacts will be written) +if ($testProjects.Count -eq 0) { + if ($gitRoot) { + $testOutputRoot = $gitRoot + } else { + $testOutputRoot = $root + } +} elseif ($testProjects.Count -eq 1) { + $testOutputRoot = $testProjects[0].DirectoryName +} else { + # Multiple test projects — find their deepest common parent directory + $dirs = $testProjects | ForEach-Object { $_.DirectoryName } + $common = $dirs[0] + foreach ($d in $dirs[1..($dirs.Count-1)]) { + $sep = [System.IO.Path]::DirectorySeparatorChar + while (-not $d.StartsWith("$common$sep", [System.StringComparison]::OrdinalIgnoreCase) -and $d -ne $common) { + $prevCommon = $common + $common = Split-Path $common -Parent + # Terminate if we can no longer move up (at filesystem root or no parent) + if ([string]::IsNullOrEmpty($common) -or $common -eq $prevCommon) { + $common = $null + break + } + } + } + if ([string]::IsNullOrEmpty($common)) { + # Fallback when no common parent directory exists (e.g., projects on different drives) + if ($gitRoot) { + $testOutputRoot = $gitRoot + } else { + $testOutputRoot = $root + } + } else { + $testOutputRoot = $common + } +} +Write-Host "TEST_OUTPUT_ROOT:$testOutputRoot" +``` + +- If `ENTRY_TYPE:NotFound` and test projects were found → use the test projects directly as entry points (run `dotnet test` on each test `.csproj`). +- If `ENTRY_TYPE:NotFound` and no test projects found → stop: `No .sln or test projects found under . Provide the path to your .NET solution or project.` +- If `TEST_PROJECTS:0` and `EXISTING_COBERTURA_COUNT` > 0 (Step 2b) → continue with existing Cobertura XML analysis (no `dotnet test` run). +- If `TEST_PROJECTS:0` and `EXISTING_COBERTURA_COUNT` == 0 → stop: `No test projects found (expected projects with 'Test' or 'Spec' in the name), and no existing Cobertura XML was provided. Add a test project or provide a Cobertura file path.` + +#### Step 2: Create the output directory + +```powershell +$coverageDir = Join-Path $testOutputRoot "TestResults" "coverage-analysis" +if (Test-Path $coverageDir) { Remove-Item $coverageDir -Recurse -Force } +New-Item -ItemType Directory -Path $coverageDir -Force | Out-Null +Write-Host "COVERAGE_DIR:$coverageDir" +``` + +This step only manages the `TestResults/coverage-analysis/` subdirectory (skill-owned outputs). It must never delete user-supplied Cobertura files — those live one level up at `TestResults/coverage.cobertura.xml` (or wherever the user pointed). If the user provided a path that *is* `TestResults/coverage-analysis/...`, copy the file aside before this step recreates the directory. + +#### Step 2b: Discover or accept existing Cobertura XML (required for the existing-data path) + +If the user supplied a Cobertura XML path explicitly, use it. Otherwise probe well-known locations and any path the user mentioned: + +```powershell +# 1. Honor a user-supplied path first (highest priority) +$coberturaFiles = @() +if ($userSuppliedCoberturaPath -and (Test-Path $userSuppliedCoberturaPath)) { + $coberturaFiles = @(Get-Item $userSuppliedCoberturaPath) +} + +# 2. Otherwise scan TestResults/ at the repo/test root for any *.cobertura.xml +if ($coberturaFiles.Count -eq 0) { + $searchPaths = @( + (Join-Path $testOutputRoot "TestResults"), + (Join-Path $root "TestResults") + ) | Where-Object { $_ -and (Test-Path $_) } | Select-Object -Unique + foreach ($sp in $searchPaths) { + $found = @(Get-ChildItem -Path $sp -Filter "*.cobertura.xml" -Recurse -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -notmatch '[/\\]coverage-analysis[/\\]raw[/\\]' }) + if ($found.Count -gt 0) { $coberturaFiles = $found; break } + } +} + +Write-Host "EXISTING_COBERTURA_COUNT:$($coberturaFiles.Count)" +$coberturaFiles | ForEach-Object { Write-Host "EXISTING_COBERTURA:$($_.FullName)" } +``` + +- If `EXISTING_COBERTURA_COUNT` > 0 → **skip Phase 2 entirely** and pass these paths to the Phase 3 scripts. +- If `EXISTING_COBERTURA_COUNT` == 0 → run Phase 2 to generate fresh coverage; the file paths to feed Phase 3 will be discovered from `/raw/` after `dotnet test`. + +#### Step 2c: Recommend ignoring `TestResults/` + +```powershell +$pattern = "**/TestResults/" +$gitRoot = (git -C $testOutputRoot rev-parse --show-toplevel 2>$null) +if ($gitRoot) { $gitRoot = [System.IO.Path]::GetFullPath($gitRoot) } +if ($gitRoot) { + $gitignorePath = Join-Path $gitRoot ".gitignore" + $alreadyIgnored = $false + if (Test-Path $gitignorePath) { + $alreadyIgnored = (Select-String -Path $gitignorePath -Pattern '^\s*(\*\*/)?TestResults/?\s*$' -Quiet) + } + if ($alreadyIgnored) { + Write-Host "GITIGNORE_RECOMMENDATION:already-present" + } else { + Write-Host "GITIGNORE_RECOMMENDATION:$pattern" + } +} else { + Write-Host "GITIGNORE_RECOMMENDATION:$pattern" +} +``` + +### Phase 2 — Test execution (skip when Cobertura XML already exists) + +Run only when no Cobertura XML is present. If the user already has coverage data, skip directly to Phase 3. + +#### Step 3: Detect coverage provider and run `dotnet test` with coverage collection + +Before running tests, detect which coverage provider the test projects use. Projects may reference +`Microsoft.Testing.Extensions.CodeCoverage` (Microsoft's built-in provider, common on .NET 9+) or +`coverlet.collector` (open-source, the default in xUnit templates). The provider determines which +`dotnet test` arguments to use — both produce Cobertura XML. + +```powershell +# Detect coverage provider per test project +$coverageProvider = "unknown" # will be set to "ms-codecoverage" or "coverlet" +$msCodeCovProjects = @() +$coverletProjects = @() +$neitherProjects = @() + +foreach ($tp in $testProjects) { + $hasMsCodeCov = Select-String -Path $tp.FullName -Pattern 'Microsoft\.Testing\.Extensions\.CodeCoverage' -Quiet + $hasCoverlet = Select-String -Path $tp.FullName -Pattern 'coverlet\.collector' -Quiet + if ($hasMsCodeCov) { $msCodeCovProjects += $tp } + elseif ($hasCoverlet) { $coverletProjects += $tp } + else { $neitherProjects += $tp } +} + +# Determine the provider strategy +if ($msCodeCovProjects.Count -gt 0 -and $coverletProjects.Count -eq 0) { + $coverageProvider = "ms-codecoverage" + Write-Host "COVERAGE_PROVIDER:ms-codecoverage (ms:$($msCodeCovProjects.Count), none:$($neitherProjects.Count))" +} elseif ($coverletProjects.Count -gt 0 -and $msCodeCovProjects.Count -eq 0) { + $coverageProvider = "coverlet" + Write-Host "COVERAGE_PROVIDER:coverlet (coverlet:$($coverletProjects.Count), none:$($neitherProjects.Count))" +} elseif ($msCodeCovProjects.Count -gt 0 -and $coverletProjects.Count -gt 0) { + $coverageProvider = "mixed-project" + Write-Host "COVERAGE_PROVIDER:mixed-project (ms:$($msCodeCovProjects.Count), coverlet:$($coverletProjects.Count), none:$($neitherProjects.Count))" +} else { + $coverageProvider = "coverlet" + Write-Host "COVERAGE_PROVIDER:none-detected — defaulting to coverlet" +} +``` + +If any discovered test projects have no provider, add one based on the selected strategy: + +```powershell +if ($coverageProvider -eq "ms-codecoverage" -and $neitherProjects.Count -gt 0) { + Write-Host "ADDING_MS_CODECOVERAGE:$($neitherProjects.Count) project(s)" + foreach ($tp in $neitherProjects) { + dotnet add $tp.FullName package Microsoft.Testing.Extensions.CodeCoverage --no-restore + Write-Host " ADDED_MS_CODECOVERAGE:$($tp.FullName)" + } + foreach ($tp in $neitherProjects) { + dotnet restore $tp.FullName --quiet + } +} + +if (($coverageProvider -eq "coverlet" -or $coverageProvider -eq "mixed-project") -and $neitherProjects.Count -gt 0) { + Write-Host "ADDING_COVERLET:$($neitherProjects.Count) project(s)" + foreach ($tp in $neitherProjects) { + dotnet add $tp.FullName package coverlet.collector --no-restore + Write-Host " ADDED:$($tp.FullName)" + } + foreach ($tp in $neitherProjects) { + dotnet restore $tp.FullName --quiet + } +} +``` + +Log each addition to the console so the developer sees what changed. Document the additions in the final report (see Output Format). + +Run one `dotnet test` per entry point for the selected strategy: + +- In `ms-codecoverage` or `coverlet` mode: run a single command for the solution entry (or one per test project if no `.sln` was found). +- In `mixed-project` mode: run one command per test project, using that project's existing provider to avoid dual-provider conflicts. + +**Coverlet** (`coverlet.collector`): + +```powershell +$rawDir = Join-Path "" "raw" +dotnet test "" ` + --collect:"XPlat Code Coverage" ` + --results-directory $rawDir ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Include="[*]*" ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Exclude="[*.Tests]*,[*.Test]*,[*Tests]*,[*Test]*,[*.Specs]*,[*.Testing]*" ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.SkipAutoProps=true +``` + +**Microsoft CodeCoverage** (`Microsoft.Testing.Extensions.CodeCoverage`): + +The command syntax depends on the .NET SDK version. In .NET 9, Microsoft.Testing.Platform arguments +must be passed after the `--` separator. In .NET 10+, `--coverage` is a top-level `dotnet test` flag. + +```powershell +$rawDir = Join-Path "" "raw" + +# Detect SDK version for correct argument placement +$sdkVersion = (dotnet --version 2>$null) +$major = if ($sdkVersion -match '^(\d+)\.') { [int]$Matches[1] } else { 9 } + +if ($major -ge 10) { + # .NET 10+: --coverage is a first-class dotnet test flag + dotnet test "" ` + --results-directory $rawDir ` + --coverage ` + --coverage-output-format cobertura ` + --coverage-output $rawDir +} else { + # .NET 9: pass Microsoft.Testing.Platform arguments after the -- separator + dotnet test "" ` + --results-directory $rawDir ` + -- --coverage --coverage-output-format cobertura --coverage-output $rawDir +} +``` + +**Mixed-project mode** (`Microsoft.Testing.Extensions.CodeCoverage` + `coverlet.collector` in the same solution): + +```powershell +$rawDir = Join-Path "" "raw" +$sdkVersion = (dotnet --version 2>$null) +$major = if ($sdkVersion -match '^(\d+)\.') { [int]$Matches[1] } else { 9 } + +foreach ($tp in $testProjects) { + $hasMsCodeCov = Select-String -Path $tp.FullName -Pattern 'Microsoft\.Testing\.Extensions\.CodeCoverage' -Quiet + if ($hasMsCodeCov) { + if ($major -ge 10) { + dotnet test $tp.FullName --results-directory $rawDir --coverage --coverage-output-format cobertura --coverage-output $rawDir + } else { + dotnet test $tp.FullName --results-directory $rawDir -- --coverage --coverage-output-format cobertura --coverage-output $rawDir + } + } else { + dotnet test $tp.FullName ` + --collect:"XPlat Code Coverage" ` + --results-directory $rawDir ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Include="[*]*" ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Exclude="[*.Tests]*,[*.Test]*,[*Tests]*,[*Test]*,[*.Specs]*,[*.Testing]*" ` + -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.SkipAutoProps=true + } +} +``` + +Exit code handling: + +- **0** — all tests passed, coverage collected +- **1** — some tests failed (coverage still collected — proceed with a warning) +- **Other** — build failure; stop and report the error + +After the run, locate coverage files: + +```powershell +$coberturaFiles = Get-ChildItem -Path (Join-Path "" "raw") -Filter "coverage.cobertura.xml" -Recurse +Write-Host "COBERTURA_COUNT:$($coberturaFiles.Count)" +$coberturaFiles | ForEach-Object { Write-Host "COBERTURA:$($_.FullName)" } +$vsCovFiles = Get-ChildItem -Path (Join-Path "" "raw") -Filter "*.coverage" -Recurse -ErrorAction SilentlyContinue +if ($vsCovFiles) { Write-Host "VS_BINARY_COVERAGE:$($vsCovFiles.Count)" } +``` + +If `COBERTURA_COUNT` is 0: + +- If `VS_BINARY_COVERAGE` > 0: warn the user — *"Found .coverage files (VS binary format) but no Cobertura XML. These were likely produced by Visual Studio's built-in collector, which outputs a binary format by default. This skill needs Cobertura XML. Re-running with the detected provider configured for Cobertura output."* Then re-run the appropriate `dotnet test` command above (Coverlet or Microsoft CodeCoverage) with Cobertura format. +- If no `.coverage` files either: stop and report — *"Coverage files not generated. Ensure `dotnet test` completed successfully and check the build output for errors."* + +### Phase 3 — Analysis (sequential) + +Run the two bundled PowerShell scripts. Both are cheap and complete in seconds. **Do not** install or invoke ReportGenerator here — that belongs in optional Phase 5, after the user-facing summary has been delivered. + +#### Step 4: Calculate CRAP scores using the bundled script + +Run `scripts/Compute-CrapScores.ps1` (co-located with this SKILL.md). It reads all Cobertura XML files, applies `CRAP(m) = comp² × (1 − cov)³ + comp` per method, and returns the top-N hotspots as JSON. + +To locate the script: find the directory containing this skill's `SKILL.md` file (the skill loader provides this context), then resolve `scripts/Compute-CrapScores.ps1` relative to it. If the script path cannot be determined, calculate CRAP scores inline using the formula below. + +```powershell +& "/scripts/Compute-CrapScores.ps1" ` + -CoberturaPath @() ` + -CrapThreshold ` + -TopN +``` + +Script outputs: `OVERALL_LINE_COVERAGE:`, `OVERALL_BRANCH_COVERAGE:` (aggregated project-wide rates across all provided Cobertura files), `TOTAL_METHODS:`, `FLAGGED_METHODS:`, `HOTSPOTS:` (top-N sorted by CrapScore descending). The OVERALL_* values are exactly what the Phase 4 summary needs for the "Line Coverage" / "Branch Coverage" rows — no separate XML parsing tool call is required. + +#### Step 5: Extract per-method coverage gaps + +Run `scripts/Extract-MethodCoverage.ps1` to get per-method coverage data for the Coverage Gaps table: + +```powershell +& "/scripts/Extract-MethodCoverage.ps1" ` + -CoberturaPath @() ` + -CoverageThreshold ` + -BranchThreshold ` + -Filter below-threshold +``` + +Script outputs: JSON array of methods below the coverage threshold, sorted by coverage ascending. Use this data to populate the Coverage Gaps by File table in the report. + +### Phase 4 — User-facing summary (MANDATORY — your next assistant response) + +As soon as Phase 3 completes, **your immediately next assistant response must contain the user-facing analysis** — do not interleave any other tool calls before it. This is the response the user (and any judge) sees. Skipping or deferring this in favor of Phase 5 (ReportGenerator) is a hard failure. + +The response must include, at minimum: + +1. Overall line and branch coverage — read directly from the `OVERALL_LINE_COVERAGE:` / `OVERALL_BRANCH_COVERAGE:` lines emitted by `Compute-CrapScores.ps1` (no extra Cobertura parsing required) +2. The Risk Hotspots table built from `Compute-CrapScores.ps1` `HOTSPOTS:` output (CRAP scores, complexity, coverage) +3. Identification of the highest-risk method(s) and what is blocking coverage +4. 1–3 prioritized, specific recommendations (which method to test, expected CRAP/coverage impact) + +Use `references/output-format.md` verbatim for fixed headings, table structures, symbols, and emoji. Use `references/guidelines.md` for prioritization rules and style. + +If Phase 5 has not yet run when you compose this summary, mark the `## 📁 Reports` section's HTML/Text/CSV/GitHub-markdown rows as `Not generated (optional — request HTML reports to enable)`. Only the `coverage-analysis.md` and raw Cobertura paths are guaranteed to exist. + +Attempt to save the same content to `TestResults/coverage-analysis/coverage-analysis.md` before delivering the response (use the editor's create/edit tool — do not shell out). If the file write fails, still deliver the summary and note the file-write failure explicitly. + +### Phase 5 — Optional: ReportGenerator HTML/CSV reports (post-summary) + +Phase 5 is **strictly optional** and runs **only after** Phase 4 has been delivered. Skip Phase 5 entirely when: + +- The user supplied existing Cobertura XML and only asked for analysis (the default for the existing-data path). +- The user is diagnosing a coverage plateau or asking "what's blocking me?" — they want the answer, not a static-site report. +- ReportGenerator is not already installed and you have no clear signal the user wants HTML reports. + +Run Phase 5 only when the user explicitly asks for HTML/CSV reports, or when the project flow requires them (e.g., a CI artifact upload step). + +#### Step 6: Verify or install ReportGenerator (only if running Phase 5) + +```powershell +$rgAvailable = $false +$rgCommand = Get-Command reportgenerator -ErrorAction SilentlyContinue +if ($rgCommand) { + $rgAvailable = $true + Write-Host "RG_INSTALLED:already-present" +} else { + $rgToolPath = Join-Path "" ".tools" + dotnet tool install dotnet-reportgenerator-globaltool --tool-path $rgToolPath + if ($LASTEXITCODE -eq 0) { + $env:PATH = "$rgToolPath$([System.IO.Path]::PathSeparator)$env:PATH" + $rgCommand = Get-Command reportgenerator -ErrorAction SilentlyContinue + if ($rgCommand) { + $rgAvailable = $true + Write-Host "RG_INSTALLED:true (tool-path: $rgToolPath)" + } else { + Write-Host "RG_INSTALLED:false" + Write-Host "RG_INSTALL_ERROR:reportgenerator-not-available" + } + } else { + Write-Host "RG_INSTALLED:false" + Write-Host "RG_INSTALL_ERROR:reportgenerator-not-available" + } +} +Write-Host "RG_AVAILABLE:$rgAvailable" +``` + +If installation fails (no internet), keep `RG_AVAILABLE:false`, leave the existing user-facing summary as the final output, and note that HTML reports were skipped. + +#### Step 7: Generate HTML/CSV reports + +```powershell +$reportsDir = Join-Path "" "reports" +if ($rgAvailable) { + reportgenerator ` + -reports:"" ` + -targetdir:$reportsDir ` + -reporttypes:"Html;TextSummary;MarkdownSummaryGithub;CsvSummary" ` + -title:"Coverage Report" ` + -tag:"coverage-analysis-skill" + + Get-Content (Join-Path $reportsDir "Summary.txt") -ErrorAction SilentlyContinue +} else { + Write-Host "REPORTGENERATOR_SKIPPED:true" +} +``` + +After Phase 5 completes successfully, you may follow up with a short message pointing the user to the generated HTML report (one paragraph, no need to repeat the summary). + +## Validation + +- Verify that at least one `coverage.cobertura.xml` file was generated after `dotnet test` (or already exists when the user supplied one) +- Confirm the assistant response contained the CRAP/risk-hotspot table — saving the markdown file is secondary +- Confirm `TestResults/coverage-analysis/coverage-analysis.md` was written and contains data +- Spot-check one method's CRAP score: `comp² × (1 − cov)³ + comp` — a method with 100% coverage should have CRAP = complexity +- If Phase 5 ran, verify `TestResults/coverage-analysis/reports/index.html` exists; otherwise the report file should mark HTML/Text/CSV rows as `Not generated` + +## Common Pitfalls + +- **No Cobertura XML generated** — the test project may lack a coverage provider. The skill auto-adds one, but if `dotnet add package` fails (offline/proxy), coverage collection silently produces nothing. Check for `.coverage` binary files as a fallback indicator. +- **Test failures (exit code 1)** — coverage is still collected from passing tests. Do not abort; proceed with partial data and note the failures in the summary. +- **Premature end before user-facing summary** — never start Phase 5 (ReportGenerator install/run) before the Phase 4 assistant response is delivered. The heavy `dotnet tool install` can crash the session or exhaust budget, leaving the user with no analysis even though the CRAP scores were already computed. +- **ReportGenerator install failure** — if `dotnet tool install` fails (no internet) during Phase 5, leave the existing Phase 4 summary as the final output and note that HTML reports were skipped. Do not retry or block on the install. +- **Method name mismatches in Cobertura** — async methods, lambdas, and local functions may have compiler-generated names. The scripts use the Cobertura method name/signature directly; verify against source if results look unexpected. +- **Mixed coverage providers** — when a solution contains both Coverlet and Microsoft CodeCoverage projects, the skill runs per-project to avoid dual-provider conflicts. This is slower but correct. diff --git a/.github/skills/run-tests/references/filter-syntax.md b/.github/skills/run-tests/references/filter-syntax.md new file mode 100644 index 0000000..03d23ec --- /dev/null +++ b/.github/skills/run-tests/references/filter-syntax.md @@ -0,0 +1,166 @@ +# Test Filter Syntax Reference + +Filter syntax depends on the **platform** and **test framework**. + +## VSTest filters (MSTest, xUnit v2, NUnit on VSTest) + +```bash +dotnet test --filter +``` + +Expression syntax: `[|&]` + +**Operators:** + +| Operator | Meaning | +|----------|---------| +| `=` | Exact match | +| `!=` | Not exact match | +| `~` | Contains | +| `!~` | Does not contain | + +**Combinators:** `|` (OR), `&` (AND). Parentheses for grouping: `(A|B)&C` + +**Supported properties by framework:** + +| Framework | Properties | +|-----------|-----------| +| MSTest | `FullyQualifiedName`, `Name`, `ClassName`, `Priority`, `TestCategory` | +| xUnit | `FullyQualifiedName`, `DisplayName`, `Traits` | +| NUnit | `FullyQualifiedName`, `Name`, `Priority`, `TestCategory` | + +An expression without an operator is treated as `FullyQualifiedName~`. + +**Examples (VSTest):** + +```bash +# Run tests whose name contains "LoginTest" +dotnet test --filter "Name~LoginTest" + +# Run a specific test class +dotnet test --filter "ClassName=MyNamespace.MyTestClass" + +# Run tests in a category +dotnet test --filter "TestCategory=Integration" + +# Exclude a category +dotnet test --filter "TestCategory!=Slow" + +# Combine: class AND category +dotnet test --filter "ClassName=MyNamespace.MyTestClass&TestCategory=Unit" + +# Either of two classes +dotnet test --filter "ClassName=MyNamespace.ClassA|ClassName=MyNamespace.ClassB" +``` + +## MTP filters — MSTest and NUnit + +MSTest and NUnit on MTP use the **same `--filter` syntax** as VSTest (same properties, operators, and combinators). The only difference is how the flag is passed: + +```bash +# .NET SDK 8/9 (after --) +dotnet test -- --filter "Name~LoginTest" + +# .NET SDK 10+ (direct) +dotnet test --filter "Name~LoginTest" +``` + +## MTP filters — xUnit (v3) + +xUnit v3 on MTP uses **framework-specific filter flags** instead of the generic `--filter` expression: + +| Flag | Description | +|------|-------------| +| `--filter-class "name"` | Run all tests in a given class | +| `--filter-not-class "name"` | Exclude all tests in a given class | +| `--filter-method "name"` | Run a specific test method | +| `--filter-not-method "name"` | Exclude a specific test method | +| `--filter-namespace "name"` | Run all tests in a namespace | +| `--filter-not-namespace "name"` | Exclude all tests in a namespace | +| `--filter-trait "name=value"` | Run tests with a matching trait | +| `--filter-not-trait "name=value"` | Exclude tests with a matching trait | + +Multiple values can be specified with a single flag: `--filter-class Foo Bar`. + +```bash +# .NET SDK 8/9 +dotnet test -- --filter-class "MyNamespace.LoginTests" + +# .NET SDK 10+ +dotnet test --filter-class "MyNamespace.LoginTests" + +# Combine: namespace + trait +dotnet test --filter-namespace "MyApp.Tests.Integration" --filter-trait "Category=Smoke" +``` + +### xUnit v3 query filter language + +For complex expressions, use `--filter-query` with a path-segment syntax: + +``` +////[traitName=traitValue] +``` + +Each segment matches against: assembly name, namespace, class name, method name. Use `*` for "match all" in any segment. Documentation: https://xunit.net/docs/query-filter-language + +```shell +# xUnit.net v3 MTP — using query language (assembly/namespace/class/method[trait]) +dotnet test -- --filter-query "/*/*/*IntegrationTests*/*[Category=Smoke]" +``` + +## MTP filters — TUnit + +TUnit uses `--treenode-filter` with a path-based syntax: + +``` +--treenode-filter "////" +``` + +Wildcards (`*`) are supported in any segment. Filter operators can be appended to test names for property-based filtering. + +| Operator | Meaning | +|----------|---------| +| `*` | Wildcard match | +| `=` | Exact property match (e.g., `[Category=Unit]`) | +| `!=` | Exclude property value | +| `&` | AND (combine conditions) | +| `\|` | OR (within a segment, requires parentheses) | + +**Examples (TUnit):** + +```bash +# All tests in a class +dotnet run --treenode-filter "/*/*/LoginTests/*" + +# A specific test +dotnet run --treenode-filter "/*/*/*/AcceptCookiesTest" + +# By namespace prefix (wildcard) +dotnet run --treenode-filter "/*/MyProject.Tests.Api*/*/*" + +# By custom property +dotnet run --treenode-filter "/*/*/*/*[Category=Smoke]" + +# Exclude by property +dotnet run --treenode-filter "/*/*/*/*[Category!=Slow]" + +# OR across classes +dotnet run --treenode-filter "/*/*/(LoginTests)|(SignupTests)/*" + +# Combined: namespace + property +dotnet run --treenode-filter "/*/MyProject.Tests.Integration/*/*/*[Priority=Critical]" +``` + +## VSTest → MTP filter translation (for migration) + +**MSTest, NUnit, and xUnit.net v2 (with `YTest.MTP.XUnit2`)**: The VSTest `--filter` syntax is identical on both VSTest and MTP. No changes needed. + +**xUnit.net v3 (native MTP)**: xUnit.net v3 does NOT support the VSTest `--filter` syntax on MTP. Translate filters using xUnit.net v3's native options: + +| VSTest `--filter` syntax | xUnit.net v3 MTP equivalent | Notes | +|---|---|---| +| `FullyQualifiedName~ClassName` | `--filter-class *ClassName*` | Wildcards required for substring match | +| `FullyQualifiedName=Ns.Class.Method` | `--filter-method Ns.Class.Method` | Exact match on fully qualified method | +| `Name=MethodName` | `--filter-method *MethodName*` | Wildcards for substring match | +| `Category=Value` (trait) | `--filter-trait "Category=Value"` | Filter by trait name/value pair | +| Complex expressions | `--filter-query "expr"` | Uses xUnit.net query filter language (see above) | diff --git a/.github/skills/run-tests/references/platform-detection.md b/.github/skills/run-tests/references/platform-detection.md new file mode 100644 index 0000000..501113b --- /dev/null +++ b/.github/skills/run-tests/references/platform-detection.md @@ -0,0 +1,53 @@ +# Test Platform and Framework Detection + +Determine **which test platform** (VSTest or Microsoft.Testing.Platform) and **which test framework** (MSTest, xUnit, NUnit, TUnit) a project uses. + +**Detection files to always check** (in order): `global.json` → `.csproj` → `Directory.Build.props` → `Directory.Packages.props` + +## Detecting the test framework + +Read the `.csproj` file **and** `Directory.Build.props` / `Directory.Packages.props` (for centrally managed dependencies) and look for: + +| Package or SDK reference | Framework | +|--------------------------|-----------| +| `MSTest` (metapackage, recommended) or `` | MSTest | +| `MSTest.TestFramework` + `MSTest.TestAdapter` | MSTest (also valid for v3/v4) | +| `xunit`, `xunit.v3`, `xunit.v3.mtp-v1`, `xunit.v3.mtp-v2`, `xunit.v3.core.mtp-v1`, `xunit.v3.core.mtp-v2` | xUnit | +| `NUnit` + `NUnit3TestAdapter` | NUnit | +| `TUnit` | TUnit (MTP only) | + +## Detecting the test platform + +The detection logic depends on the .NET SDK version. Run `dotnet --version` to determine it. + +### .NET SDK 10+ + +On .NET 10+, the `global.json` `test.runner` setting is the **authoritative source**: + +- If `global.json` contains `"test": { "runner": "Microsoft.Testing.Platform" }` → **MTP** +- If `global.json` has `"runner": "VSTest"`, or no `test` section exists → **VSTest** + +> **Important**: On .NET 10+, `` alone does **not** switch to MTP. The `global.json` runner setting takes precedence. If the runner is VSTest (or unset), the project uses VSTest regardless of `TestingPlatformDotnetTestSupport`. + +### .NET SDK 8 or 9 + +On older SDKs, check these signals in priority order: + +**1. Check the `` MSBuild property.** Look in the `.csproj`, `Directory.Build.props`, **and** `Directory.Packages.props`. If set to `true` in **any** of these files, the project uses **MTP**. + +> **Critical**: Always read `Directory.Build.props` and `Directory.Packages.props` if they exist. MTP properties are frequently set there instead of in the `.csproj`, so checking only the project file will miss them. + +**2. Check project-level signals:** + +| Signal | Platform | +|--------|----------| +| `` as project SDK | **MTP** by default | +| `true` | **MTP** runner (xUnit) | +| `true` | **MTP** runner (MSTest) | +| `true` | **MTP** runner (NUnit) | +| `Microsoft.Testing.Platform` package referenced directly | **MTP** | +| `TUnit` package referenced | **MTP** (TUnit is MTP-only) | + +> **Note**: The presence of `Microsoft.NET.Test.Sdk` does **not** necessarily mean VSTest. Some frameworks (e.g., MSTest) pull it in transitively for compatibility, even when MTP is enabled. Do not use this package as a signal on its own — always check the MTP signals above first. + +> **Key distinction**: VSTest is the classic platform that uses `vstest.console` under the hood. Microsoft.Testing.Platform (MTP) is the newer, faster platform. Both can be invoked via `dotnet test`, but their filter syntax and CLI options differ. diff --git a/.github/skills/run-tests/skill.md b/.github/skills/run-tests/skill.md new file mode 100644 index 0000000..e9bd695 --- /dev/null +++ b/.github/skills/run-tests/skill.md @@ -0,0 +1,197 @@ +--- +name: run-tests +description: > + Runs .NET tests with dotnet test. Use when user says "run tests", "execute + tests", "dotnet test", "test filter", "filter by category", "filter by + class", "run only specific tests", "tests not running", or needs to + detect the test platform (VSTest or Microsoft.Testing.Platform), identify the + test framework, apply test filters, or troubleshoot test execution failures. + Covers MSTest, xUnit, NUnit, and TUnit across both VSTest and MTP platforms. + Also use for treenode-filter, --filter-class, --filter-trait, and other + framework-specific filter syntax. + DO NOT USE FOR: writing or generating test code, CI/CD pipeline + configuration, or debugging failing test logic. +--- + +# Run .NET Tests + +Detect the test platform and framework, run tests, and apply filters using `dotnet test`. + +## When to Use + +- User wants to run tests in a .NET project +- User needs to run a subset of tests using filters +- User needs help detecting which test platform (VSTest vs MTP) or framework is in use +- User wants to understand the correct filter syntax for their setup + +## When Not to Use + +- User needs to write or generate test code (use `writing-mstest-tests` for MSTest, or general coding assistance for other frameworks) +- User needs to migrate from VSTest to MTP (use `migrate-vstest-to-mtp`) +- User wants to iterate on failing tests without rebuilding (use `mtp-hot-reload`) +- User needs CI/CD pipeline configuration (use CI-specific skills) +- User needs to debug a test (use debugging skills) + +## Inputs + +| Input | Required | Description | +|-------|----------|-------------| +| Project or solution path | No | Path to the test project (.csproj) or solution (.sln). Defaults to current directory. | +| Filter expression | No | Filter expression to select specific tests | +| Target framework | No | Target framework moniker to run against (e.g., `net8.0`) | + +## Workflow + +### Quick Reference + +| Platform | SDK | Command pattern | +|----------|-----|----------------| +| VSTest | Any | `dotnet test [] [--filter ] [--logger trx]` | +| MTP | 8 or 9 | `dotnet test [] -- ` | +| MTP | 10+ | `dotnet test --project ` | + +**Detection files to always check** (in order): `global.json` -> `.csproj` -> `Directory.Build.props` -> `Directory.Packages.props` + +### Step 1: Detect the test platform and framework + +1. Run `dotnet --version` to determine the .NET SDK version +2. Read `global.json`, `.csproj`, `Directory.Build.props`, and `Directory.Packages.props` +3. Follow the detection procedure in [references/platform-detection.md](references/platform-detection.md) to determine: + - **Test framework**: MSTest, xUnit, NUnit, or TUnit + - **Test platform**: VSTest or Microsoft.Testing.Platform (MTP) + +### Step 2: Run tests + +#### VSTest (any .NET SDK version) + +```bash +dotnet test [ | | | | ] +``` + +Common flags: + +| Flag | Description | +|------|-------------| +| `--framework ` | Target a specific framework in multi-TFM projects (e.g., `net8.0`) | +| `--no-build` | Skip build, use previously built output | +| `--filter ` | Run selected tests (see [Step 3](#step-3-run-filtered-tests)) | +| `--logger trx` | Generate TRX results file | +| `--collect "Code Coverage"` | Collect code coverage using Microsoft Code Coverage (built-in, always available) | +| `--blame` | Enable blame mode to detect tests that crash the host | +| `--blame-crash` | Collect a crash dump when the test host crashes | +| `--blame-hang-timeout ` | Abort test if it hangs longer than duration (e.g., `5min`) | +| `-v ` | Verbosity: `quiet`, `minimal`, `normal`, `detailed`, `diagnostic` | + +#### MTP with .NET SDK 8 or 9 + +With `true`, `dotnet test` bridges to MTP but uses VSTest-style argument parsing. MTP-specific arguments must be passed after `--`: + +```bash +dotnet test [ | | | | ] -- +``` + +#### MTP with .NET SDK 10+ + +With the `global.json` runner set to `Microsoft.Testing.Platform`, `dotnet test` natively understands MTP arguments without `--`: + +```bash +dotnet test + [--project ] + [--solution ] + [--test-modules ] + [] +``` + +Examples: + +```bash +# Run all tests in a project +dotnet test --project path/to/MyTests.csproj + +# Run all tests in a directory containing a project +dotnet test --project path/to/ + +# Run all tests in a solution (sln, slnf, slnx) +dotnet test --solution path/to/MySolution.sln + +# Run all tests in a directory containing a solution +dotnet test --solution path/to/ + +# Run with MTP flags +dotnet test --project path/to/MyTests.csproj --report-trx --blame-hang-timeout 5min +``` + +> **Note**: The .NET 10+ `dotnet test` syntax does **not** accept a bare positional argument like the VSTest syntax. Use `--project`, `--solution`, or `--test-modules` to specify the target. + +#### Common MTP flags + +These flags apply to MTP on both SDK versions. On SDK 8/9, pass after `--`; on SDK 10+, pass directly. + +**Built-in flags (always available):** + +| Flag | Description | +|------|-------------| +| `--no-build` | Skip build, use previously built output | +| `--framework ` | Target a specific framework in multi-TFM projects | +| `--results-directory ` | Directory for test result output | +| `--diagnostic` | Enable diagnostic logging for the test platform | +| `--diagnostic-output-directory ` | Directory for diagnostic log output | + +**Extension-dependent flags (require the corresponding extension package to be registered):** + +| Flag | Requires | Description | +|------|----------|-------------| +| `--filter ` | Framework-specific (not all frameworks support this) | Run selected tests (see [Step 3](#step-3-run-filtered-tests)) | +| `--report-trx` | `Microsoft.Testing.Extensions.TrxReport` | Generate TRX results file | +| `--report-trx-filename ` | `Microsoft.Testing.Extensions.TrxReport` | Set TRX output filename | +| `--blame-hang-timeout ` | `Microsoft.Testing.Extensions.HangDump` | Abort test if it hangs longer than duration (e.g., `5min`) | +| `--blame-crash` | `Microsoft.Testing.Extensions.CrashDump` | Collect a crash dump when the test host crashes | +| `--coverage` | `Microsoft.Testing.Extensions.CodeCoverage` | Collect code coverage using Microsoft Code Coverage | + +> Some frameworks (e.g., MSTest) bundle common extensions by default. Others may require explicit package references. If a flag is not recognized, check that the corresponding extension package is referenced in the project. + +#### Alternative MTP invocations + +MTP test projects are standalone executables. Beyond `dotnet test`, they can be run directly: + +```bash +# Build and run +dotnet run --project + +# Run a previously built DLL +dotnet exec + +# Run the executable directly (Windows) + +``` + +These alternative invocations accept MTP command line arguments directly (no `--` separator needed). + +### Step 3: Run filtered tests + +See [references/filter-syntax.md](references/filter-syntax.md) for the complete filter syntax for each platform and framework combination. Key points: + +- **VSTest** (MSTest, xUnit v2, NUnit): `dotnet test --filter ` with `=`, `!=`, `~`, `!~` operators +- **MTP -- MSTest and NUnit**: Same `--filter` syntax as VSTest; pass after `--` on SDK 8/9, directly on SDK 10+ +- **MTP -- xUnit v3**: Uses `--filter-class`, `--filter-method`, `--filter-trait` (not VSTest expression syntax) +- **MTP -- TUnit**: Uses `--treenode-filter` with path-based syntax + +## Validation + +- [ ] Test platform (VSTest or MTP) was correctly identified +- [ ] Test framework (MSTest, xUnit, NUnit, TUnit) was correctly identified +- [ ] Correct `dotnet test` invocation was used for the detected platform and SDK version +- [ ] Filter expressions used the syntax appropriate for the platform and framework +- [ ] Test results were clearly reported to the user + +## Common Pitfalls + +| Pitfall | Solution | +|---------|----------| +| Missing `Microsoft.NET.Test.Sdk` in a VSTest project | Tests won't be discovered. Add `` | +| Using VSTest `--filter` syntax with xUnit v3 on MTP | xUnit v3 on MTP uses `--filter-class`, `--filter-method`, etc. -- not the VSTest expression syntax | +| Passing MTP args without `--` on .NET SDK 8/9 | Before .NET 10, MTP args must go after `--`: `dotnet test -- --report-trx` | +| Using `--` for MTP args on .NET SDK 10+ | On .NET 10+, MTP args are passed directly: `dotnet test --report-trx` (using `--` still works but is unnecessary) | +| Multi-TFM project runs tests for all frameworks | Use `--framework ` to target a specific framework | +| `global.json` runner setting ignored | Requires .NET 10+ SDK. On older SDKs, use `` MSBuild property instead | +| TUnit `--treenode-filter` not recognized | TUnit is MTP-only. On .NET SDK 10+ use `dotnet test`; on older SDKs use `dotnet run` since VSTest-mode `dotnet test` does not support TUnit | diff --git a/.github/skills/test-anti-patterns/skill.md b/.github/skills/test-anti-patterns/skill.md new file mode 100644 index 0000000..8c4e9d9 --- /dev/null +++ b/.github/skills/test-anti-patterns/skill.md @@ -0,0 +1,136 @@ +--- +name: test-anti-patterns +description: "Detects anti-patterns and code smells in .NET test suites. Use when the user asks to review test quality, find test smells, identify flaky test indicators, or audit tests for common mistakes. Covers assertion quality, test isolation, naming, flakiness indicators, over-mocking, and structural problems. Works with MSTest, xUnit, NUnit, and TUnit." +--- + +# Test Anti-Pattern Detection + +Analyze .NET test code for anti-patterns, code smells, and quality issues that undermine test reliability, maintainability, and diagnostic value. + +## When to Use + +- User asks to review test quality or find test smells +- User wants to know why tests are flaky or unreliable +- User asks "are my tests good?" or "what's wrong with my tests?" +- User requests a test audit or test code review +- User wants to improve existing test code + +## When Not to Use + +- User wants to write new tests from scratch (use `writing-mstest-tests`) +- User wants to run or execute tests (use `run-tests`) +- User wants to migrate between test frameworks or versions (use migration skills) +- User wants to measure code coverage (out of scope) + +## Inputs + +| Input | Required | Description | +|-------|----------|-------------| +| Test code | Yes | One or more test files or classes to analyze | +| Production code | No | The code under test, for context on what tests should verify | +| Specific concern | No | A focused area like "flakiness" or "naming" to narrow the review | + +## Workflow + +### Step 1: Gather the test code + +Read the test files the user wants reviewed. If the user points to a directory or project, scan for all test files (files containing `[TestClass]`, `[TestMethod]`, `[Fact]`, `[Test]`, or `[Theory]` attributes). + +If production code is available, read it too -- this is critical for detecting tests that are coupled to implementation details rather than behavior. + +### Step 2: Scan for anti-patterns + +Check each test file against the anti-pattern catalog below. Report findings grouped by severity. + +#### Critical -- Tests that give false confidence + +| Anti-Pattern | What to Look For | +|---|---| +| **No assertions** | Test methods that execute code but never assert anything. A passing test without assertions proves nothing. | +| **Swallowed exceptions** | `try { ... } catch { }` or `catch (Exception)` without rethrowing or asserting. Failures are silently hidden. | +| **Assert in catch block only** | `try { Act(); } catch (Exception ex) { Assert.Fail(ex.Message); }` -- use `Assert.ThrowsException` or equivalent instead. The test passes when no exception is thrown even if the result is wrong. | +| **Always-true assertions** | `Assert.IsTrue(true)`, `Assert.AreEqual(x, x)`, or conditions that can never fail. | +| **Commented-out assertions** | Assertions that were disabled but the test still runs, giving the illusion of coverage. | + +#### High -- Tests likely to cause pain + +| Anti-Pattern | What to Look For | +|---|---| +| **Flakiness indicators** | `Thread.Sleep(...)`, `Task.Delay(...)` for synchronization, `DateTime.Now`/`DateTime.UtcNow` without abstraction, `Random` without a seed, environment-dependent paths. | +| **Test ordering dependency** | Static mutable fields modified across tests, `[TestInitialize]` that doesn't fully reset state, tests that fail when run individually but pass in suite (or vice versa). | +| **Over-mocking** | More mock setup lines than actual test logic. Verifying exact call sequences on mocks rather than outcomes. Mocking types the test owns. | +| **Implementation coupling** | Testing private methods via reflection, asserting on internal state, verifying exact method call counts on collaborators instead of observable behavior. | +| **Broad exception assertions** | `Assert.ThrowsException(...)` instead of the specific exception type. Also: `[ExpectedException(typeof(Exception))]`. | + +#### Medium -- Maintainability and clarity issues + +| Anti-Pattern | What to Look For | +|---|---| +| **Poor naming** | Test names like `Test1`, `TestMethod`, names that don't describe the scenario or expected outcome. Good: `Add_NegativeNumber_ThrowsArgumentException`. | +| **Magic values** | Unexplained numbers or strings in arrange/assert: `Assert.AreEqual(42, result)` -- what does 42 mean? | +| **Duplicate tests** | Three or more test methods with near-identical bodies that differ only in a single input value. Should be data-driven (`[DataRow]`, `[Theory]`, `[TestCase]`). Note: Two tests covering distinct boundary conditions (e.g., zero vs. negative) are NOT duplicates -- separate tests for different edge cases provide clearer failure diagnostics and are a valid practice. | +| **Giant tests** | Test methods exceeding ~30 lines or testing multiple behaviors at once. Hard to diagnose when they fail. | +| **Assertion messages that repeat the assertion** | `Assert.AreEqual(expected, actual, "Expected and actual are not equal")` adds no information. Messages should describe the business meaning. | +| **Missing AAA separation** | Arrange, Act, Assert phases are interleaved or indistinguishable. | + +#### Low -- Style and hygiene + +| Anti-Pattern | What to Look For | +|---|---| +| **Unused test infrastructure** | `[TestInitialize]`/`[SetUp]` that does nothing, test helper methods that are never called. | +| **IDisposable not disposed** | Test creates `HttpClient`, `Stream`, or other disposable objects without `using` or cleanup. | +| **Console.WriteLine debugging** | Leftover `Console.WriteLine` or `Debug.WriteLine` statements used during test development. | +| **Inconsistent naming convention** | Mix of naming styles in the same test class (e.g., some use `Method_Scenario_Expected`, others use `ShouldDoSomething`). | + +### Step 3: Calibrate severity honestly + +Before reporting, re-check each finding against these severity rules: + +- **Critical/High**: Only for issues that cause tests to give false confidence or be unreliable. A test that always passes regardless of correctness is Critical. Flaky shared state is High. +- **Medium**: Only for issues that actively harm maintainability -- 5+ nearly-identical tests, truly meaningless names like `Test1`. +- **Low**: Cosmetic naming mismatches, minor style preferences, assertion messages that could be better. When in doubt, rate Low. +- **Not an issue**: Separate tests for distinct boundary conditions (zero vs. negative vs. null). Explicit per-test setup instead of `[TestInitialize]` (this *improves* isolation). Tests that are short and clear but could theoretically be consolidated. + +IMPORTANT: If the tests are well-written, say so clearly up front. Do not inflate severity to justify the review. A review that finds zero Critical/High issues and only minor Low suggestions is a valid and valuable outcome. Lead with what the tests do well. + +### Step 4: Report findings + +Present findings in this structure: + +1. **Summary** -- Total issues found, broken down by severity (Critical / High / Medium / Low). If tests are well-written, lead with that assessment. +2. **Critical and High findings** -- List each with: + - The anti-pattern name + - The specific location (file, method name, line) + - A brief explanation of why it's a problem + - A concrete fix (show before/after code when helpful) +3. **Medium and Low findings** -- Summarize in a table unless the user wants full detail +4. **Positive observations** -- Call out things the tests do well (sealed class, specific exception types, data-driven tests, clear AAA structure, proper use of fakes, good naming). Don't only report negatives. + +### Step 5: Prioritize recommendations + +If there are many findings, recommend which to fix first: + +1. **Critical** -- Fix immediately, these tests may be giving false confidence +2. **High** -- Fix soon, these cause flakiness or maintenance burden +3. **Medium/Low** -- Fix opportunistically during related edits + +## Validation + +- [ ] Every finding includes a specific location (not just a general warning) +- [ ] Every Critical/High finding includes a concrete fix +- [ ] Report covers all categories (assertions, isolation, naming, structure) +- [ ] Positive observations are included alongside problems +- [ ] Recommendations are prioritized by severity + +## Common Pitfalls + +| Pitfall | Solution | +|---------|----------| +| Reporting style issues as critical | Naming and formatting are Medium/Low, never Critical | +| Suggesting rewrites instead of targeted fixes | Show minimal diffs -- change the assertion, not the whole test | +| Flagging intentional design choices | If `Thread.Sleep` is in an integration test testing actual timing, that's not an anti-pattern. Consider context. | +| Inventing false positives on clean code | If tests follow best practices, say so. A review finding "0 Critical, 0 High, 1 Low" is perfectly valid. Don't inflate findings to justify the review. | +| Flagging separate boundary tests as duplicates | Two tests for zero and negative inputs test different edge cases. Only flag as duplicates when 3+ tests have truly identical bodies differing by a single value. | +| Rating cosmetic issues as Medium | Naming mismatches (e.g., method name says `ArgumentException` but asserts `ArgumentOutOfRangeException`) are Low, not Medium -- the test still works correctly. | +| Ignoring the test framework | xUnit uses `[Fact]`/`[Theory]`, NUnit uses `[Test]`/`[TestCase]`, MSTest uses `[TestMethod]`/`[DataRow]` -- use correct terminology | +| Missing the forest for the trees | If 80% of tests have no assertions, lead with that systemic issue rather than listing every instance | diff --git a/.gitignore b/.gitignore index e33e025..ab415b1 100644 --- a/.gitignore +++ b/.gitignore @@ -361,5 +361,5 @@ MigrationBackup/ # Fody - auto-generated XML schema FodyWeavers.xsd -/SimpleRetryTools/SimpleRetry.xml + /src/SimpleRetry/SimpleRetry.xml diff --git a/README.md b/README.md index aeb4a2f..644c797 100644 --- a/README.md +++ b/README.md @@ -1 +1,273 @@ -# WebApplication1 \ No newline at end of file +# SimpleRetry + +SimpleRetry is a small .NET library for executing asynchronous operations with a configurable retry policy. + +It provides an `IRetryExecutor` service that can retry failed operations, retry returned results, apply a per-attempt timeout, calculate retry delays using different backoff strategies, and run custom logic before each retry. + +## Features + +- Execute asynchronous operations with retry support. +- Configure the maximum number of retry attempts. +- Configure a delay between retries. +- Choose a backoff strategy: + - `Constant` + - `Linear` + - `Exponential` +- Apply a timeout to each operation attempt. +- Decide which exceptions or returned results should be retried with `ShouldHandle`. +- Override the delay for a specific outcome with `RetryDelayGenerator`. +- Dispose or otherwise release returned results that are discarded before a retry. +- Run custom logic before each retry with `OnRetry`. +- Register multiple keyed retry policies with dependency injection. +- Add retry support to `HttpClient` with `AddHttpSimpleRetry`. + +## Configuration + +Register retry policies with `AddSimpleRetry`: + +```csharp +using SimpleRetry; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddSimpleRetry("ExternalApi", options => +{ + options.MaxRetryCount = 3; + options.RetryDelay = TimeSpan.FromSeconds(2); + options.AttemptTimeout = TimeSpan.FromSeconds(10); + options.BackoffType = BackoffType.Linear; + options.ShouldHandle = outcome => outcome.Exception is HttpRequestException; + options.OnRetry = arguments => + { + Console.WriteLine($"Retry {arguments.AttemptNumber} of {arguments.MaxRetryCount} after {arguments.RetryDelay} because of {arguments.Outcome.Exception?.Message}"); + return Task.CompletedTask; + }; +}); +``` + +### Options + +| Option | Default | Description | +| --- | --- | --- | +| `MaxRetryCount` | `3` | Maximum number of retry attempts after the first failed execution. | +| `RetryDelay` | `TimeSpan.FromSeconds(2)` | Base delay used between retry attempts. | +| `AttemptTimeout` | `null` | Maximum duration allowed for each operation attempt. `null` disables the timeout. | +| `BackoffType` | `BackoffType.Constant` | Strategy used to calculate the delay before the next retry. | +| `ShouldHandle` | `outcome => outcome.IsException` | Predicate that determines whether an exception or returned result should be retried. | +| `OnRetry` | `null` | Optional callback invoked before each retry attempt. | +| `RetryDelayGenerator` | `null` | Optional callback that can override the delay for a specific retry outcome. Return `null` to use `RetryDelay` and `BackoffType`. | +| `OnResultDiscarded` | `null` | Optional callback invoked when a returned result is handled and discarded because a retry is about to start. | + +## Retry outcomes + +`ShouldHandle` receives a `RetryOutcome`. The outcome can represent either an exception or a returned result. + +```csharp +options.ShouldHandle = outcome => outcome switch +{ + { Exception: HttpRequestException or TimeoutException } => true, + { Result: HttpResponseMessage { IsSuccessStatusCode: false } } => true, + _ => false +}; +``` + +You can also inspect the result by type: + +```csharp +options.ShouldHandle = outcome => outcome.Exception is HttpRequestException + || (outcome.TryGetResult(out HttpResponseMessage? response) && !response.IsSuccessStatusCode); +``` + +If an operation returns a handled result and retries are exhausted, the last result is returned to the caller. + +## Backoff strategies + +Given `RetryDelay = TimeSpan.FromSeconds(2)`, the retry delays are calculated as follows: + +| Backoff type | Attempt 1 | Attempt 2 | Attempt 3 | +| --- | --- | --- | --- | +| `Constant` | 2s | 2s | 2s | +| `Linear` | 2s | 4s | 6s | +| `Exponential` | 2s | 4s | 8s | + +## Usage + +Inject the keyed `IRetryExecutor` that matches the policy you registered: + +```csharp +using Microsoft.Extensions.DependencyInjection; +using SimpleRetry; + +app.MapGet("/api/weather", async ([FromKeyedServices("ExternalApi")] IRetryExecutor retryExecutor, CancellationToken cancellationToken) => +{ + await retryExecutor.ExecuteAsync(async token => + { + using var httpClient = new HttpClient(); + using var response = await httpClient.GetAsync("https://example.com/weather", token); + response.EnsureSuccessStatusCode(); + }, cancellationToken); + + return Results.Ok(); +}); +``` + +You can retry based on the returned value: + +```csharp +app.MapGet("/api/weather", async ([FromKeyedServices("ExternalApi")] IRetryExecutor retryExecutor, CancellationToken cancellationToken) => +{ + using var response = await retryExecutor.ExecuteAsync(async token => + { + using var httpClient = new HttpClient(); + return await httpClient.GetAsync("https://example.com/weather", token); + }, cancellationToken); + + return Results.StatusCode((int)response.StatusCode); +}); +``` + +Use the generic overload when the operation returns a value: + +```csharp +app.MapGet("/api/value", async ([FromKeyedServices("ExternalApi")] IRetryExecutor retryExecutor, CancellationToken cancellationToken) => +{ + var value = await retryExecutor.ExecuteAsync(async token => + { + await Task.Delay(TimeSpan.FromMilliseconds(100), token); + return 42; + }, cancellationToken); + + return Results.Ok(value); +}); +``` + +## Multiple policies + +You can register more than one retry policy by using different service keys: + +```csharp +builder.Services + .AddSimpleRetry("ExternalApi", options => + { + options.MaxRetryCount = 3; + options.RetryDelay = TimeSpan.FromSeconds(2); + options.BackoffType = BackoffType.Exponential; + options.ShouldHandle = outcome => outcome.Exception is HttpRequestException; + }) + .AddSimpleRetry("Database", options => + { + options.MaxRetryCount = 5; + options.RetryDelay = TimeSpan.FromMilliseconds(200); + options.BackoffType = BackoffType.Linear; + }); +``` + +Then resolve the policy you need: + +```csharp +public sealed class MyService([FromKeyedServices("Database")] IRetryExecutor retryExecutor) +{ + public Task SaveAsync(CancellationToken cancellationToken) + => retryExecutor.ExecuteAsync(async token => + { + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + }, cancellationToken); +} +``` + +## Per-attempt timeout + +`AttemptTimeout` limits each individual execution attempt. It is not a total timeout for the whole retry operation. + +For example, with `MaxRetryCount = 2` and `AttemptTimeout = TimeSpan.FromSeconds(3)`, the operation may be attempted up to three times, and each attempt can run for up to three seconds. + +When an attempt exceeds the configured timeout, SimpleRetry throws a `RetryTimeoutException`. Timeout failures caused by `AttemptTimeout` are retryable even when `ShouldHandle` returns `false`. + +The timeout is cooperative: SimpleRetry creates a linked cancellation token for each attempt and cancels it when the attempt timeout expires. The operation receives that token and can release its resources normally. + +Caller cancellation is still treated differently. If the caller's `CancellationToken` is canceled, `OperationCanceledException` is propagated immediately and no further retry is attempted. + +## Custom retry delays + +Use `RetryDelayGenerator` when the delay depends on the failure or result: + +```csharp +options.RetryDelayGenerator = outcome => +{ + if (outcome.Result is HttpResponseMessage response && response.Headers.RetryAfter?.Delta is TimeSpan retryAfter) + { + return retryAfter; + } + + return null; +}; +``` + +Returning `null` uses the normal `RetryDelay` and `BackoffType` calculation. + +## Discarded results + +When a returned result is handled by the policy, SimpleRetry retries the operation and discards that result. Use `OnResultDiscarded` to release resources owned by discarded values: + +```csharp +options.OnResultDiscarded = outcome => +{ + if (outcome.Result is IDisposable disposable) + { + disposable.Dispose(); + } +}; +``` + +The callback is invoked only for handled results. Exceptions are not reported through `OnResultDiscarded`. + +## HTTP retries + +Use `AddHttpSimpleRetry` to add the built-in HTTP retry handler to an `HttpClient`: + +```csharp +builder.Services.AddHttpClient("ExternalApi", client => +{ + client.BaseAddress = new("https://example.com"); +}) +.AddHttpSimpleRetry(options => +{ + options.MaxRetryCount = 3; +}); +``` + +The HTTP policy is registered as a keyed retry policy using the HTTP client name. For example, the previous registration stores its `RetryPolicyOptions` and `IRetryExecutor` under the `"ExternalApi"` key. + +The default HTTP policy: + +- Retries `HttpRequestException`. +- Retries `RetryTimeoutException` caused by `AttemptTimeout`. +- Retries HTTP `408 Request Timeout`, `429 Too Many Requests`, and `5xx` responses. +- Uses `AttemptTimeout = TimeSpan.FromSeconds(10)`. +- Uses `RetryDelay = TimeSpan.FromSeconds(2)`. +- Uses `BackoffType.Exponential`. +- Honors `Retry-After` when the response contains either a delta or a date. +- Disposes handled `HttpResponseMessage` instances that are discarded before retrying. + +If a response does not contain `Retry-After`, the handler falls back to the configured `RetryDelay` and `BackoffType`. + +### Request handling across attempts + +Each HTTP attempt resends the original `HttpRequestMessage`, reusing its `HttpContent` instance, so nothing is copied when a retry is not needed. This mirrors the behavior of the standard `Microsoft.Extensions.Http.Resilience` handler. + +The trade-off of not cloning is that any mutation applied by the inner handlers accumulates across attempts: headers can end up duplicated or overwritten, and a retry follows the URI that a redirect handler rewrote on the message instead of the original one. + +Because the body must be replayable across attempts, a request whose content is a `StreamContent` is rejected with an `InvalidOperationException`: its source stream is consumed by the first attempt and may not be seekable. + +## Cancellation + +Pass a `CancellationToken` to stop the current operation or the delay before the next retry: + +```csharp +await retryExecutor.ExecuteAsync(async cancellationToken => +{ + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); +}, cancellationToken); +``` + +When cancellation is requested, `OperationCanceledException` is propagated immediately and no further retry is attempted. diff --git a/SimpleRetry.slnx b/SimpleRetry.slnx index 7f5befe..9d0563f 100644 --- a/SimpleRetry.slnx +++ b/SimpleRetry.slnx @@ -2,8 +2,10 @@ + + - + diff --git a/global.json b/global.json new file mode 100644 index 0000000..3140116 --- /dev/null +++ b/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/samples/WebApplication1/Program.cs b/samples/SimpleRetrySample/Program.cs similarity index 67% rename from samples/WebApplication1/Program.cs rename to samples/SimpleRetrySample/Program.cs index 6c5743f..65cfc8a 100644 --- a/samples/WebApplication1/Program.cs +++ b/samples/SimpleRetrySample/Program.cs @@ -10,11 +10,19 @@ options.MaxRetryCount = 3; options.RetryDelay = TimeSpan.FromSeconds(2); options.BackoffType = BackoffType.Linear; - options.ShouldHandle = ex => ex is HttpRequestException; // Only retry on HttpRequestException, + options.ShouldHandle = outcome => outcome switch + { + { Exception: HttpRequestException or TaskCanceledException } => true, + //{ Exception: TaskCanceledException { InnerException: TimeoutException } } => true, + { Result: HttpResponseMessage { IsSuccessStatusCode: false } } => true, + _ => false + }; + //options.ShouldHandle = outcome => outcome.Exception is HttpRequestException + // || (outcome.TryGetResult(out HttpResponseMessage? response) && response?.IsSuccessStatusCode == false); options.OnRetry = args => { // Handle the retry event (e.g., logging) - Console.WriteLine($"Retry {args.AttemptNumber} of {args.MaxRetryCount} after {args.RetryDelay} due to {args.Exception?.Message}"); + Console.WriteLine($"Retry {args.AttemptNumber} of {args.MaxRetryCount} after {args.RetryDelay} due to {args.Outcome.Exception?.Message}"); return Task.CompletedTask; }; @@ -24,6 +32,8 @@ options.RetryDelay = TimeSpan.FromSeconds(1); }); +builder.Services.AddHttpClient().AddHttpSimpleRetry(); + var app = builder.Build(); // Configure the HTTP request pipeline. diff --git a/samples/WebApplication1/Properties/launchSettings.json b/samples/SimpleRetrySample/Properties/launchSettings.json similarity index 100% rename from samples/WebApplication1/Properties/launchSettings.json rename to samples/SimpleRetrySample/Properties/launchSettings.json diff --git a/samples/WebApplication1/WebApplication1.csproj b/samples/SimpleRetrySample/SimpleRetrySample.csproj similarity index 100% rename from samples/WebApplication1/WebApplication1.csproj rename to samples/SimpleRetrySample/SimpleRetrySample.csproj diff --git a/samples/WebApplication1/appsettings.Development.json b/samples/SimpleRetrySample/appsettings.Development.json similarity index 100% rename from samples/WebApplication1/appsettings.Development.json rename to samples/SimpleRetrySample/appsettings.Development.json diff --git a/samples/WebApplication1/appsettings.json b/samples/SimpleRetrySample/appsettings.json similarity index 100% rename from samples/WebApplication1/appsettings.json rename to samples/SimpleRetrySample/appsettings.json diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 0000000..268fee9 --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,15 @@ + + + + $(MSBuildThisFileDirectory)artifacts + true + embedded + true + true + + + + + + + diff --git a/src/SimpleRetry/DefaultRetryExecutor.cs b/src/SimpleRetry/DefaultRetryExecutor.cs index fc504bf..b62ec3b 100644 --- a/src/SimpleRetry/DefaultRetryExecutor.cs +++ b/src/SimpleRetry/DefaultRetryExecutor.cs @@ -4,19 +4,30 @@ namespace SimpleRetry; internal class DefaultRetryExecutor(RetryPolicyOptions options, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) : IRetryExecutor { - /// public async Task ExecuteAsync(Func operation, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(operation); - await ExecuteAsync(async retryCancellationToken => + var attempt = 0; + + while (true) { - await operation(retryCancellationToken).ConfigureAwait(false); - return null; - }, cancellationToken).ConfigureAwait(false); + try + { + await ExecuteOperationAsync(operation, cancellationToken).ConfigureAwait(false); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) when (ShouldRetry(RetryOutcome.FromException(exception), attempt)) + { + await WaitForNextAttemptAsync(RetryOutcome.FromException(exception), ++attempt, cancellationToken).ConfigureAwait(false); + } + } } - /// public async Task ExecuteAsync(Func> operation, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(operation); @@ -25,28 +36,87 @@ public async Task ExecuteAsync(Func> operation, while (true) { + RetryOutcome outcome; + try { - return await operation(cancellationToken).ConfigureAwait(false); + var result = await ExecuteOperationAsync(operation, cancellationToken).ConfigureAwait(false); + + outcome = RetryOutcome.FromResult(result); + + if (!ShouldRetry(outcome, attempt)) + { + return result; + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } - catch (Exception exception) when (attempt < options.MaxRetryCount && (options.ShouldHandle?.Invoke(exception) ?? true)) + catch (Exception exception) when (ShouldRetry(RetryOutcome.FromException(exception), attempt)) { - attempt++; + outcome = RetryOutcome.FromException(exception); + } - var retryDelay = GetRetryDelay(attempt); - await OnRetryAsync(attempt, retryDelay, exception).ConfigureAwait(false); + await WaitForNextAttemptAsync(outcome, ++attempt, cancellationToken).ConfigureAwait(false); + } + } - await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); - } + private Task ExecuteOperationAsync(Func operation, CancellationToken cancellationToken) + => ExecuteOperationAsync(async token => + { + await operation(token).ConfigureAwait(false); + return null; + }, cancellationToken); + + private async Task ExecuteOperationAsync(Func> operation, CancellationToken cancellationToken) + { + // The attempt timeout is applied by cancelling a linked token instead of just giving up on the returned task, + // so that the operation itself observes the cancellation and can release its resources. + if (options.AttemptTimeout is not TimeSpan attemptTimeout) + { + return await operation(cancellationToken).ConfigureAwait(false); + } + + // The linked source merges the caller cancellation with the attempt timeout into a single token, so the operation + // is cancelled by whichever happens first, and it can be cancelled without touching the caller token, which is not owned here. + // It is created per attempt, so every retry starts with a fresh timeout and the registration on the caller token is released by the using. + using var timeoutCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCancellation.CancelAfter(attemptTimeout); + + try + { + return await operation(timeoutCancellation.Token).ConfigureAwait(false); + } + // Only the linked source being cancelled means the timeout expired; if the caller token is cancelled too, + // the original exception is propagated as-is, so the cancellation is not turned into a retriable timeout. + catch (OperationCanceledException exception) when (timeoutCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new RetryTimeoutException(attemptTimeout, exception); } } - private Task OnRetryAsync(int attempt, TimeSpan retryDelay, Exception exception) - => options.OnRetry?.Invoke(new(attempt, options.MaxRetryCount, retryDelay, exception, serviceProvider, loggerFactory)) ?? Task.CompletedTask; + // A timeout is produced by the policy itself, so it is always retried regardless of the configured + // predicate, which would otherwise have to know about an exception type it never throws. + private bool ShouldRetry(RetryOutcome outcome, int attempt) + => attempt < options.MaxRetryCount && (outcome.Exception is RetryTimeoutException || (options.ShouldHandle?.Invoke(outcome) ?? true)); + + private async Task WaitForNextAttemptAsync(RetryOutcome outcome, int attempt, CancellationToken cancellationToken) + { + var retryDelay = options.RetryDelayGenerator?.Invoke(outcome) ?? GetRetryDelay(attempt); + + if (options.OnRetry is not null) + { + await options.OnRetry(new(attempt, options.MaxRetryCount, retryDelay, outcome, serviceProvider, loggerFactory)).ConfigureAwait(false); + } + + if (!outcome.IsException) + { + options.OnResultDiscarded?.Invoke(outcome); + } + + await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); + } private TimeSpan GetRetryDelay(int attempt) => options.BackoffType switch { diff --git a/src/SimpleRetry/HttpRetryDelegatingHandler.cs b/src/SimpleRetry/HttpRetryDelegatingHandler.cs new file mode 100644 index 0000000..faa87fb --- /dev/null +++ b/src/SimpleRetry/HttpRetryDelegatingHandler.cs @@ -0,0 +1,152 @@ +using System.Net; + +namespace SimpleRetry; + +/// +/// Retries transient HTTP failures by replaying the outgoing request through the configured . +/// +/// The executor that applies the retry policy to each send attempt. +/// +/// to buffer non-replayable request bodies in memory so that they can be sent again on every +/// attempt; to reject such requests up front. +/// +/// +/// to send a fresh copy of the request message on every attempt, so that mutations applied +/// by the inner handlers (added headers, rewritten URIs) never leak into the following attempts; +/// to send the very same instance every time. +/// +/// +/// +/// By default the handler resends the original request message through a plain base.SendAsync(request, …), +/// exactly like the standard Microsoft.Extensions.Http.Resilience handler: no message is allocated per +/// attempt. The cost is that every mutation applied by the inner handlers +/// accumulates across attempts: headers can end up duplicated or overwritten, and a retry follows the URI that a +/// redirect handler rewrote on the message instead of the original one. Setting +/// trades those extra allocations for a clean request state on every attempt. +/// +/// +/// Either way the caller's instance is reused instead of being copied, so the body must be +/// replayable: a consumes its source stream during the first send, so it can only be +/// retried when materializes it in memory beforehand. +/// +/// +internal sealed class HttpRetryDelegatingHandler(IRetryExecutor executor, bool bufferRequestContent = false, bool cloneRequest = false) : DelegatingHandler +{ + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + if (request.Content is StreamContent content) + { + if (!bufferRequestContent) + { + throw new InvalidOperationException($"{nameof(StreamContent)} content cannot be cloned, because its source stream is consumed by the first attempt and may not be seekable. Enable request content buffering to retry requests with this kind of content."); + } + + // Serializing the content into its own internal buffer makes every later send replay the buffered bytes + // instead of the source stream, so the same HttpContent instance can be reused by all the attempts. +#if NET9_0_OR_GREATER + await content.LoadIntoBufferAsync(cancellationToken).ConfigureAwait(false); +#else + await content.LoadIntoBufferAsync().WaitAsync(cancellationToken).ConfigureAwait(false); +#endif + } + + return await executor.ExecuteAsync(async attemptCancellationToken => + { + if (!cloneRequest) + { + // Resending the original message allocates nothing, at the cost of carrying over whatever the inner + // handlers changed on it during the previous attempt. + return await base.SendAsync(request, attemptCancellationToken).ConfigureAwait(false); + } + + // Each attempt sends a new request instance that preserves the original method, URI, headers, options, + // version and body, so that nothing the inner handlers change survives into the following attempts. + var attemptRequest = CloneRequest(request); + + try + { + return await base.SendAsync(attemptRequest, attemptCancellationToken).ConfigureAwait(false); + } + finally + { + // Disposing an HttpRequestMessage disposes its content, which here belongs to the caller and must + // survive both the remaining attempts and the caller's own usage, so detach it first. + attemptRequest.Content = null; + attemptRequest.Dispose(); + } + }, cancellationToken).ConfigureAwait(false); + } + + /// + /// Determines whether the outcome of an HTTP attempt is transient and can be retried. + /// + /// + /// This is the default used by the standard HTTP resilience + /// handler. Because it is expressed as an outcome predicate, callers can replace it entirely to change both + /// the handled exceptions and the handled status codes. + /// + internal static bool ShouldHandle(RetryOutcome outcome) => outcome switch + { + { Exception: HttpRequestException or RetryTimeoutException } => true, + { Result: HttpResponseMessage response } => response.StatusCode is HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests or >= HttpStatusCode.InternalServerError, + _ => false + }; + + /// + /// Returns the delay requested by the server through the Retry-After response header, if any. + /// + /// + /// This is the default used by the standard HTTP + /// resilience handler. Returning lets the configured backoff decide the delay. + /// + internal static TimeSpan? GetRetryAfterDelay(RetryOutcome outcome) + { + if (outcome.Result is not HttpResponseMessage response) + { + return null; + } + + return response.Headers.RetryAfter switch + { + { Delta: TimeSpan delta } => Max(delta, TimeSpan.Zero), + { Date: DateTimeOffset date } => Max(date - DateTimeOffset.UtcNow, TimeSpan.Zero), + _ => null + }; + + static TimeSpan Max(TimeSpan value, TimeSpan minimum) => value < minimum ? minimum : value; + } + + /// + /// Releases a response that is being retried and will therefore never be returned to the caller. + /// + /// + /// This is the default used by the standard HTTP resilience + /// handler; without it, every retried response would keep its connection and stream alive until collected. + /// + internal static void DisposeDiscardedResponse(RetryOutcome outcome) + => (outcome.Result as HttpResponseMessage)?.Dispose(); + + private static HttpRequestMessage CloneRequest(HttpRequestMessage request) + { + var clone = new HttpRequestMessage(request.Method, request.RequestUri) + { + Content = request.Content, + Version = request.Version, + VersionPolicy = request.VersionPolicy + }; + + foreach (var header in request.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + foreach (var option in request.Options) + { + clone.Options.Set(new HttpRequestOptionsKey(option.Key), option.Value); + } + + return clone; + } +} diff --git a/src/SimpleRetry/OnRetryArguments.cs b/src/SimpleRetry/OnRetryArguments.cs index 3650ee1..c4dfb3e 100644 --- a/src/SimpleRetry/OnRetryArguments.cs +++ b/src/SimpleRetry/OnRetryArguments.cs @@ -8,10 +8,10 @@ namespace SimpleRetry; /// The current retry attempt number. /// The maximum number of retry attempts configured for the operation. /// The delay before the next retry attempt. -/// The exception that caused the retry. +/// The outcome that caused the retry. /// The service provider associated with the retry executor. /// The logger factory available to retry callbacks. -public sealed class OnRetryArguments(int attemptNumber, int maxRetryCount, TimeSpan retryDelay, Exception exception, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) +public sealed class OnRetryArguments(int attemptNumber, int maxRetryCount, TimeSpan retryDelay, RetryOutcome outcome, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) { /// /// Gets the current retry attempt number. @@ -29,9 +29,9 @@ public sealed class OnRetryArguments(int attemptNumber, int maxRetryCount, TimeS public TimeSpan RetryDelay { get; } = retryDelay; /// - /// Gets the exception that caused the retry. + /// Gets the outcome that caused the retry, which can be either an exception or a handled result. /// - public Exception Exception { get; } = exception; + public RetryOutcome Outcome { get; } = outcome; /// /// Gets the service provider associated with the retry executor. diff --git a/src/SimpleRetry/RetryOutcome.cs b/src/SimpleRetry/RetryOutcome.cs new file mode 100644 index 0000000..80b2817 --- /dev/null +++ b/src/SimpleRetry/RetryOutcome.cs @@ -0,0 +1,81 @@ +using System.Diagnostics.CodeAnalysis; + +namespace SimpleRetry; + +/// +/// Represents the outcome of a single execution attempt, which is either a result or an exception. +/// +/// +/// This type mirrors the outcome-based model used by reactive resilience strategies, so that a retry can be +/// triggered not only by a thrown exception but also by a result that the caller considers a failure. +/// +/// +public readonly struct RetryOutcome +{ + private RetryOutcome(object? result, Exception? exception) + { + Result = result; + Exception = exception; + } + + /// + /// Gets the exception thrown by the operation, or if the operation completed successfully. + /// + public Exception? Exception { get; } + + /// + /// Gets the value returned by the operation, or if the operation threw an exception + /// or returned no value. + /// + public object? Result { get; } + + /// + /// Gets a value indicating whether the operation failed with an exception. + /// + [MemberNotNullWhen(true, nameof(Exception))] + public bool IsException => Exception is not null; + + /// + /// Creates an outcome that represents a failed execution attempt. + /// + /// The exception thrown by the operation. + /// An outcome holding . + /// is . + public static RetryOutcome FromException(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + + return new(null, exception); + } + + /// + /// Creates an outcome that represents a completed execution attempt. + /// + /// The value returned by the operation. + /// An outcome holding . + public static RetryOutcome FromResult(object? result) => new(result, null); + + /// + /// Attempts to get the result of the operation as the specified type. + /// + /// The expected type of the result. + /// When this method returns, contains the typed result, if available. + /// + /// if the outcome holds a result of type ; otherwise, . + /// + /// + /// This helper keeps predicates readable, because the outcome + /// itself is not generic and therefore exposes as . + /// + public bool TryGetResult(out T result) + { + if (Exception is null && Result is T typedResult) + { + result = typedResult; + return true; + } + + result = default!; + return false; + } +} diff --git a/src/SimpleRetry/RetryPolicyOptions.cs b/src/SimpleRetry/RetryPolicyOptions.cs index 5c04146..7915f81 100644 --- a/src/SimpleRetry/RetryPolicyOptions.cs +++ b/src/SimpleRetry/RetryPolicyOptions.cs @@ -15,18 +15,54 @@ public class RetryPolicyOptions /// public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(2); + /// + /// Gets or sets the maximum amount of time allowed for each operation attempt. A value disables the timeout. + /// + public TimeSpan? AttemptTimeout { get; set; } + /// /// Gets or sets the strategy used to calculate the delay between retry attempts. /// public BackoffType BackoffType { get; set; } = BackoffType.Constant; /// - /// Gets or sets the predicate used to determine whether an exception should be handled by the retry policy. + /// Gets or sets the predicate used to determine whether the outcome of an execution attempt should be + /// handled by the retry policy. /// - public Func ShouldHandle { get; set; } = _ => true; + /// + /// The predicate receives both faulted and successful outcomes, so a retry can be triggered by a returned + /// value as well as by an exception. The default implementation retries every handled exception and never + /// retries a successful result. + /// + /// + /// + /// options.ShouldHandle = outcome => outcome.Exception is HttpRequestException + /// || (outcome.TryGetResult(out HttpResponseMessage? response) && !response.IsSuccessStatusCode); + /// + /// + public Func ShouldHandle { get; set; } = static outcome => outcome.IsException; /// /// Gets or sets the asynchronous callback invoked before each retry attempt. /// public Func? OnRetry { get; set; } + + /// + /// Gets or sets the callback used to override the delay before the next retry attempt. + /// + /// + /// Returning falls back to the delay computed from and + /// . This is how the HTTP policy honors the Retry-After response header. + /// + public Func? RetryDelayGenerator { get; set; } + + /// + /// Gets or sets the callback invoked when a result is discarded because the operation is about to be retried. + /// + /// + /// Only results are reported, never exceptions. This is the hook that lets a policy release resources owned by + /// the discarded value, such as disposing an result that will never be returned to + /// the caller. + /// + public Action? OnResultDiscarded { get; set; } } diff --git a/src/SimpleRetry/RetryTimeoutException.cs b/src/SimpleRetry/RetryTimeoutException.cs new file mode 100644 index 0000000..39e2126 --- /dev/null +++ b/src/SimpleRetry/RetryTimeoutException.cs @@ -0,0 +1,18 @@ +namespace SimpleRetry; + +/// +/// Represents a timeout produced by the retry executor when an operation exceeds the configured request timeout. +/// +/// +/// Initializes a new instance of the class. +/// +/// The configured timeout that was exceeded. +/// The exception produced by the underlying timeout operation. +public sealed class RetryTimeoutException(TimeSpan timeout, Exception? innerException = null) + : TimeoutException($"The operation exceeded the configured retry timeout of {timeout}.", innerException) +{ + /// + /// Gets the configured timeout that was exceeded. + /// + public TimeSpan Timeout { get; } = timeout; +} diff --git a/src/SimpleRetry/ServiceCollectionExtensions.cs b/src/SimpleRetry/ServiceCollectionExtensions.cs index 0977fcf..236c8be 100644 --- a/src/SimpleRetry/ServiceCollectionExtensions.cs +++ b/src/SimpleRetry/ServiceCollectionExtensions.cs @@ -10,6 +10,66 @@ namespace SimpleRetry; /// public static class ServiceCollectionExtensions { + extension(IHttpClientBuilder builder) + { + /// + /// Adds a standard SimpleRetry delegating handler that retries transient HTTP failures. + /// + /// The HTTP client builder for chaining additional registrations. + public IHttpClientBuilder AddHttpSimpleRetry() + => builder.AddHttpSimpleRetry(static _ => { }); + + /// + /// Adds a standard SimpleRetry delegating handler that retries transient HTTP failures. + /// + /// The callback used to configure the retry policy. + /// The HTTP client builder for chaining additional registrations. + public IHttpClientBuilder AddHttpSimpleRetry(Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(configure); + + return builder.AddHttpSimpleRetry((_, options) => configure(options)); + } + + /// + /// Adds a standard SimpleRetry delegating handler that retries transient HTTP failures using configuration that can resolve services from the provider. + /// + /// The callback used to configure the retry policy. + /// The HTTP client builder for chaining additional registrations. + /// + /// The retry policy is registered as a keyed service using the HTTP client name, so the very same + /// that runs standalone operations also drives the HTTP pipeline. + /// + public IHttpClientBuilder AddHttpSimpleRetry(Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(configure); + + builder.Services.AddKeyedSingleton(builder.Name, (services, _) => + { + var options = new RetryPolicyOptions() + { + AttemptTimeout = TimeSpan.FromSeconds(10), + BackoffType = BackoffType.Exponential, + RetryDelay = TimeSpan.FromSeconds(2), + RetryDelayGenerator = HttpRetryDelegatingHandler.GetRetryAfterDelay, + ShouldHandle = HttpRetryDelegatingHandler.ShouldHandle, + OnResultDiscarded = HttpRetryDelegatingHandler.DisposeDiscardedResponse + }; + + configure(services, options); + return options; + }); + + AddRetryExecutor(builder.Services); + + builder.AddHttpMessageHandler(services => new HttpRetryDelegatingHandler(services.GetRequiredKeyedService(builder.Name))); + + return builder; + } + } + extension(IServiceCollection services) { /// @@ -23,11 +83,7 @@ public IServiceCollection AddSimpleRetry(object? serviceKey, Action configure(options)); return services; } @@ -65,4 +121,4 @@ private static void AddRetryExecutor(IServiceCollection services) return new DefaultRetryExecutor(options, services, loggerFactory); }); } -} \ No newline at end of file +} diff --git a/src/SimpleRetry/SimpleRetry.csproj b/src/SimpleRetry/SimpleRetry.csproj index 48d03a1..d74bd0e 100644 --- a/src/SimpleRetry/SimpleRetry.csproj +++ b/src/SimpleRetry/SimpleRetry.csproj @@ -1,9 +1,10 @@  - net10.0 + net8.0;net9.0;net10.0 enable enable + latest SimpleRetry.xml Marco Minerva Marco Minerva @@ -18,14 +19,32 @@ git https://github.com/marcominerva/SimpleRetry.git master - README.md + README.md + + + + + + + + + + + + + + + + + + @@ -37,4 +56,8 @@ + + + + diff --git a/src/SimpleRetry/version.json b/src/SimpleRetry/version.json new file mode 100644 index 0000000..e072ce7 --- /dev/null +++ b/src/SimpleRetry/version.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", + "version": "3.3", + "publicReleaseRefSpec": [ + "^refs/heads/master$" // we release out of master + ], + "nugetPackageVersion": { + "semVer": 2 + }, + "pathFilters": [ "." ] +} diff --git a/tests/SimpleRetry.UnitTests/DefaultRetryExecutorExecuteAsyncTests.cs b/tests/SimpleRetry.UnitTests/DefaultRetryExecutorExecuteAsyncTests.cs new file mode 100644 index 0000000..58c5958 --- /dev/null +++ b/tests/SimpleRetry.UnitTests/DefaultRetryExecutorExecuteAsyncTests.cs @@ -0,0 +1,611 @@ +using System.Net; +using Microsoft.Extensions.Logging.Abstractions; + +namespace SimpleRetry.UnitTests; + +public partial class DefaultRetryExecutorExecuteAsyncTests +{ + [Fact] + public void RetryPolicyOptionsWhenCreatedThenUsesExpectedDefaults() + { + var options = new RetryPolicyOptions(); + + Assert.Equal(3, options.MaxRetryCount); + Assert.Equal(TimeSpan.FromSeconds(2), options.RetryDelay); + Assert.Null(options.AttemptTimeout); + Assert.Equal(BackoffType.Constant, options.BackoffType); + Assert.True(options.ShouldHandle(RetryOutcome.FromException(new InvalidOperationException()))); + Assert.False(options.ShouldHandle(RetryOutcome.FromResult("result"))); + Assert.Null(options.OnRetry); + } + + [Fact] + public async Task WhenOperationSucceedsThenRunsOnce() + { + var executor = CreateExecutor(new()); + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task WhenHandledExceptionIsThrownThenRetriesOperation() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException + }); + + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts == 1) + { + throw new InvalidOperationException(); + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(2, attempts); + } + + [Fact] + public async Task WhenAttemptTimeoutExpiresThenRetriesOperation() + { + var handledExceptions = new List(); + var retryExceptions = new List(); + var attemptTimeout = TimeSpan.FromSeconds(3); + var operationDuration = TimeSpan.FromSeconds(5); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + AttemptTimeout = attemptTimeout, + ShouldHandle = outcome => + { + handledExceptions.Add(outcome.Exception!); + return false; + }, + OnRetry = arguments => + { + retryExceptions.Add(arguments.Outcome.Exception); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + var exception = await Assert.ThrowsAsync(() => executor.ExecuteAsync(cancellationToken => + { + attempts++; + return Task.Delay(operationDuration, cancellationToken); + }, TestContext.Current.CancellationToken)); + + Assert.Equal(attemptTimeout, exception.Timeout); + + Assert.Equal(2, attempts); + Assert.Empty(handledExceptions); + + var retryException = Assert.Single(retryExceptions); + Assert.IsType(retryException); + } + + [Fact] + public async Task WhenAttemptTimeoutExpiresThenCancelsRunningOperation() + { + var observedCancellation = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 0, + RetryDelay = TimeSpan.Zero, + AttemptTimeout = TimeSpan.FromMilliseconds(50) + }); + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(async cancellationToken => + { + using var registration = cancellationToken.Register(() => observedCancellation.TrySetResult(true)); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, TestContext.Current.CancellationToken)); + + Assert.True(await observedCancellation.Task); + } + + [Fact] + public async Task WhenCallerCancellationIsSignaledDuringAttemptTimeoutThenDoesNotThrowRetryTimeoutException() + { + using var cancellationTokenSource = new CancellationTokenSource(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.Zero, + AttemptTimeout = TimeSpan.FromMinutes(1) + }); + + await Assert.ThrowsAnyAsync(() => executor.ExecuteAsync(async cancellationToken => + { + cancellationTokenSource.Cancel(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }, cancellationTokenSource.Token)); + } + + [Fact] + public async Task WhenMaxRetryCountIsZeroThenDoesNotRetry() + { + var retryCalled = false; + + var executor = CreateExecutor(new() + { + MaxRetryCount = 0, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException, + OnRetry = _ => + { + retryCalled = true; + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + throw new InvalidOperationException(); + }, TestContext.Current.CancellationToken)); + + Assert.Equal(1, attempts); + Assert.False(retryCalled); + } + + [Fact] + public async Task WhenAttemptTimeoutIsNullThenDoesNotApplyTimeout() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + AttemptTimeout = null + }); + + var attempts = 0; + + await executor.ExecuteAsync(async cancellationToken => + { + attempts++; + await Task.Delay(TimeSpan.FromMilliseconds(50), cancellationToken); + }, TestContext.Current.CancellationToken); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task WhenOperationThrowsTimeoutExceptionThenUsesShouldHandle() + { + var handledExceptions = new List(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + AttemptTimeout = TimeSpan.FromSeconds(1), + ShouldHandle = outcome => + { + handledExceptions.Add(outcome.Exception!); + return false; + } + }); + + var attempts = 0; + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + throw new TimeoutException(); + }, TestContext.Current.CancellationToken)); + + Assert.Equal(1, attempts); + + var handledException = Assert.Single(handledExceptions); + Assert.IsType(handledException); + } + + [Fact] + public async Task WhenShouldHandleThrowsThenPropagatesOriginalException() + { + var operationException = new InvalidOperationException(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + ShouldHandle = _ => throw new ApplicationException() + }); + + var attempts = 0; + + var exception = await Assert.ThrowsAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + throw operationException; + }, TestContext.Current.CancellationToken)); + + Assert.Same(operationException, exception); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task WhenShouldHandleIsNotConfiguredThenRetriesOperation() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero + }); + + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts == 1) + { + throw new InvalidOperationException(); + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(2, attempts); + } + + [Fact] + public async Task WhenHandledAndUnhandledExceptionsAreThrownThenRetriesOnlyHandledExceptions() + { + var retryExceptions = new List(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException, + OnRetry = arguments => + { + retryExceptions.Add(arguments.Outcome.Exception); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts < 3) + { + throw new InvalidOperationException(); + } + + throw new NotSupportedException(); + }, TestContext.Current.CancellationToken)); + + Assert.Equal(3, attempts); + Assert.All(retryExceptions, exception => Assert.IsType(exception)); + } + + [Theory] + [InlineData(BackoffType.Constant, 2, 2, 2)] + [InlineData(BackoffType.Linear, 2, 4, 6)] + [InlineData(BackoffType.Exponential, 2, 4, 8)] + public async Task WhenBackoffTypeVariesThenReportsExpectedRetryDelays(BackoffType backoffType, int firstDelayMilliseconds, int secondDelayMilliseconds, int thirdDelayMilliseconds) + { + var retryDelays = new List(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.FromMilliseconds(2), + BackoffType = backoffType, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException, + OnRetry = arguments => + { + retryDelays.Add(arguments.RetryDelay); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts <= 3) + { + throw new InvalidOperationException(); + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(4, attempts); + Assert.Equal([ + TimeSpan.FromMilliseconds(firstDelayMilliseconds), + TimeSpan.FromMilliseconds(secondDelayMilliseconds), + TimeSpan.FromMilliseconds(thirdDelayMilliseconds) + ], retryDelays); + } + + [Fact] + public async Task WhenMaxRetryCountIsGreaterThanOneThenRetriesUntilOperationSucceeds() + { + var retryAttempts = new List(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 2, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException, + OnRetry = arguments => + { + retryAttempts.Add(arguments.AttemptNumber); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts <= 2) + { + throw new InvalidOperationException(); + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(3, attempts); + Assert.Equal([1, 2], retryAttempts); + } + + [Fact] + public async Task WhenRetryIsAttemptedThenPassesExpectedOnRetryArguments() + { + var retryArguments = new List(); + var exceptionToHandle = new InvalidOperationException(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 2, + RetryDelay = TimeSpan.FromMilliseconds(25), + ShouldHandle = outcome => ReferenceEquals(outcome.Exception, exceptionToHandle), + OnRetry = arguments => + { + retryArguments.Add(arguments); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts == 1) + { + throw exceptionToHandle; + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + var arguments = Assert.Single(retryArguments); + + Assert.Equal(1, arguments.AttemptNumber); + Assert.Equal(2, arguments.MaxRetryCount); + Assert.Equal(TimeSpan.FromMilliseconds(25), arguments.RetryDelay); + Assert.Same(exceptionToHandle, arguments.Outcome.Exception); + Assert.Same(NullServiceProvider.Instance, arguments.ServiceProvider); + Assert.Same(NullLoggerFactory.Instance, arguments.LoggerFactory); + } + + [Fact] + public async Task WhenCancellationTokenIsSignaledDuringRetryDelayThenThrowsImmediately() + { + using var cancellationTokenSource = new CancellationTokenSource(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.FromMinutes(1), + ShouldHandle = outcome => outcome.Exception is InvalidOperationException, + OnRetry = async _ => await cancellationTokenSource.CancelAsync() + }); + + var attempts = 0; + + await Assert.ThrowsAnyAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + throw new InvalidOperationException(); + }, cancellationTokenSource.Token)); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task WhenCancellationTokenIsSignaledThenThrowsImmediately() + { + using var cancellationTokenSource = new CancellationTokenSource(); + await cancellationTokenSource.CancelAsync(); + + var retryCalled = false; + var executor = CreateExecutor(new() + { + MaxRetryCount = 3, + RetryDelay = TimeSpan.Zero, + ShouldHandle = _ => true, + OnRetry = _ => + { + retryCalled = true; + return Task.CompletedTask; + } + }); + + var attempts = 0; + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(cancellationToken => + { + attempts++; + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + }, cancellationTokenSource.Token)); + + Assert.Equal(1, attempts); + Assert.False(retryCalled); + } + + [Fact] + public async Task WhenOperationSucceedsThenReturnsResult() + { + var executor = CreateExecutor(new()); + + var result = await executor.ExecuteAsync(_ => Task.FromResult(42), TestContext.Current.CancellationToken); + + Assert.Equal(42, result); + } + + [Fact] + public async Task WhenHandledExceptionIsThrownThenRetriesOperationAndReturnsResult() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException + }); + + var attempts = 0; + + var result = await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts == 1) + { + throw new InvalidOperationException(); + } + + return Task.FromResult(42); + }, TestContext.Current.CancellationToken); + + Assert.Equal(42, result); + Assert.Equal(2, attempts); + } + + [Fact] + public async Task WhenHandledResultIsReturnedThenRetriesOperationAndReturnsResult() + { + var outcomes = new List(); + + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome is { Result: HttpStatusCode.ServiceUnavailable }, + OnRetry = arguments => + { + outcomes.Add(arguments.Outcome); + return Task.CompletedTask; + } + }); + + var attempts = 0; + + var result = await executor.ExecuteAsync(_ => + { + attempts++; + return Task.FromResult(attempts == 1 ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK); + }, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, result); + Assert.Equal(2, attempts); + + var outcome = Assert.Single(outcomes); + Assert.Null(outcome.Exception); + Assert.Equal(HttpStatusCode.ServiceUnavailable, outcome.Result); + } + + [Fact] + public async Task WhenHandledResultIsReturnedAndRetriesAreExhaustedThenReturnsLastResult() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 2, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome is { Result: HttpStatusCode.ServiceUnavailable } + }); + + var attempts = 0; + + var result = await executor.ExecuteAsync(_ => + { + attempts++; + return Task.FromResult(HttpStatusCode.ServiceUnavailable); + }, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.ServiceUnavailable, result); + Assert.Equal(3, attempts); + } + + [Fact] + public async Task WhenUnhandledExceptionIsThrownThenDoesNotRetry() + { + var executor = CreateExecutor(new() + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + ShouldHandle = outcome => outcome.Exception is InvalidOperationException + }); + + var attempts = 0; + + await Assert.ThrowsAsync(() => executor.ExecuteAsync(_ => + { + attempts++; + throw new NotSupportedException(); + }, TestContext.Current.CancellationToken)); + + Assert.Equal(1, attempts); + } + + private static DefaultRetryExecutor CreateExecutor(RetryPolicyOptions options) + => new(options, serviceProvider: NullServiceProvider.Instance, NullLoggerFactory.Instance); + + private sealed class NullServiceProvider : IServiceProvider + { + public static NullServiceProvider Instance { get; } = new(); + + public object? GetService(Type serviceType) => null; + } +} diff --git a/tests/SimpleRetry.UnitTests/HttpRetryDelegatingHandlerTests.cs b/tests/SimpleRetry.UnitTests/HttpRetryDelegatingHandlerTests.cs new file mode 100644 index 0000000..afeb7d9 --- /dev/null +++ b/tests/SimpleRetry.UnitTests/HttpRetryDelegatingHandlerTests.cs @@ -0,0 +1,498 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; + +namespace SimpleRetry.UnitTests; + +public class HttpRetryDelegatingHandlerTests +{ + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesTransientStatusThenRetriesRequest() + { + var handler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? new(HttpStatusCode.InternalServerError) : new(HttpStatusCode.OK)); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesRequestTimeoutThenRetriesRequest() + { + var handler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? new(HttpStatusCode.RequestTimeout) : new(HttpStatusCode.OK)); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesTooManyRequestsWithRetryAfterThenUsesRetryAfterDelay() + { + var retryDelay = TimeSpan.FromDays(1); + var retryAfterDelay = TimeSpan.FromSeconds(5); + var observedDelays = new List(); + + var handler = new SequenceHttpMessageHandler(attempt => + { + var response = attempt == 1 ? new HttpResponseMessage(HttpStatusCode.TooManyRequests) : new HttpResponseMessage(HttpStatusCode.OK); + + if (attempt == 1) + { + response.Headers.RetryAfter = new(retryAfterDelay); + } + + return response; + }); + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = retryDelay; + options.OnRetry = arguments => + { + observedDelays.Add(arguments.RetryDelay); + return Task.CompletedTask; + }; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + + var observedDelay = Assert.Single(observedDelays); + Assert.Equal(retryAfterDelay, observedDelay); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesHttpRequestExceptionThenRetriesRequest() + { + var handler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? throw new HttpRequestException() : new(HttpStatusCode.OK)); + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerRetriesRequestThenDisposesDiscardedResponse() + { + var discardedContent = new TrackingHttpContent(); + + var handler = new SequenceHttpMessageHandler(attempt => attempt == 1 ? new(HttpStatusCode.InternalServerError) { Content = discardedContent } + : new(HttpStatusCode.OK)); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.True(discardedContent.IsDisposed); + } + + [Fact] + public async Task WhenMultipleHttpClientsAreRegisteredThenEachUsesItsOwnRetryPolicy() + { + var retryingHandler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? new(HttpStatusCode.InternalServerError) : new(HttpStatusCode.OK)); + var nonRetryingHandler = new SequenceHttpMessageHandler(static _ => new(HttpStatusCode.InternalServerError)); + + var services = new ServiceCollection(); + + services.AddHttpClient("retrying") + .ConfigurePrimaryHttpMessageHandler(() => retryingHandler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + services.AddHttpClient("non-retrying") + .ConfigurePrimaryHttpMessageHandler(() => nonRetryingHandler) + .AddHttpSimpleRetry(options => options.MaxRetryCount = 0); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var retryingResponse = await httpClientFactory.CreateClient("retrying").GetAsync("https://example.com", TestContext.Current.CancellationToken); + using var nonRetryingResponse = await httpClientFactory.CreateClient("non-retrying").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(2, retryingHandler.SendCount); + Assert.Equal(1, nonRetryingHandler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerAttemptTimeoutExpiresThenRetriesRequest() + { + var attemptTimeout = TimeSpan.FromMilliseconds(100); + var handler = new DelayingHttpMessageHandler(attempt => attempt == 1 ? Timeout.InfiniteTimeSpan : TimeSpan.Zero); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + options.AttemptTimeout = attemptTimeout; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + var httpClient = httpClientFactory.CreateClient("test"); + httpClient.Timeout = Timeout.InfiniteTimeSpan; + + using var response = await httpClient.GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerAttemptTimeoutIsExhaustedThenThrowsRetryTimeoutException() + { + var attemptTimeout = TimeSpan.FromMilliseconds(100); + var handler = new DelayingHttpMessageHandler(static _ => Timeout.InfiniteTimeSpan); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + options.AttemptTimeout = attemptTimeout; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + var httpClient = httpClientFactory.CreateClient("test"); + httpClient.Timeout = Timeout.InfiniteTimeSpan; + + var exception = await Assert.ThrowsAsync(() => httpClient.GetAsync("https://example.com", TestContext.Current.CancellationToken)); + + Assert.Equal(attemptTimeout, exception.Timeout); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesRetryAfterDateThenUsesRetryAfterDelay() + { + var retryAfterDate = DateTimeOffset.UtcNow.AddSeconds(30); + var observedDelays = new List(); + + var handler = new SequenceHttpMessageHandler(attempt => + { + var response = new HttpResponseMessage(attempt == 1 ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK); + + if (attempt == 1) + { + response.Headers.RetryAfter = new(retryAfterDate); + } + + return response; + }); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.FromDays(1); + options.OnRetry = arguments => + { + observedDelays.Add(arguments.RetryDelay); + return Task.CompletedTask; + }; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + var observedDelay = Assert.Single(observedDelays); + Assert.InRange(observedDelay, TimeSpan.FromSeconds(20), TimeSpan.FromSeconds(30)); + } + + [Theory] + [InlineData(BackoffType.Constant, 2, 2, 2)] + [InlineData(BackoffType.Linear, 2, 4, 6)] + [InlineData(BackoffType.Exponential, 2, 4, 8)] + public async Task WhenHttpRetryDelegatingHandlerReceivesNoRetryAfterThenUsesConfiguredBackoff(BackoffType backoffType, int firstDelayMilliseconds, int secondDelayMilliseconds, int thirdDelayMilliseconds) + { + var observedDelays = new List(); + var handler = new SequenceHttpMessageHandler(static attempt => new(attempt <= 3 ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK)); + + var services = new ServiceCollection(); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 3; + options.RetryDelay = TimeSpan.FromMilliseconds(2); + options.BackoffType = backoffType; + options.OnRetry = arguments => + { + observedDelays.Add(arguments.RetryDelay); + return Task.CompletedTask; + }; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal([ + TimeSpan.FromMilliseconds(firstDelayMilliseconds), + TimeSpan.FromMilliseconds(secondDelayMilliseconds), + TimeSpan.FromMilliseconds(thirdDelayMilliseconds) + ], observedDelays); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerReceivesStreamContentWithoutBufferingThenThrowsInvalidOperationException() + { + var handler = new SequenceHttpMessageHandler(static _ => new(HttpStatusCode.OK)); + + using var client = CreateClient(handler, bufferRequestContent: false); + using var content = new StreamContent(new MemoryStream("payload"u8.ToArray())); + + var exception = await Assert.ThrowsAsync(() + => client.PostAsync("https://example.com", content, TestContext.Current.CancellationToken)); + + Assert.Contains(nameof(StreamContent), exception.Message); + Assert.Equal(0, handler.SendCount); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerBuffersStreamContentThenEveryAttemptSendsTheSameBody() + { + var bodies = new List(); + + var handler = new RecordingBodyHttpMessageHandler(bodies, static attempt => attempt == 1 ? HttpStatusCode.InternalServerError : HttpStatusCode.OK); + + using var client = CreateClient(handler, bufferRequestContent: true); + using var content = new StreamContent(new NonSeekableStream("payload"u8.ToArray())); + + using var response = await client.PostAsync("https://example.com", content, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["payload", "payload"], bodies); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerRetriesRequestThenDoesNotDisposeRequestContent() + { + var bodies = new List(); + + var handler = new RecordingBodyHttpMessageHandler(bodies, static attempt => attempt == 1 ? HttpStatusCode.InternalServerError : HttpStatusCode.OK); + + using var client = CreateClient(handler, bufferRequestContent: false); + using var content = new StringContent("payload"); + + using var response = await client.PostAsync("https://example.com", content, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["payload", "payload"], bodies); + + // The caller still owns the content, so it must be usable after the retried request completed. + Assert.Equal("payload", await content.ReadAsStringAsync(TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerDoesNotCloneRequestThenEveryAttemptUsesTheSameRequestInstance() + { + var requests = new List(); + + var handler = new RecordingHttpMessageHandler(requests, static attempt => attempt == 1 ? HttpStatusCode.InternalServerError : HttpStatusCode.OK); + + using var client = CreateClient(handler, cloneRequest: false); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.com"); + + using var response = await client.SendAsync(request, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Same(request, Assert.Single(requests.Distinct())); + } + + [Fact] + public async Task WhenHttpRetryDelegatingHandlerClonesRequestThenInnerHandlerMutationsDoNotLeakIntoNextAttempt() + { + var requests = new List(); + + var handler = new RecordingHttpMessageHandler(requests, static attempt => attempt == 1 ? HttpStatusCode.InternalServerError : HttpStatusCode.OK) + { + OnRequest = static request => request.Headers.Add("X-Attempt", "1") + }; + + using var client = CreateClient(handler, cloneRequest: true); + using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.com"); + + using var response = await client.SendAsync(request, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, requests.Distinct().Count()); + Assert.All(requests, attemptRequest => Assert.Single(attemptRequest.Headers.GetValues("X-Attempt"))); + Assert.False(request.Headers.Contains("X-Attempt")); + } + + private static HttpClient CreateClient(HttpMessageHandler innerHandler, bool bufferRequestContent = false, bool cloneRequest = false) + { + var options = new RetryPolicyOptions + { + MaxRetryCount = 1, + RetryDelay = TimeSpan.Zero, + ShouldHandle = HttpRetryDelegatingHandler.ShouldHandle, + OnResultDiscarded = HttpRetryDelegatingHandler.DisposeDiscardedResponse + }; + + var services = new ServiceCollection().BuildServiceProvider(); + var executor = new DefaultRetryExecutor(options, services, NullLoggerFactory.Instance); + + return new HttpClient(new HttpRetryDelegatingHandler(executor, bufferRequestContent, cloneRequest) { InnerHandler = innerHandler }); + } + + private sealed class SequenceHttpMessageHandler(Func createResponse) : HttpMessageHandler + { + public int SendCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + SendCount++; + return Task.FromResult(createResponse(SendCount)); + } + } + + private sealed class DelayingHttpMessageHandler(Func getDelay) : HttpMessageHandler + { + public int SendCount { get; private set; } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + SendCount++; + await Task.Delay(getDelay(SendCount), cancellationToken); + + return new(HttpStatusCode.OK); + } + } + + private sealed class RecordingHttpMessageHandler(List requests, Func getStatusCode) : HttpMessageHandler + { + private int sendCount; + + public Action? OnRequest { get; init; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + sendCount++; + OnRequest?.Invoke(request); + requests.Add(request); + + return Task.FromResult(new HttpResponseMessage(getStatusCode(sendCount))); + } + } + + private sealed class RecordingBodyHttpMessageHandler(List bodies, Func getStatusCode) : HttpMessageHandler + { + private int sendCount; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + sendCount++; + bodies.Add(await request.Content!.ReadAsStringAsync(cancellationToken)); + + return new(getStatusCode(sendCount)); + } + } + + private sealed class NonSeekableStream(byte[] content) : MemoryStream(content) + { + public override bool CanSeek => false; + } + + private sealed class TrackingHttpContent : HttpContent + { + public bool IsDisposed { get; private set; } + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) => Task.CompletedTask; + + protected override bool TryComputeLength(out long length) + { + length = 0; + return true; + } + + protected override void Dispose(bool disposing) + { + IsDisposed = true; + base.Dispose(disposing); + } + } +} diff --git a/tests/SimpleRetry.UnitTests/ServiceCollectionExtensionsTests.cs b/tests/SimpleRetry.UnitTests/ServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..853619c --- /dev/null +++ b/tests/SimpleRetry.UnitTests/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,170 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; + +namespace SimpleRetry.UnitTests; + +public class ServiceCollectionExtensionsTests +{ + [Fact] + public void AddSimpleRetryWhenConfigureIsNullThenThrowsArgumentNullException() + { + var services = new ServiceCollection(); + + var exception = Assert.Throws(() => services.AddSimpleRetry("test", (Action)null!)); + + Assert.Equal("configure", exception.ParamName); + } + + [Fact] + public void AddSimpleRetryWhenRegisteredThenReturnsSameServiceCollection() + { + var services = new ServiceCollection(); + + var result = services.AddSimpleRetry("test", static options => options.MaxRetryCount = 1); + + Assert.Same(services, result); + } + + [Fact] + public void AddSimpleRetryWhenResolvedThenUsesConfiguredOptions() + { + var services = new ServiceCollection(); + + services.AddSimpleRetry("test", static options => + { + options.MaxRetryCount = 7; + options.RetryDelay = TimeSpan.FromSeconds(5); + }); + + using var serviceProvider = services.BuildServiceProvider(); + var options = serviceProvider.GetRequiredKeyedService("test"); + + Assert.Equal(7, options.MaxRetryCount); + Assert.Equal(TimeSpan.FromSeconds(5), options.RetryDelay); + } + + [Fact] + public void AddSimpleRetryWhenConfigureUsesServiceProviderThenProvidesServiceProvider() + { + var marker = new MarkerService(9); + + var services = new ServiceCollection(); + services.AddSingleton(marker); + + services.AddSimpleRetry("test", static (serviceProvider, options) => + { + var marker = serviceProvider.GetRequiredService(); + options.MaxRetryCount = marker.MaxRetryCount; + }); + + using var serviceProvider = services.BuildServiceProvider(); + var options = serviceProvider.GetRequiredKeyedService("test"); + + Assert.Equal(marker.MaxRetryCount, options.MaxRetryCount); + } + + [Fact] + public async Task AddSimpleRetryWhenExecutorIsResolvedThenUsesKeyedRetryPolicy() + { + var services = new ServiceCollection(); + services.AddSimpleRetry("test", static options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + options.ShouldHandle = static outcome => outcome.Exception is InvalidOperationException; + }); + + using var serviceProvider = services.BuildServiceProvider(); + var executor = serviceProvider.GetRequiredKeyedService("test"); + var attempts = 0; + + await executor.ExecuteAsync(_ => + { + attempts++; + + if (attempts == 1) + { + throw new InvalidOperationException(); + } + + return Task.CompletedTask; + }, TestContext.Current.CancellationToken); + + Assert.Equal(2, attempts); + } + + [Fact] + public async Task AddHttpSimpleRetryWhenRegisteredThenRetriesTransientStatusCode() + { + var handler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? new(HttpStatusCode.InternalServerError) : new(HttpStatusCode.OK)); + var services = new ServiceCollection(); + + var builder = services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler); + + var result = builder.AddHttpSimpleRetry(options => + { + options.MaxRetryCount = 1; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Same(builder, result); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public async Task AddHttpSimpleRetryWhenConfigureUsesServiceProviderThenProvidesServiceProvider() + { + var marker = new MarkerService(1); + var handler = new SequenceHttpMessageHandler(static attempt => attempt == 1 ? new(HttpStatusCode.InternalServerError) : new(HttpStatusCode.OK)); + + var services = new ServiceCollection(); + services.AddSingleton(marker); + + services.AddHttpClient("test") + .ConfigurePrimaryHttpMessageHandler(() => handler) + .AddHttpSimpleRetry(static (serviceProvider, options) => + { + var marker = serviceProvider.GetRequiredService(); + options.MaxRetryCount = marker.MaxRetryCount; + options.RetryDelay = TimeSpan.Zero; + }); + + await using var serviceProvider = services.BuildServiceProvider(); + var httpClientFactory = serviceProvider.GetRequiredService(); + using var response = await httpClientFactory.CreateClient("test").GetAsync("https://example.com", TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, handler.SendCount); + } + + [Fact] + public void AddHttpSimpleRetryWhenConfigureIsNullThenThrowsArgumentNullException() + { + var services = new ServiceCollection(); + var builder = services.AddHttpClient("test"); + + var exception = Assert.Throws(() => builder.AddHttpSimpleRetry((Action)null!)); + + Assert.Equal("configure", exception.ParamName); + } + + private sealed record MarkerService(int MaxRetryCount); + + private sealed class SequenceHttpMessageHandler(Func createResponse) : HttpMessageHandler + { + public int SendCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + SendCount++; + return Task.FromResult(createResponse(SendCount)); + } + } +} diff --git a/tests/SimpleRetry.UnitTests/SimpleRetry.UnitTests.csproj b/tests/SimpleRetry.UnitTests/SimpleRetry.UnitTests.csproj new file mode 100644 index 0000000..2b9282b --- /dev/null +++ b/tests/SimpleRetry.UnitTests/SimpleRetry.UnitTests.csproj @@ -0,0 +1,26 @@ + + + + enable + enable + net10.0 + Exe + true + false + latest + false + + + + + + + + + + + + + + +