diff --git a/src/Main.ps1 b/src/Main.ps1 index cba6dad83..c723bea50 100644 --- a/src/Main.ps1 +++ b/src/Main.ps1 @@ -548,6 +548,13 @@ function Invoke-Pester { # Write-PesterDebugMessage is used regardless of WriteScreenPlugin. Resolve-OutputConfiguration -PesterPreference $PesterPreference + # Resolve the shuffle seed once for the whole run (#2425), so it is reported a single + # time and shared by every container - including parallel workers, which each receive + # this resolved configuration. ShuffleSeed 0 means "pick a new seed for this run". + if ($PesterPreference.Run.Shuffle.Value -and 0 -eq $PesterPreference.Run.ShuffleSeed.Value) { + $PesterPreference.Run.ShuffleSeed = [System.Random]::new().Next(1, [int]::MaxValue) + } + if ('None' -ne $PesterPreference.Output.Verbosity.Value) { $plugins.Add((Get-WriteScreenPlugin -Verbosity $PesterPreference.Output.Verbosity.Value)) } diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index eefce47bc..d0f8588a4 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -80,6 +80,15 @@ function New-PesterState { Stack = [Collections.Stack]@() + # [System.Random] used to shuffle the execution order of containers, blocks and tests + # when Run.Shuffle is enabled. Seeded from Run.ShuffleSeed so a run can be repeated. + # Stays $null when Run.Shuffle is disabled. + ShuffleRandom = $null + + # Set of block containers that opt out of shuffling via a '#pester:no-shuffle' comment. + # Their blocks and tests keep declaration order even when Run.Shuffle is enabled. + NoShuffleContainers = $null + # Captured here so the <> template expansion (which runs in the user's session state) can # invoke it via "& $____Pester.FormatNicelyForTemplate" while the function itself stays bound # to the Pester module session state, where Format-Nicely2 is available (#2744). @@ -2034,6 +2043,34 @@ function Invoke-Test { $state.PluginData = $PluginData $state.Configuration = $Configuration + # Shuffled execution order (#2425). The seed is normally resolved once in Invoke-Pester + # and written back to the configuration so it can be reported and repeated. When Invoke-Test + # is called directly (e.g. from tests) with Run.Shuffle enabled but no seed, pick one here. + if ($PesterPreference.Run.Shuffle.Value) { + $shuffleSeed = $PesterPreference.Run.ShuffleSeed.Value + if (0 -eq $shuffleSeed) { + $shuffleSeed = [System.Random]::new().Next(1, [int]::MaxValue) + $PesterPreference.Run.ShuffleSeed = $shuffleSeed + } + + $state.ShuffleRandom = [System.Random]::new($shuffleSeed) + + # A file or script block can opt out with a '#pester:no-shuffle' comment. Collect those + # containers so their blocks and tests keep declaration order during discovery post-processing. + $state.NoShuffleContainers = [System.Collections.Generic.HashSet[object]]::new() + foreach ($container in $BlockContainer) { + if (Test-BlockContainerIsNoShuffle -Container $container) { + $null = $state.NoShuffleContainers.Add($container) + } + } + + # Shuffle the order the containers (test files / script blocks) run in. The blocks and + # tests inside each container are shuffled later, during discovery post-processing. + if (@($BlockContainer).Count -gt 1) { + $BlockContainer = Get-ShuffledOrder -Random $state.ShuffleRandom -InputObject $BlockContainer + } + } + # # TODO: this it potentially unreliable, because suppressed errors are written to Error as well. And the errors are captured only from the caller state. So let's use it only as a useful indicator during migration and see how it works in production code. # # finding if there were any non-terminating errors during the run, user can clear the array, and the array has fixed size so we can't just try to detect if there is any difference by counts before and after. So I capture the last known error in that state and try to find it in the array after the run @@ -2153,6 +2190,67 @@ function Invoke-Test { $executedContainers } +function Get-ShuffledOrder { + # Fisher-Yates shuffle. Returns a new array with the items in a random but + # deterministic order for a given seeded [System.Random], so a run can be repeated. + param ( + [Parameter(Mandatory = $true)] + [System.Random] $Random, + $InputObject + ) + + $items = [object[]]@($InputObject) + for ($i = $items.Length - 1; $i -gt 0; $i--) { + $j = $Random.Next(0, $i + 1) + if ($i -ne $j) { + $tmp = $items[$i] + $items[$i] = $items[$j] + $items[$j] = $tmp + } + } + + # comma to return the array as a single object, preventing pipeline unrolling + , $items +} + +function Test-BlockContainerIsNoShuffle { + # Returns $true when a container opts out of shuffling (#2425) via a file-level comment directive, + # parsed similarly to PowerShell's #requires: + # + # #pester:no-shuffle + # + # The marker is matched against real comment tokens using the PowerShell tokenizer, so it is + # recognized only inside comments and never inside strings or here-strings. It may appear anywhere + # in the file or script block. Blocks and tests in a marked container keep their declaration order. + [OutputType([bool])] + param ( + [Parameter(Mandatory = $true)] + $Container + ) + + $tokens = $null + $parseErrors = $null + + if ('File' -eq $Container.Type) { + $null = [System.Management.Automation.Language.Parser]::ParseFile($Container.Item.FullName, [ref] $tokens, [ref] $parseErrors) + } + elseif ('ScriptBlock' -eq $Container.Type) { + $null = [System.Management.Automation.Language.Parser]::ParseInput($Container.Item.ToString(), [ref] $tokens, [ref] $parseErrors) + } + else { + return $false + } + + foreach ($token in $tokens) { + if ($token.Kind -eq [System.Management.Automation.Language.TokenKind]::Comment -and + $token.Text -match '^#\s*pester:no-shuffle\b') { + return $true + } + } + + return $false +} + function PostProcess-DiscoveredBlock { param ( [Parameter(Mandatory = $true)] @@ -2178,6 +2276,30 @@ function PostProcess-DiscoveredBlock { $b.Root = $RootBlock $b.BlockContainer = $BlockContainer + # Shuffle the order of this block's direct children (its child blocks and tests, kept + # together in .Order) when Run.Shuffle is enabled. This shuffles same-level items only: + # the Describes in a file, the Describes/Contexts in a Describe, and the Its in a block. + # We do it here, before First/Last are marked below, and rebuild .Blocks and .Tests to + # follow the shuffled .Order so the one-time setup/teardown boundaries match the real + # execution order. Uses the run's seeded RNG so the order is repeatable (#2425). Containers + # that opt out with '#pester:no-shuffle' are skipped and keep their declaration order. + $containerOptedOut = $null -ne $state.NoShuffleContainers -and $state.NoShuffleContainers.Contains($BlockContainer) + if ($null -ne $state.ShuffleRandom -and -not $containerOptedOut -and $b.Order.Count -gt 1) { + $shuffledOrder = Get-ShuffledOrder -Random $state.ShuffleRandom -InputObject $b.Order + $b.Order.Clear() + $b.Blocks.Clear() + $b.Tests.Clear() + foreach ($item in $shuffledOrder) { + $null = $b.Order.Add($item) + if ('Test' -eq $item.ItemType) { + $null = $b.Tests.Add($item) + } + else { + $null = $b.Blocks.Add($item) + } + } + } + $tests = $b.Tests if ($b.IsRoot) { diff --git a/src/csharp/Pester/RunConfiguration.cs b/src/csharp/Pester/RunConfiguration.cs index 8ce6bbfd1..46ce42a1c 100644 --- a/src/csharp/Pester/RunConfiguration.cs +++ b/src/csharp/Pester/RunConfiguration.cs @@ -37,6 +37,8 @@ public class RunConfiguration : ConfigurationSection private StringOption _skipRemainingOnFailure; private BoolOption _failOnNullOrEmptyForEach; private StringOption _repoRoot; + private BoolOption _shuffle; + private IntOption _shuffleSeed; public static RunConfiguration Default { get { return new RunConfiguration(); } } public static RunConfiguration ShallowClone(RunConfiguration configuration) @@ -62,6 +64,8 @@ public RunConfiguration(IDictionary configuration) : this() configuration.AssignObjectIfNotNull(nameof(SkipRemainingOnFailure), v => SkipRemainingOnFailure = v); configuration.AssignValueIfNotNull(nameof(FailOnNullOrEmptyForEach), v => FailOnNullOrEmptyForEach = v); configuration.AssignObjectIfNotNull(nameof(RepoRoot), v => RepoRoot = v); + configuration.AssignValueIfNotNull(nameof(Shuffle), v => Shuffle = v); + configuration.AssignValueIfNotNull(nameof(ShuffleSeed), v => ShuffleSeed = v); } } @@ -80,6 +84,8 @@ public RunConfiguration(IDictionary configuration) : this() ParallelThrottleLimit = new IntOption("EXPERIMENTAL: Maximum number of test files to run at the same time when Run.Parallel is enabled, passed through to 'ForEach-Object -Parallel -ThrottleLimit'. The default 0 uses all available processors ([Environment]::ProcessorCount). Set a lower number to cap how many runspaces run concurrently. Only used when Run.Parallel is enabled.", 0); SkipRemainingOnFailure = new StringOption("Skips remaining tests after failure for selected scope, options are None, Run, Container and Block.", "None"); FailOnNullOrEmptyForEach = new BoolOption("Fails discovery when -ForEach is provided $null or @() in a block or test. Can be overridden for a specific Describe/Context/It using -AllowNullOrEmptyForEach.", true); + Shuffle = new BoolOption("Shuffle the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.ShuffleSeed so a run can be repeated, and helps surface hidden dependencies between tests. A single file can opt out with a '#pester:no-shuffle' comment.", false); + ShuffleSeed = new IntOption("Seed used to shuffle execution order when Run.Shuffle is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.ShuffleSeed to that value.", 0); RepoRoot = new StringOption("EXPERIMENTAL: Root directory of the repository. Found by searching for the .git directory recursively. When not found, the current working directory is used. Before each test file is discovered and run - in both sequential and parallel runs - Pester dot-sources a 'Pester.BeforeContainer.ps1' from this directory if one is present, so helper modules or dot-sourced setup the parent session would normally provide are available to every container. This is especially useful in parallel runs where each worker starts from a clean runspace and re-runs it.", FindRepoRoot()); } @@ -307,6 +313,38 @@ public StringOption RepoRoot } } + public BoolOption Shuffle + { + get { return _shuffle; } + set + { + if (_shuffle == null) + { + _shuffle = value; + } + else + { + _shuffle = new BoolOption(_shuffle, value.Value); + } + } + } + + public IntOption ShuffleSeed + { + get { return _shuffleSeed; } + set + { + if (_shuffleSeed == null) + { + _shuffleSeed = value; + } + else + { + _shuffleSeed = new IntOption(_shuffleSeed, value.Value); + } + } + } + private static string FindRepoRoot() { var originalDir = Directory.GetCurrentDirectory(); diff --git a/src/en-US/about_PesterConfiguration.help.txt b/src/en-US/about_PesterConfiguration.help.txt index 3cceba8c1..8522867e1 100644 --- a/src/en-US/about_PesterConfiguration.help.txt +++ b/src/en-US/about_PesterConfiguration.help.txt @@ -85,6 +85,14 @@ SECTIONS AND OPTIONS Type: string Default value: '' + Shuffle: Shuffle the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.ShuffleSeed so a run can be repeated, and helps surface hidden dependencies between tests. A single file can opt out with a '#pester:no-shuffle' comment. + Type: bool + Default value: $false + + ShuffleSeed: Seed used to shuffle execution order when Run.Shuffle is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.ShuffleSeed to that value. + Type: int + Default value: 0 + Filter: Tag: Tags of Describe, Context or It to be run. Use 'None' to run only tests that have no tags. Type: string[] diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 8f7123643..9b34f57cf 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -1,6 +1,7 @@ $script:ReportStrings = DATA { @{ VersionMessage = "Pester v{0}" + ShuffleMessage = "Shuffling execution order using seed {0}. Set 'Run.ShuffleSeed = {0}' to repeat this order." CoverageMessage = 'Covered {2:0.##}% / {5:0.##}%. {3:N0} analyzed {0} in {4:N0} {1}.' MissedSingular = 'Missed command:' @@ -528,6 +529,11 @@ function Get-WriteScreenPlugin ($Verbosity) { $parallelSuffix = if ($Context.Parallel) { ' in parallel' } else { '' } Write-PesterHostMessage -ForegroundColor $ReportTheme.Container "`nRunning tests from $(@($Context.BlockContainers).Length) files$parallelSuffix." } + + if ($PesterPreference.Run.Shuffle.Value) { + # Report the resolved seed so a randomized run (#2425) can be repeated. + Write-PesterHostMessage -ForegroundColor $ReportTheme.Discovery ($ReportStrings.ShuffleMessage -f $PesterPreference.Run.ShuffleSeed.Value) + } } $p.ContainerDiscoveryEnd = { diff --git a/tst/Pester.RSpec.Shuffle.ts.ps1 b/tst/Pester.RSpec.Shuffle.ts.ps1 new file mode 100644 index 000000000..176b98a83 --- /dev/null +++ b/tst/Pester.RSpec.Shuffle.ts.ps1 @@ -0,0 +1,317 @@ +param ([switch] $PassThru, [switch] $NoBuild) + +Get-Module P, PTestHelpers, Pester, Axiom | Remove-Module + +Import-Module $PSScriptRoot\p.psm1 -DisableNameChecking +Import-Module $PSScriptRoot\axiom\Axiom.psm1 -DisableNameChecking + +if (-not $NoBuild) { & "$PSScriptRoot\..\build.ps1" } +Import-Module $PSScriptRoot\..\bin\Pester.psd1 + +$global:PesterPreference = @{ + Debug = @{ + ShowFullErrors = $true + } + Output = @{ + Verbosity = 'None' + } +} +$PSDefaultParameterValues = @{} + +# A container with a known structure. Every It records its path into $global:__order when it runs, +# so we can observe the real execution order at every level: +# - the Describes directly in the file (A, B, C), +# - the Context/Describe nested in a Describe (A\A-inner), +# - the Its in a block (a1..a2, c1..c3, ...). +$script:SampleBlock = { + Describe 'A' { + It 'a1' { $global:__order.Add('A.a1') } + It 'a2' { $global:__order.Add('A.a2') } + Context 'A-inner' { + It 'ai1' { $global:__order.Add('A.inner.ai1') } + It 'ai2' { $global:__order.Add('A.inner.ai2') } + } + } + Describe 'B' { + It 'b1' { $global:__order.Add('B.b1') } + It 'b2' { $global:__order.Add('B.b2') } + } + Describe 'C' { + It 'c1' { $global:__order.Add('C.c1') } + It 'c2' { $global:__order.Add('C.c2') } + It 'c3' { $global:__order.Add('C.c3') } + } +} + +function Get-ExecutionOrder { + param ( + [ScriptBlock] $ScriptBlock = $script:SampleBlock, + [switch] $Shuffle, + [int] $Seed = 0 + ) + + $global:__order = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $ScriptBlock + $c.Run.Shuffle = [bool]$Shuffle + $c.Run.ShuffleSeed = $Seed + $c.Run.PassThru = $true + $c.Output.Verbosity = 'None' + $r = Invoke-Pester -Configuration $c + + [PSCustomObject]@{ + Order = $global:__order.ToArray() + OrderString = $global:__order -join ',' + ResolvedSeed = $r.Configuration.Run.ShuffleSeed.Value + Result = $r + } +} + +i -PassThru:$PassThru { + b "Run.Shuffle configuration options" { + t "Run.Shuffle exists and defaults to disabled" { + $c = [PesterConfiguration]::Default + $c.Run.Shuffle.Value | Verify-False + } + + t "Run.ShuffleSeed exists and defaults to 0" { + $c = [PesterConfiguration]::Default + $c.Run.ShuffleSeed.Value | Verify-Equal 0 + } + + t "Run.Shuffle can be enabled and Run.ShuffleSeed can be set" { + $c = [PesterConfiguration]::Default + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 123 + $c.Run.Shuffle.Value | Verify-True + $c.Run.ShuffleSeed.Value | Verify-Equal 123 + } + + t "options can be set from a hashtable" { + $c = [PesterConfiguration]@{ Run = @{ Shuffle = $true; ShuffleSeed = 99 } } + $c.Run.Shuffle.Value | Verify-True + $c.Run.ShuffleSeed.Value | Verify-Equal 99 + } + } + + b "Default order (Run.Shuffle disabled)" { + t "keeps the discovery (declaration) order" { + $r = Get-ExecutionOrder + $r.OrderString | Verify-Equal 'A.a1,A.a2,A.inner.ai1,A.inner.ai2,B.b1,B.b2,C.c1,C.c2,C.c3' + } + } + + b "Shuffled order is repeatable" { + t "the same seed produces the same order across runs" { + $first = Get-ExecutionOrder -Shuffle -Seed 42 + $second = Get-ExecutionOrder -Shuffle -Seed 42 + $first.OrderString | Verify-Equal $second.OrderString + } + + t "a randomized run differs from the declaration order" { + $ordered = Get-ExecutionOrder + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 + ($shuffled.OrderString -ne $ordered.OrderString) | Verify-True + } + + t "different seeds produce different orders" { + $a = Get-ExecutionOrder -Shuffle -Seed 42 + $b = Get-ExecutionOrder -Shuffle -Seed 7 + ($a.OrderString -ne $b.OrderString) | Verify-True + } + } + + b "Shuffled order shuffles every level, dropping nothing" { + t "runs exactly the same set of tests, only reordered" { + $ordered = Get-ExecutionOrder + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 + + $shuffled.Order.Count | Verify-Equal $ordered.Order.Count + $expected = $ordered.Order | Sort-Object + $actual = $shuffled.Order | Sort-Object + ($actual -join ',') | Verify-Equal ($expected -join ',') + } + + t "reorders top-level Describes in a file" { + # Reduce each entry to its top-level Describe and keep the order they first appear in. + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 + $topLevel = @($shuffled.Order | ForEach-Object { ($_ -split '\.')[0] } | Select-Object -Unique) + (($topLevel -join ',') -ne 'A,B,C') | Verify-True + } + + t "reorders the Its inside a block" { + # Seeds are chosen so the C block's tests are not in declaration order. + $found = $false + foreach ($seed in 1..20) { + $shuffled = Get-ExecutionOrder -Shuffle -Seed $seed + $cTests = @($shuffled.Order | Where-Object { $_ -like 'C.*' }) + if (($cTests -join ',') -ne 'C.c1,C.c2,C.c3') { $found = $true; break } + } + $found | Verify-True + } + + t "reorders the Its inside a nested Context" { + $found = $false + foreach ($seed in 1..20) { + $shuffled = Get-ExecutionOrder -Shuffle -Seed $seed + $inner = @($shuffled.Order | Where-Object { $_ -like 'A.inner.*' }) + if (($inner -join ',') -ne 'A.inner.ai1,A.inner.ai2') { $found = $true; break } + } + $found | Verify-True + } + } + + b "Auto seed (Run.ShuffleSeed = 0)" { + t "resolves a non-zero seed and reports it on the result configuration" { + $r = Get-ExecutionOrder -Shuffle -Seed 0 + ($r.ResolvedSeed -ne 0) | Verify-True + } + + t "does not mutate the caller's configuration object" { + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $script:SampleBlock + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 0 + $c.Output.Verbosity = 'None' + $global:__order = [System.Collections.Generic.List[string]]::new() + $null = Invoke-Pester -Configuration $c + # The run works on a merged copy, so the caller's seed stays 0 (a fresh seed each run). + $c.Run.ShuffleSeed.Value | Verify-Equal 0 + } + + t "the reported seed reproduces the same order" { + $auto = Get-ExecutionOrder -Shuffle -Seed 0 + $repro = Get-ExecutionOrder -Shuffle -Seed $auto.ResolvedSeed + $auto.OrderString | Verify-Equal $repro.OrderString + } + } + + b "Shuffled order keeps setup and teardown correct" { + t "one-time and each setup/teardown still run the right number of times" { + # If shuffling broke the First/Last markers, one-time setup/teardown would fire at the + # wrong item. Count invocations to prove they stay correct under a shuffled order. + $global:__oneTime = 0 + $global:__each = 0 + $sb = { + Describe 'S' { + BeforeAll { $global:__oneTime++ } + AfterAll { $global:__oneTime++ } + BeforeEach { $global:__each++ } + AfterEach { $global:__each++ } + It 's1' { 1 | Should -Be 1 } + It 's2' { 1 | Should -Be 1 } + It 's3' { 1 | Should -Be 1 } + } + } + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $sb + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 42 + $c.Run.PassThru = $true + $c.Output.Verbosity = 'None' + $r = Invoke-Pester -Configuration $c + + $r.PassedCount | Verify-Equal 3 + $r.FailedCount | Verify-Equal 0 + # BeforeAll + AfterAll once each. + $global:__oneTime | Verify-Equal 2 + # BeforeEach + AfterEach for each of the 3 tests. + $global:__each | Verify-Equal 6 + } + } + + b "Shuffled file order" { + t "shuffles the order test files run in, repeatably" { + $folder = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().Guid) + $null = New-Item -ItemType Directory -Path $folder -Force + foreach ($n in 'One', 'Two', 'Three', 'Four', 'Five') { + Set-Content -Path (Join-Path $folder "$n.Tests.ps1") -Value "Describe '$n' { It 'i' { `$global:__forder.Add('$n') } }" + } + try { + function Get-FileOrder ([int] $Seed) { + $global:__forder = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.Path = $folder + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = $Seed + $c.Output.Verbosity = 'None' + $null = Invoke-Pester -Configuration $c + $global:__forder -join ',' + } + + $ordered = ('One', 'Two', 'Three', 'Four', 'Five') -join ',' + $a = Get-FileOrder -Seed 12345 + $b = Get-FileOrder -Seed 12345 + + # same set of files ran + (($a -split ',' | Sort-Object) -join ',') | Verify-Equal (($ordered -split ',' | Sort-Object) -join ',') + # repeatable + $a | Verify-Equal $b + # actually reordered + ($a -ne $ordered) | Verify-True + } + finally { Remove-Item -Path $folder -Recurse -Force } + } + } + + b "Opting out with #pester:no-shuffle" { + t "a script block with the directive keeps its declaration order while shuffle is on" { + $sb = { + # pester:no-shuffle + Describe 'A' { + It 'a1' { $global:__order.Add('A.a1') } + It 'a2' { $global:__order.Add('A.a2') } + Context 'inner' { + It 'ai1' { $global:__order.Add('A.inner.ai1') } + It 'ai2' { $global:__order.Add('A.inner.ai2') } + } + } + Describe 'B' { + It 'b1' { $global:__order.Add('B.b1') } + It 'b2' { $global:__order.Add('B.b2') } + } + } + $declared = 'A.a1,A.a2,A.inner.ai1,A.inner.ai2,B.b1,B.b2' + # Try several seeds; none should be able to reorder the opted-out container. + foreach ($seed in 1, 7, 42, 2024) { + $r = Get-ExecutionOrder -ScriptBlock $sb -Shuffle -Seed $seed + $r.OrderString | Verify-Equal $declared + } + } + + t "the directive keeps one file ordered while other files still shuffle" { + $folder = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().Guid) + $null = New-Item -ItemType Directory -Path $folder -Force + # Ordered.Tests.ps1 opts out; its 4 Its must stay in order. The other files are only there + # to make sure shuffle is actually active in the run. + Set-Content -Path (Join-Path $folder 'Ordered.Tests.ps1') -Value @' +# pester:no-shuffle +Describe 'Ordered' { + It 'o1' { $global:__order2.Add('o1') } + It 'o2' { $global:__order2.Add('o2') } + It 'o3' { $global:__order2.Add('o3') } + It 'o4' { $global:__order2.Add('o4') } +} +'@ + foreach ($n in 'Free1', 'Free2', 'Free3') { + Set-Content -Path (Join-Path $folder "$n.Tests.ps1") -Value "Describe '$n' { It 'a' { 1 | Should -Be 1 }; It 'b' { 1 | Should -Be 1 } }" + } + try { + $global:__order2 = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.Path = $folder + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 42 + $c.Output.Verbosity = 'None' + $c.Run.PassThru = $true + $r = Invoke-Pester -Configuration $c + + # the opted-out file kept its declaration order + ($global:__order2 -join ',') | Verify-Equal 'o1,o2,o3,o4' + # and everything still ran + $r.FailedCount | Verify-Equal 0 + } + finally { Remove-Item -Path $folder -Recurse -Force } + } + } +}