diff --git a/.bazelrc b/.bazelrc index de01a87443f..93fcb0c1645 100644 --- a/.bazelrc +++ b/.bazelrc @@ -190,7 +190,7 @@ common:ci-windows-cross --strategy=TestRunner=local # V8 embeds IsolateData offsets in snapshot builtins; Windows snapshots must be # generated by a Windows mksnapshot binary rather than the Linux RBE host tool. common:ci-windows-cross --strategy=V8Mksnapshot=local -common:ci-windows-cross --local_test_jobs=4 +common:ci-windows-cross --local_test_jobs=8 common:ci-windows-cross --test_env=RUST_TEST_THREADS=1 # Native Windows CI still covers the PowerShell parser-process tests. The # cross-built gnullvm binaries currently hang in those tests when run on the diff --git a/.github/scripts/measure-windows-bazel.ps1 b/.github/scripts/measure-windows-bazel.ps1 new file mode 100644 index 00000000000..339c9c6c2ad --- /dev/null +++ b/.github/scripts/measure-windows-bazel.ps1 @@ -0,0 +1,200 @@ +<# +Collect lightweight host telemetry around Windows Bazel CI runs. + +The monitor mode intentionally uses built-in Windows performance counters so +the workflow does not need to install a profiler on the runner. The summarize +mode keeps the useful aggregate in the job log while the raw CSV remains +available as an artifact for deeper analysis. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateSet('monitor', 'summarize')] + [string]$Mode, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [string]$StopFile, + + [ValidateRange(1, 60)] + [int]$IntervalSeconds = 15 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-SingleCounterValue { + param( + [Parameter(Mandatory = $true)] + [object]$CounterSample, + + [Parameter(Mandatory = $true)] + [string]$PathPattern + ) + + $sample = $CounterSample.CounterSamples | + Where-Object { $_.Path -like $PathPattern } | + Select-Object -First 1 + if ($null -eq $sample) { + return 0 + } + return [double]$sample.CookedValue +} + +function Get-SummedCounterValue { + param( + [Parameter(Mandatory = $true)] + [object]$CounterSample, + + [Parameter(Mandatory = $true)] + [string]$PathPattern + ) + + $samples = $CounterSample.CounterSamples | + Where-Object { $_.Path -like $PathPattern } + if ($null -eq $samples) { + return 0 + } + return [double](($samples | Measure-Object -Property CookedValue -Sum).Sum) +} + +function Get-Percentile { + param( + [Parameter(Mandatory = $true)] + [double[]]$Values, + + [Parameter(Mandatory = $true)] + [ValidateRange(0, 100)] + [int]$Percentile + ) + + if ($Values.Count -eq 0) { + return 0 + } + $sorted = $Values | Sort-Object + $index = [math]::Ceiling(($Percentile / 100) * $sorted.Count) - 1 + return [double]$sorted[[math]::Max(0, $index)] +} + +function Get-NumericColumn { + param( + [Parameter(Mandatory = $true)] + [object[]]$Rows, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [double[]]@($Rows | ForEach-Object { [double]$_.$Name }) +} + +function Format-BytesPerSecond { + param([double]$Value) + + return '{0:N1} MiB/s' -f ($Value / 1MB) +} + +function Write-Summary { + if (-not (Test-Path -LiteralPath $OutputPath)) { + Write-Output "No Windows Bazel telemetry file found at $OutputPath." + return + } + + $rows = @(Import-Csv -LiteralPath $OutputPath) + if ($rows.Count -eq 0) { + Write-Output "Windows Bazel telemetry file at $OutputPath had no samples." + return + } + + $cpu = Get-NumericColumn -Rows $rows -Name cpu_percent + $processorQueue = Get-NumericColumn -Rows $rows -Name processor_queue_length + $diskRead = Get-NumericColumn -Rows $rows -Name disk_read_bytes_per_sec + $diskWrite = Get-NumericColumn -Rows $rows -Name disk_write_bytes_per_sec + $diskLatency = Get-NumericColumn -Rows $rows -Name disk_seconds_per_transfer + $diskQueue = Get-NumericColumn -Rows $rows -Name disk_queue_length + $networkReceived = Get-NumericColumn -Rows $rows -Name network_received_bytes_per_sec + $networkSent = Get-NumericColumn -Rows $rows -Name network_sent_bytes_per_sec + $memory = Get-NumericColumn -Rows $rows -Name available_memory_mib + + $sampledSeconds = $rows.Count * $IntervalSeconds + $networkBytes = (($networkReceived | Measure-Object -Average).Average + ($networkSent | Measure-Object -Average).Average) * $sampledSeconds + $diskBytes = (($diskRead | Measure-Object -Average).Average + ($diskWrite | Measure-Object -Average).Average) * $sampledSeconds + + Write-Output 'Windows Bazel host telemetry summary:' + Write-Output (' samples: {0} (~{1:N0}s)' -f $rows.Count, $sampledSeconds) + Write-Output (' cpu: avg {0:N1}%, p95 {1:N1}%, max {2:N1}%; processor queue max {3:N1}' -f + ($cpu | Measure-Object -Average).Average, + (Get-Percentile -Values $cpu -Percentile 95), + ($cpu | Measure-Object -Maximum).Maximum, + ($processorQueue | Measure-Object -Maximum).Maximum) + Write-Output (' disk: read avg {0}, write avg {1}, p95 latency {2:N1}ms, queue max {3:N1}, sampled total {4:N2} GiB' -f + (Format-BytesPerSecond (($diskRead | Measure-Object -Average).Average)), + (Format-BytesPerSecond (($diskWrite | Measure-Object -Average).Average)), + ((Get-Percentile -Values $diskLatency -Percentile 95) * 1000), + ($diskQueue | Measure-Object -Maximum).Maximum, + ($diskBytes / 1GB)) + Write-Output (' network: receive avg {0}, send avg {1}, sampled total {2:N2} GiB' -f + (Format-BytesPerSecond (($networkReceived | Measure-Object -Average).Average)), + (Format-BytesPerSecond (($networkSent | Measure-Object -Average).Average)), + ($networkBytes / 1GB)) + Write-Output (' memory: minimum {0:N0} MiB available' -f ($memory | Measure-Object -Minimum).Minimum) +} + +if ($Mode -eq 'summarize') { + Write-Summary + exit 0 +} + +if ([string]::IsNullOrWhiteSpace($StopFile)) { + throw 'monitor mode requires -StopFile.' +} + +$outputDirectory = Split-Path -Parent $OutputPath +if (-not [string]::IsNullOrWhiteSpace($outputDirectory)) { + New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null +} + +$counterPaths = @( + '\Processor(_Total)\% Processor Time', + '\System\Processor Queue Length', + '\PhysicalDisk(_Total)\Disk Read Bytes/sec', + '\PhysicalDisk(_Total)\Disk Write Bytes/sec', + '\PhysicalDisk(_Total)\Avg. Disk sec/Transfer', + '\PhysicalDisk(_Total)\Avg. Disk Queue Length', + '\Network Interface(*)\Bytes Received/sec', + '\Network Interface(*)\Bytes Sent/sec', + '\Memory\Available MBytes' +) + +while (-not (Test-Path -LiteralPath $StopFile)) { + try { + $sample = Get-Counter -Counter $counterPaths -ErrorAction Stop + $row = [pscustomobject]@{ + timestamp = (Get-Date).ToUniversalTime().ToString('o') + cpu_percent = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\processor(_total)\% processor time' + processor_queue_length = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\system\processor queue length' + disk_read_bytes_per_sec = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\disk read bytes/sec' + disk_write_bytes_per_sec = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\disk write bytes/sec' + disk_seconds_per_transfer = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\avg. disk sec/transfer' + disk_queue_length = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\avg. disk queue length' + network_received_bytes_per_sec = Get-SummedCounterValue -CounterSample $sample -PathPattern '*\network interface(*)\bytes received/sec' + network_sent_bytes_per_sec = Get-SummedCounterValue -CounterSample $sample -PathPattern '*\network interface(*)\bytes sent/sec' + available_memory_mib = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\memory\available mbytes' + } + $row | Export-Csv -LiteralPath $OutputPath -Append -NoTypeInformation + } + catch { + Write-Warning "Windows Bazel telemetry sample failed: $($_.Exception.Message)" + } + + # Check the stop file once a second so the monitor does not add up to one + # full sampling interval to the measured workflow step during cleanup. + for ($second = 0; $second -lt $IntervalSeconds; $second += 1) { + if (Test-Path -LiteralPath $StopFile) { + break + } + Start-Sleep -Seconds 1 + } +} diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 3e48b741b76..eae40844c66 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -143,10 +143,14 @@ jobs: - 2 - 3 - 4 + - 5 + - 6 + - 7 + - 8 runs-on: group: ${{ github.event.repository.name }}-runners labels: ${{ github.event.repository.name }}-windows-x64 - name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/4 + name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/8 environment: name: bazel deployment: false @@ -175,12 +179,26 @@ jobs: - name: bazel test shard env: BAZEL_TEST_SHARD: ${{ matrix.shard }} - BAZEL_TEST_SHARD_COUNT: 4 + BAZEL_TEST_SHARD_COUNT: 8 BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + WINDOWS_BAZEL_TELEMETRY_PATH: ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv + WINDOWS_BAZEL_TELEMETRY_STOP_FILE: ${{ runner.temp }}/windows-bazel-telemetry-stop-shard-${{ matrix.shard }} shell: bash run: | set -euo pipefail + rm -f "${WINDOWS_BAZEL_TELEMETRY_PATH}" "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" + pwsh -NoProfile -File .github/scripts/measure-windows-bazel.ps1 \ + -Mode monitor \ + -OutputPath "${WINDOWS_BAZEL_TELEMETRY_PATH}" \ + -StopFile "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" & + telemetry_pid=$! + stop_telemetry() { + touch "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" + wait "${telemetry_pid}" || true + } + trap stop_telemetry EXIT + bazel_test_query='tests(//...) except tests(//third_party/v8:all) except attr(tags, "manual", tests(//...))' mapfile -t bazel_targets < <( ./.github/scripts/run-bazel-query-ci.sh --output=label -- "${bazel_test_query}" \ @@ -211,8 +229,13 @@ jobs: --skip_incompatible_explicit_targets --test_tag_filters=-argument-comment-lint --test_verbose_timeout_warnings + # Make every uploaded experiment revision execute its selected + # tests instead of reusing the prior revision's test action cache. + --test_env=CODEX_BAZEL_TEST_CACHE_BUST=${GITHUB_SHA} --build_metadata=COMMIT_SHA=${GITHUB_SHA} --build_metadata=TAG_windows_test_shard=${BAZEL_TEST_SHARD} + --profile=${RUNNER_TEMP}/windows-bazel-profile-shard-${BAZEL_TEST_SHARD}.gz + --noslim_profile ) ./.github/scripts/run-bazel-ci.sh \ @@ -225,6 +248,24 @@ jobs: -- \ "${selected_targets[@]}" + - name: Summarize Windows Bazel telemetry + if: always() && !cancelled() + continue-on-error: true + shell: pwsh + run: | + ./.github/scripts/measure-windows-bazel.ps1 -Mode summarize -OutputPath "${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv" + + - name: Upload Windows Bazel telemetry + if: always() && !cancelled() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: windows-bazel-telemetry-shard-${{ matrix.shard }} + path: | + ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv + ${{ runner.temp }}/windows-bazel-profile-shard-${{ matrix.shard }}.gz + if-no-files-found: warn + - name: Upload Bazel execution logs if: always() && !cancelled() continue-on-error: true