From bba3a2e56f7a340c8e946a1e5507c7572a3ac005 Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Tue, 7 Jul 2026 01:53:09 +0000 Subject: [PATCH 1/4] ci: measure Windows Bazel test bottlenecks --- .github/scripts/measure-windows-bazel.ps1 | 200 ++++++++++++++++++++++ .github/workflows/bazel.yml | 58 +++++++ 2 files changed, 258 insertions(+) create mode 100644 .github/scripts/measure-windows-bazel.ps1 diff --git a/.github/scripts/measure-windows-bazel.ps1 b/.github/scripts/measure-windows-bazel.ps1 new file mode 100644 index 000000000000..339c9c6c2ad0 --- /dev/null +++ b/.github/scripts/measure-windows-bazel.ps1 @@ -0,0 +1,200 @@ +<# +Collect lightweight host telemetry around Windows Bazel CI runs. + +The monitor mode intentionally uses built-in Windows performance counters so +the workflow does not need to install a profiler on the runner. The summarize +mode keeps the useful aggregate in the job log while the raw CSV remains +available as an artifact for deeper analysis. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateSet('monitor', 'summarize')] + [string]$Mode, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [string]$StopFile, + + [ValidateRange(1, 60)] + [int]$IntervalSeconds = 15 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-SingleCounterValue { + param( + [Parameter(Mandatory = $true)] + [object]$CounterSample, + + [Parameter(Mandatory = $true)] + [string]$PathPattern + ) + + $sample = $CounterSample.CounterSamples | + Where-Object { $_.Path -like $PathPattern } | + Select-Object -First 1 + if ($null -eq $sample) { + return 0 + } + return [double]$sample.CookedValue +} + +function Get-SummedCounterValue { + param( + [Parameter(Mandatory = $true)] + [object]$CounterSample, + + [Parameter(Mandatory = $true)] + [string]$PathPattern + ) + + $samples = $CounterSample.CounterSamples | + Where-Object { $_.Path -like $PathPattern } + if ($null -eq $samples) { + return 0 + } + return [double](($samples | Measure-Object -Property CookedValue -Sum).Sum) +} + +function Get-Percentile { + param( + [Parameter(Mandatory = $true)] + [double[]]$Values, + + [Parameter(Mandatory = $true)] + [ValidateRange(0, 100)] + [int]$Percentile + ) + + if ($Values.Count -eq 0) { + return 0 + } + $sorted = $Values | Sort-Object + $index = [math]::Ceiling(($Percentile / 100) * $sorted.Count) - 1 + return [double]$sorted[[math]::Max(0, $index)] +} + +function Get-NumericColumn { + param( + [Parameter(Mandatory = $true)] + [object[]]$Rows, + + [Parameter(Mandatory = $true)] + [string]$Name + ) + + return [double[]]@($Rows | ForEach-Object { [double]$_.$Name }) +} + +function Format-BytesPerSecond { + param([double]$Value) + + return '{0:N1} MiB/s' -f ($Value / 1MB) +} + +function Write-Summary { + if (-not (Test-Path -LiteralPath $OutputPath)) { + Write-Output "No Windows Bazel telemetry file found at $OutputPath." + return + } + + $rows = @(Import-Csv -LiteralPath $OutputPath) + if ($rows.Count -eq 0) { + Write-Output "Windows Bazel telemetry file at $OutputPath had no samples." + return + } + + $cpu = Get-NumericColumn -Rows $rows -Name cpu_percent + $processorQueue = Get-NumericColumn -Rows $rows -Name processor_queue_length + $diskRead = Get-NumericColumn -Rows $rows -Name disk_read_bytes_per_sec + $diskWrite = Get-NumericColumn -Rows $rows -Name disk_write_bytes_per_sec + $diskLatency = Get-NumericColumn -Rows $rows -Name disk_seconds_per_transfer + $diskQueue = Get-NumericColumn -Rows $rows -Name disk_queue_length + $networkReceived = Get-NumericColumn -Rows $rows -Name network_received_bytes_per_sec + $networkSent = Get-NumericColumn -Rows $rows -Name network_sent_bytes_per_sec + $memory = Get-NumericColumn -Rows $rows -Name available_memory_mib + + $sampledSeconds = $rows.Count * $IntervalSeconds + $networkBytes = (($networkReceived | Measure-Object -Average).Average + ($networkSent | Measure-Object -Average).Average) * $sampledSeconds + $diskBytes = (($diskRead | Measure-Object -Average).Average + ($diskWrite | Measure-Object -Average).Average) * $sampledSeconds + + Write-Output 'Windows Bazel host telemetry summary:' + Write-Output (' samples: {0} (~{1:N0}s)' -f $rows.Count, $sampledSeconds) + Write-Output (' cpu: avg {0:N1}%, p95 {1:N1}%, max {2:N1}%; processor queue max {3:N1}' -f + ($cpu | Measure-Object -Average).Average, + (Get-Percentile -Values $cpu -Percentile 95), + ($cpu | Measure-Object -Maximum).Maximum, + ($processorQueue | Measure-Object -Maximum).Maximum) + Write-Output (' disk: read avg {0}, write avg {1}, p95 latency {2:N1}ms, queue max {3:N1}, sampled total {4:N2} GiB' -f + (Format-BytesPerSecond (($diskRead | Measure-Object -Average).Average)), + (Format-BytesPerSecond (($diskWrite | Measure-Object -Average).Average)), + ((Get-Percentile -Values $diskLatency -Percentile 95) * 1000), + ($diskQueue | Measure-Object -Maximum).Maximum, + ($diskBytes / 1GB)) + Write-Output (' network: receive avg {0}, send avg {1}, sampled total {2:N2} GiB' -f + (Format-BytesPerSecond (($networkReceived | Measure-Object -Average).Average)), + (Format-BytesPerSecond (($networkSent | Measure-Object -Average).Average)), + ($networkBytes / 1GB)) + Write-Output (' memory: minimum {0:N0} MiB available' -f ($memory | Measure-Object -Minimum).Minimum) +} + +if ($Mode -eq 'summarize') { + Write-Summary + exit 0 +} + +if ([string]::IsNullOrWhiteSpace($StopFile)) { + throw 'monitor mode requires -StopFile.' +} + +$outputDirectory = Split-Path -Parent $OutputPath +if (-not [string]::IsNullOrWhiteSpace($outputDirectory)) { + New-Item -ItemType Directory -Path $outputDirectory -Force | Out-Null +} + +$counterPaths = @( + '\Processor(_Total)\% Processor Time', + '\System\Processor Queue Length', + '\PhysicalDisk(_Total)\Disk Read Bytes/sec', + '\PhysicalDisk(_Total)\Disk Write Bytes/sec', + '\PhysicalDisk(_Total)\Avg. Disk sec/Transfer', + '\PhysicalDisk(_Total)\Avg. Disk Queue Length', + '\Network Interface(*)\Bytes Received/sec', + '\Network Interface(*)\Bytes Sent/sec', + '\Memory\Available MBytes' +) + +while (-not (Test-Path -LiteralPath $StopFile)) { + try { + $sample = Get-Counter -Counter $counterPaths -ErrorAction Stop + $row = [pscustomobject]@{ + timestamp = (Get-Date).ToUniversalTime().ToString('o') + cpu_percent = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\processor(_total)\% processor time' + processor_queue_length = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\system\processor queue length' + disk_read_bytes_per_sec = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\disk read bytes/sec' + disk_write_bytes_per_sec = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\disk write bytes/sec' + disk_seconds_per_transfer = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\avg. disk sec/transfer' + disk_queue_length = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\physicaldisk(_total)\avg. disk queue length' + network_received_bytes_per_sec = Get-SummedCounterValue -CounterSample $sample -PathPattern '*\network interface(*)\bytes received/sec' + network_sent_bytes_per_sec = Get-SummedCounterValue -CounterSample $sample -PathPattern '*\network interface(*)\bytes sent/sec' + available_memory_mib = Get-SingleCounterValue -CounterSample $sample -PathPattern '*\memory\available mbytes' + } + $row | Export-Csv -LiteralPath $OutputPath -Append -NoTypeInformation + } + catch { + Write-Warning "Windows Bazel telemetry sample failed: $($_.Exception.Message)" + } + + # Check the stop file once a second so the monitor does not add up to one + # full sampling interval to the measured workflow step during cleanup. + for ($second = 0; $second -lt $IntervalSeconds; $second += 1) { + if (Test-Path -LiteralPath $StopFile) { + break + } + Start-Sleep -Seconds 1 + } +} diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 3e48b741b76f..083b5e6f8157 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -177,10 +177,44 @@ jobs: BAZEL_TEST_SHARD: ${{ matrix.shard }} BAZEL_TEST_SHARD_COUNT: 4 BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} + WINDOWS_BAZEL_ETL_PATH: ${{ runner.temp }}/windows-bazel-trace-shard-${{ matrix.shard }}.etl + WINDOWS_BAZEL_TELEMETRY_PATH: ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv + WINDOWS_BAZEL_TELEMETRY_STOP_FILE: ${{ runner.temp }}/windows-bazel-telemetry-stop-shard-${{ matrix.shard }} shell: bash run: | set -euo pipefail + rm -f "${WINDOWS_BAZEL_ETL_PATH}" "${WINDOWS_BAZEL_TELEMETRY_PATH}" "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" + pwsh -NoProfile -File .github/scripts/measure-windows-bazel.ps1 \ + -Mode monitor \ + -OutputPath "${WINDOWS_BAZEL_TELEMETRY_PATH}" \ + -StopFile "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" & + telemetry_pid=$! + etw_started=0 + # Keep the trace broad enough to distinguish CPU, filesystem, disk, + # and network waits, but use light profiles to limit observer cost. + if wpr.exe \ + -start CPU.light \ + -start DiskIO.light \ + -start FileIO.light \ + -start Network.light \ + -filemode; then + etw_started=1 + else + echo "WPR resource profiles could not start; continuing with counter telemetry only." >&2 + fi + stop_telemetry() { + touch "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" + wait "${telemetry_pid}" || true + if [[ "${etw_started}" -eq 1 ]]; then + if ! wpr.exe -stop "${WINDOWS_BAZEL_ETL_PATH}" -skipPdbGen -compress; then + echo "WPR resource trace could not be saved; cancelling its session." >&2 + wpr.exe -cancel || true + fi + fi + } + trap stop_telemetry EXIT + bazel_test_query='tests(//...) except tests(//third_party/v8:all) except attr(tags, "manual", tests(//...))' mapfile -t bazel_targets < <( ./.github/scripts/run-bazel-query-ci.sh --output=label -- "${bazel_test_query}" \ @@ -211,8 +245,13 @@ jobs: --skip_incompatible_explicit_targets --test_tag_filters=-argument-comment-lint --test_verbose_timeout_warnings + # Make every uploaded experiment revision execute its selected + # tests instead of reusing the prior revision's test action cache. + --test_env=CODEX_BAZEL_TEST_CACHE_BUST=${GITHUB_SHA} --build_metadata=COMMIT_SHA=${GITHUB_SHA} --build_metadata=TAG_windows_test_shard=${BAZEL_TEST_SHARD} + --profile=${RUNNER_TEMP}/windows-bazel-profile-shard-${BAZEL_TEST_SHARD}.gz + --noslim_profile ) ./.github/scripts/run-bazel-ci.sh \ @@ -225,6 +264,25 @@ jobs: -- \ "${selected_targets[@]}" + - name: Summarize Windows Bazel telemetry + if: always() && !cancelled() + continue-on-error: true + shell: pwsh + run: | + ./.github/scripts/measure-windows-bazel.ps1 -Mode summarize -OutputPath "${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv" + + - name: Upload Windows Bazel telemetry + if: always() && !cancelled() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: windows-bazel-telemetry-shard-${{ matrix.shard }} + path: | + ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv + ${{ runner.temp }}/windows-bazel-profile-shard-${{ matrix.shard }}.gz + ${{ runner.temp }}/windows-bazel-trace-shard-${{ matrix.shard }}.etl + if-no-files-found: warn + - name: Upload Bazel execution logs if: always() && !cancelled() continue-on-error: true From 9c4725e9f99d523bdc9d1bf280875b408304752b Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Tue, 7 Jul 2026 02:15:33 +0000 Subject: [PATCH 2/4] ci: drop heavyweight ETW from repeated measurements --- .github/workflows/bazel.yml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 083b5e6f8157..b1703224b6bf 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -177,41 +177,21 @@ jobs: BAZEL_TEST_SHARD: ${{ matrix.shard }} BAZEL_TEST_SHARD_COUNT: 4 BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} - WINDOWS_BAZEL_ETL_PATH: ${{ runner.temp }}/windows-bazel-trace-shard-${{ matrix.shard }}.etl WINDOWS_BAZEL_TELEMETRY_PATH: ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv WINDOWS_BAZEL_TELEMETRY_STOP_FILE: ${{ runner.temp }}/windows-bazel-telemetry-stop-shard-${{ matrix.shard }} shell: bash run: | set -euo pipefail - rm -f "${WINDOWS_BAZEL_ETL_PATH}" "${WINDOWS_BAZEL_TELEMETRY_PATH}" "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" + rm -f "${WINDOWS_BAZEL_TELEMETRY_PATH}" "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" pwsh -NoProfile -File .github/scripts/measure-windows-bazel.ps1 \ -Mode monitor \ -OutputPath "${WINDOWS_BAZEL_TELEMETRY_PATH}" \ -StopFile "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" & telemetry_pid=$! - etw_started=0 - # Keep the trace broad enough to distinguish CPU, filesystem, disk, - # and network waits, but use light profiles to limit observer cost. - if wpr.exe \ - -start CPU.light \ - -start DiskIO.light \ - -start FileIO.light \ - -start Network.light \ - -filemode; then - etw_started=1 - else - echo "WPR resource profiles could not start; continuing with counter telemetry only." >&2 - fi stop_telemetry() { touch "${WINDOWS_BAZEL_TELEMETRY_STOP_FILE}" wait "${telemetry_pid}" || true - if [[ "${etw_started}" -eq 1 ]]; then - if ! wpr.exe -stop "${WINDOWS_BAZEL_ETL_PATH}" -skipPdbGen -compress; then - echo "WPR resource trace could not be saved; cancelling its session." >&2 - wpr.exe -cancel || true - fi - fi } trap stop_telemetry EXIT @@ -280,7 +260,6 @@ jobs: path: | ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv ${{ runner.temp }}/windows-bazel-profile-shard-${{ matrix.shard }}.gz - ${{ runner.temp }}/windows-bazel-trace-shard-${{ matrix.shard }}.etl if-no-files-found: warn - name: Upload Bazel execution logs From 7ac6d5cf1554f196db3e8466393d6566fef0146b Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Tue, 7 Jul 2026 03:54:49 +0000 Subject: [PATCH 3/4] ci: increase Windows Bazel local test jobs --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index de01a87443fe..93fcb0c16450 100644 --- a/.bazelrc +++ b/.bazelrc @@ -190,7 +190,7 @@ common:ci-windows-cross --strategy=TestRunner=local # V8 embeds IsolateData offsets in snapshot builtins; Windows snapshots must be # generated by a Windows mksnapshot binary rather than the Linux RBE host tool. common:ci-windows-cross --strategy=V8Mksnapshot=local -common:ci-windows-cross --local_test_jobs=4 +common:ci-windows-cross --local_test_jobs=8 common:ci-windows-cross --test_env=RUST_TEST_THREADS=1 # Native Windows CI still covers the PowerShell parser-process tests. The # cross-built gnullvm binaries currently hang in those tests when run on the From 9e81fc5e60f01c5f9e966e18d115af9ac97344cb Mon Sep 17 00:00:00 2001 From: Adam Perry Date: Tue, 7 Jul 2026 06:03:09 +0000 Subject: [PATCH 4/4] ci: measure eight Windows Bazel test shards --- .github/workflows/bazel.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b1703224b6bf..eae40844c666 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -143,10 +143,14 @@ jobs: - 2 - 3 - 4 + - 5 + - 6 + - 7 + - 8 runs-on: group: ${{ github.event.repository.name }}-runners labels: ${{ github.event.repository.name }}-windows-x64 - name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/4 + name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/8 environment: name: bazel deployment: false @@ -175,7 +179,7 @@ jobs: - name: bazel test shard env: BAZEL_TEST_SHARD: ${{ matrix.shard }} - BAZEL_TEST_SHARD_COUNT: 4 + BAZEL_TEST_SHARD_COUNT: 8 BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} WINDOWS_BAZEL_TELEMETRY_PATH: ${{ runner.temp }}/windows-bazel-telemetry-shard-${{ matrix.shard }}.csv WINDOWS_BAZEL_TELEMETRY_STOP_FILE: ${{ runner.temp }}/windows-bazel-telemetry-stop-shard-${{ matrix.shard }}