Preserve runner samples and report current rolling outcomes

.github/workflows/project.yml

@@ -163,18 +163,17 @@ jobs:
     env: *secrets
     steps: *steps
 
-  publish:
+  collect:
     needs: [plan, cold, warm, rolling]
-    if: ${{ !cancelled() && needs.plan.result == 'success' && github.ref == 'refs/heads/main' && inputs.mode != 'preflight' && needs.plan.outputs.matrix != '[]' }}
+    if: ${{ !cancelled() && needs.plan.result == 'success' && inputs.mode != 'preflight' && needs.plan.outputs.matrix != '[]' }}
     runs-on: ubuntu-24.04
     permissions:
-      contents: write
-      actions: write
+      contents: read
+      actions: read
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          ref: main
-          fetch-depth: 0
+          persist-credentials: false
           sparse-checkout-cone-mode: false
           sparse-checkout: |
             /*
@@ -192,6 +191,35 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           RUN_ID: ${{ github.run_id }}
         run: bundle exec bin/bench post-job-saves --run-id "$RUN_ID" --results results
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
+        with:
+          name: recorded-results-${{ github.run_attempt }}
+          path: results
+          if-no-files-found: error
+
+  publish:
+    needs: [plan, collect]
+    if: ${{ !cancelled() && needs.collect.result == 'success' && github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+      actions: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: main
+          fetch-depth: 0
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !/results/
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: recorded-results-${{ github.run_attempt }}
+          path: results
+      - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
+        with:
+          bundler-cache: true
       - continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}

docs/results.md

@@ -1,5 +1,7 @@
 # Benchmark measurements
 
+The report starts with the latest recorded rolling attempt for each active lane and runner. It includes failed, excluded and unmeasured records, their date, CLI version and source run. A successful matched cohort can end before this latest attempt. Setup failures that produce no record remain visible only in the workflow logs.
+
 The main results table has one row per lane, runner, phase, step, commit, CLI version, attempt, save timing and observed CPU. Its timings exclude failed builds, failed or missing output checks, warm records without a passing cold record in the same scope, and rolling steps that restored nothing from the lane's Actions cache after the lane's first step ("Seed failed"). "Cache API errors" copies the error count from the CLI session summary. Customer-accessible CLI/action logs remain benchmark evidence. Internal investigation notes and server-side MCP snapshots are excluded.
 
 The second table, "Matched rolling steps", compares lanes only at the rolling steps where every lane on a runner passed, grouped by rolling series and the BoringCache CLI version of that step. A lane's first step on its rolling cache starts empty, so it is left out. "Steps with cache API errors" counts the matched steps whose BoringCache session reported errors. Failed attempts remain evidence, and a rerun attempt is recorded next to the first one.
@@ -11,18 +13,20 @@ Two harness fixes on 2026-10-08 (UTC) bound which rolling records compare:
 
 The matched table leaves out records continued from a fresh run's scope, Actions cache steps that restored nothing, post-job saves whose time was not measured, and runners with only one lane. Records from before `144b6ddd` on a fixed scope carry no `cache_restored_key`, so check their dates. `harness.commit` names the benchmark commit that produced a record, and `harness.timer` what its `seconds` covers.
 
-`seconds` runs from `bin/bench start`, after checkout, preparation and the BoringCache CLI download, to `bin/bench record`: the lane's cache restore, the build and the lane's in-job cache save. It excludes the output check and is not the full job duration. Kache, mr-boxington, the BuildKit cache dance and Namespace cache volumes save in post-job steps after the record is written; the publish job reads those steps' durations from the job log into `post_job_save_seconds`, and the report adds them to the timing. The kache and mr-boxington actions also install their tool after the timer starts, which took under a second in Zed run 37994555454, about the time of the BoringCache CLI download.
+`seconds` runs from `bin/bench start`, after checkout, preparation and the BoringCache CLI download, to `bin/bench record`: the lane's cache restore, the build and the lane's in-job cache save. It excludes the output check and is not the full job duration. Kache, mr-boxington, the BuildKit cache dance and Namespace cache volumes save in post-job steps after the record is written; the collect job reads those steps' durations from the job log into `post_job_save_seconds`, and the report adds them to the timing. The kache and mr-boxington actions also install their tool after the timer starts, which took under a second in Zed run 37994555454, about the time of the BoringCache CLI download.
 
 Cold starts with a fresh local workspace. Providers with shared remote caches may still have existing data unless the lane isolates the cache or disables reads. Runner labels also do not guarantee identical CPUs. Each record stores the machine it ran on and the CPU it observed. Remote builder lanes (Depot and Namespace builders) record the builder as their machine; the CPU they observe is the calling runner's, so it is kept as `observed.runner_machine`, and the builder's own hardware is not recorded. Check the lane configuration and recorded machine before interpreting a timing difference.
 
 `base` and `plus` describe declared cache capabilities. They are separate comparisons; a compiler-cache-only lane does not promise the same reuse as a lane that also restores the build directory.
 
-Experimental lanes are excluded from the report. A matched step must use the same source commit in every included lane. Lanes with failed or missing measurements still count toward the required comparison set after their first recorded step.
+Experimental lanes are excluded from the report. A matched step must use the same source commit in every included lane. Active rolling lanes with failed or missing measurements still count toward the required comparison set after their first recorded step. Lanes reserved for samples no longer enter the rolling comparison; their original records remain in `results/`.
 
 New Actions records set `cache_restore_tracked` when the harness ran an Actions cache restore. A missing key after that scope's first step is a seed failure even if the scope has never restored successfully. Older records lack that distinction; the report identifies their restoring scopes from successful restored keys. Providers without a tracked restored key cannot be classified by this field.
 
 ## Runner samples
 
 Weekly and manual samples appear as individual runs in `samples` and the Runner samples table, apart from rolling medians. Rows retain source commit, cache source, observed CPU and cores, timed operations, exit status and output check. Missing post-job timing is unmeasured, not zero. The selected GitHub source run is metadata for a mutable cache tag, not an immutable version identifier.
 
+Every branch run collects results and post-job save durations in a `recorded-results-<attempt>` artifact. Only main publishes those records to the repository. Branch artifacts can be reviewed and added without rerunning the measured builds. The 20 October PostHog and Zed sample records retain their original branch run and harness references.
+
 BoringCache samples restore the completed GitHub commit's cache without saving. Native Depot, Namespace and Blacksmith caches retain their own history and include saving. These are different usage patterns; their elapsed-time difference is not a controlled cache-provider speedup. GitHub and third-party BoringCache samples within one dispatch use the same commit, scope and operations. Machine allocation can still vary. See [the runner and dependency procedure](../tools/README.md#weekly-runner-samples) for schedules and remaining resource differences.

lib/bench/cli.rb

@@ -424,11 +424,15 @@ def image
       def report
         options = parse(results: File.join(@catalog.root, "results"), out: File.join(@catalog.root, "data"))
         runs = @catalog.cases.flat_map do |kase|
-          kase.runs.reject { |lane, _| kase.tool.lane(lane).experimental? }.flat_map { |lane, runners| Array(runners).map { "#{kase.id} #{lane} #{it}" } }
+          kase.runs.reject { |lane, _| kase.tool.lane(lane).experimental? }.flat_map do |lane, runners|
+            Array(runners).reject { Array(kase.samples[lane]).include?(it) }.map { "#{kase.id} #{lane} #{it}" }
+          end
         end
         nightly = @catalog.cases.flat_map { |kase| kase.nightly.flat_map { |lane, runners| Array(runners).map { "#{kase.id} #{lane} #{it}" } } }
         sample_runs = @catalog.cases.flat_map do |kase|
-          kase.samples.keys.flat_map { |name| Array(kase.tool.lane(name).allowed_runners).map { "#{kase.id} #{name} #{it}" } }
+          kase.samples.keys.reject { kase.tool.lane(it).experimental? }.flat_map do |name|
+            (kase.tool.lane(name).allowed_runners || @catalog.runners.keys).map { "#{kase.id} #{name} #{it}" }
+          end
         end
         Report.new(options[:results], runs:, nightly:, sample_runs:).write(options[:out])
         @out.puts "wrote #{File.join(options[:out], "results.json")} and report.md"

lib/bench/report.rb

@@ -15,9 +15,7 @@ def initialize(results_dir, runs: nil, nightly: [], sample_runs: [])
     end
 
     def records
-      @records ||= Dir.glob(File.join(@results_dir, "**", "*.json")).sort.map { JSON.parse(File.read(it)) }
-        .reject { it["experimental"] || it["excluded"] }
-        .select { @runs.nil? || @runs.include?("#{it["adapter_command"]}/#{it["case"]} #{it["lane"]} #{it["runner"]}") || (it["phase"] == "sample" && @sample_runs.include?("#{it["adapter_command"]}/#{it["case"]} #{it["lane"]} #{it["runner"]}")) }
+      @records ||= source_records.reject { it["excluded"] }
     end
 
     def rows
@@ -26,7 +24,7 @@ def rows
 
     def write(out_dir)
       FileUtils.mkdir_p(out_dir)
-      File.write(File.join(out_dir, "results.json"), JSON.pretty_generate("rows" => rows, "matched" => matched, "read_only" => read_only, "samples" => samples) + "\n")
+      File.write(File.join(out_dir, "results.json"), JSON.pretty_generate("rows" => rows, "matched" => matched, "read_only" => read_only, "samples" => samples, "latest" => latest) + "\n")
       File.write(File.join(out_dir, "report.md"), markdown)
     end
 
@@ -64,7 +62,10 @@ def markdown
       end
       matched_table = matched.map { |row| row.values_at(*MATCHED).map { it.nil? ? "-" : it } }
       read_only_table = read_only.map { |row| row.values_at(*READ_ONLY).map { it.nil? ? "-" : it } }
-      ["# Benchmark results", "", line(COLUMNS), line(COLUMNS.map { "---" }), *table.map { line(it) }, "",
+      latest_columns = %w[Tool Case Lane Runner Step Commit CLI Started Attempt Status Seconds Run]
+      latest_rows = latest.map { it.values_at("adapter_command", "case", "lane", "runner", "step", "sha", "boringcache", "started_at", "attempt", "status", "seconds", "run_url").map { |value| value.nil? ? "unmeasured" : value } }
+      ["# Benchmark results", "", "## Latest recorded rolling attempts", "", line(latest_columns), line(latest_columns.map { "---" }), *latest_rows.map { line(it) }, "",
+       "## Measurements", "", line(COLUMNS), line(COLUMNS.map { "---" }), *table.map { line(it) }, "",
        "## Matched rolling steps", "",
        "Rolling steps at which every lane on the runner passed, each on its own rolling cache after that cache's first step. Steps continued from a fresh run's cache and Actions cache steps that restored nothing are left out. Times include post-job saves.", "",
        line(MATCHED_COLUMNS), line(MATCHED_COLUMNS.map { "---" }), *matched_table.map { line(it) }, "",
@@ -74,16 +75,42 @@ def markdown
        "## Runner samples", "", sample_table, ""].join("\n")
     end
 
+    def latest
+      source_records.select { it["phase"] == ROLLING && !it["read_only"] }
+        .group_by { it.values_at("adapter_command", "case", "lane", "runner") }.values.map do |group|
+          record = group.max_by { [it["started_at"].to_s, it["step"].to_i, it["attempt"].to_i] }
+          status = if record["excluded"]
+            "excluded"
+          elsif !passed?(record)
+            "failed"
+          elsif save_timing(record) == "post-job" || !timed_seconds(record)
+            "unmeasured"
+          elsif !seeded?(record)
+            "seed failed"
+          else
+            "passed"
+          end
+          record.slice("adapter_command", "case", "lane", "runner", "step", "sha", "started_at", "attempt", "run_url", "excluded").merge(
+            "boringcache" => cli(record), "status" => status, "seconds" => (timed_seconds(record) if status == "passed"))
+        end.sort_by { it.values_at("adapter_command", "case", "lane", "runner") }
+    end
+
     def samples
       records.select { it["phase"] == "sample" }.map do |record|
-        record.slice("adapter_command", "case", "lane", "level", "runner", "machine", "sha", "step", "run_url", "cache_source", "exit_status", "output_ok", "job_conclusion").merge(
+        record.slice("adapter_command", "case", "lane", "level", "runner", "machine", "sha", "step", "run_url", "started_at", "cache_source", "exit_status", "output_ok", "job_conclusion").merge(
           "boringcache" => cli(record), "cpu" => record.dig("observed", "machine", "cpu"), "cores" => record.dig("observed", "machine", "cores"),
           "cache" => record["read_only"] ? "GitHub build" : record["provider"], "operations" => record["read_only"] ? "restore, build" : "restore, build, save",
           "seconds" => (timed_seconds(record) if passed?(record) && save_timing(record) != "post-job"))
       end.sort_by { it.values_at("adapter_command", "case", "run_url", "lane", "runner").map(&:to_s) }
     end
 
     private
+      def source_records
+        @source_records ||= Dir.glob(File.join(@results_dir, "**", "*.json")).sort.map { JSON.parse(File.read(it)) }
+          .reject { it["experimental"] }
+          .select { @runs.nil? || @runs.include?("#{it["adapter_command"]}/#{it["case"]} #{it["lane"]} #{it["runner"]}") || ((it["phase"] == "sample" || it["read_only"]) && @sample_runs.include?("#{it["adapter_command"]}/#{it["case"]} #{it["lane"]} #{it["runner"]}")) }
+      end
+
       def sample_table
         columns = %w[Tool Case Run Commit Lane Level Machine CPU Cores Cache Operations CLI Seconds Exit Output Job]
         body = samples.map do |sample|