From be53edb2b5b2b0abf69b8395e1e5695d53ea157a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 5 Jul 2026 16:40:34 -0400
Subject: [PATCH 01/45] use problem matcher instead of custom ruby filter

---
 .github/problem-matchers/javadoc.json |  16 +++
 .github/workflows/build_and_test.yml  |   4 +
 docs/_plugins/build_api_docs.rb       | 142 +-------------------------
 project/GlobalSettings.scala          |  15 +++
 project/SparkBuild.scala              |   1 -
 5 files changed, 36 insertions(+), 142 deletions(-)
 create mode 100644 .github/problem-matchers/javadoc.json
 create mode 100644 project/GlobalSettings.scala

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
new file mode 100644
index 0000000000000..c3707b92b1a75
--- /dev/null
+++ b/.github/problem-matchers/javadoc.json
@@ -0,0 +1,16 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "javadoc-doclint",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^\\[warn\\]\\s+(.+?):(\\d+)(?::\\d+)?:\\s+error:\\s+(.+)$",
+          "file": 1,
+          "line": 2,
+          "message": 3
+        }
+      ]
+    }
+  ]
+}
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index b117b16929960..cd3cb107b1033 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1344,6 +1344,8 @@ jobs:
         gem install bundler -v 2.4.22
         cd docs
         bundle install --retry=100
+    - name: Register javadoc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build for branch-3.5 and branch-4.0
       if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
       run: |
@@ -1438,6 +1440,8 @@ jobs:
         echo "SKIP_SQLDOC: $SKIP_SQLDOC"
         cd docs
         bundle exec jekyll build
+    - name: Remove problem matcher
+      run: echo "::remove-matcher owner=javadoc-doclint::"
     - name: Tar documentation
       if: github.repository != 'apache/spark'
       run: tar cjf site.tar.bz2 docs/_site
diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb
index 429cef5aa026c..d48f7f0c1e700 100644
--- a/docs/_plugins/build_api_docs.rb
+++ b/docs/_plugins/build_api_docs.rb
@@ -131,147 +131,7 @@ def build_spark_scala_and_java_docs_if_necessary
 
   command = "build/sbt -Pkinesis-asl unidoc"
   puts "Running '#{command}'..."
-
-  # Two filter passes on the unidoc output, plus an additive fatal-error summary:
-  #
-  # 1. Genjavadoc-stub diagnostic blocks (~28 `[error]` lines on stubs under
-  #    `target/java/`, plus 3-5 continuation lines each). Inert because
-  #    `--ignore-source-errors` is set; matched by message text so legitimate
-  #    doclint diagnostics on stub paths still pass through.
-  #
-  # 2. `-verbose` progress lines (~13K total): `Loading source file ...`,
-  #    `[parsing started/completed ...]`, `[loading /path/X.class]`,
-  #    `Generating .../X.html`. These are dominant in the log when `-verbose`
-  #    is set (which it is in `JavaUnidoc / unidoc / javacOptions` to surface
-  #    per-file `error: reference not found` diagnostics) but carry no signal
-  #    of their own. Suppressing them brings the visible log from ~17K to ~5K
-  #    lines on a typical run while leaving every diagnostic untouched.
-  #
-  # 3. Fatal-error summary (additive, drops no log lines). The filtered log is
-  #    still ~4K lines and most `error:` text in it is non-fatal source-loading
-  #    chatter, so the build-failing diagnostics are hard to spot. After the
-  #    pipe closes, we print a `Fatal javadoc errors (N): ...` block and emit
-  #    `::error file=,line=::` GitHub Actions annotations so they surface in the
-  #    PR check panel. Captured strictly within the Standard Doclet phase
-  #    bracketed by `Building tree for all the packages and classes...` and
-  #    `Building index for all classes...`, which is where doclint diagnostics
-  #    are emitted -- this matches what javadoc counts toward exit code 1.
-  #    Self-checked against javadoc's own `N errors` summary line; a mismatch
-  #    emits a `::warning::` so future phase-marker drift is visible.
-  ansi = /\e\[[0-9;]*[A-Za-z]/
-  stub_header = %r{
-    \[(?:error|warn)\]\s+
-    \S*?/target/java/\S+\.java:\d+(?::\d+)?:\s+
-    error:\s+
-    (?:cannot\s+find\s+symbol
-     |illegal\s+combination\s+of\s+modifiers
-     |non-static\s+type\s+variable\b
-     |.*?\s+is\s+not\s+public\s+in\s+\S+;\s+cannot\s+be\s+accessed\s+from\s+outside\s+package)
-  }x
-  stub_cont = %r{\A\s*\[(?:error|warn)\]\s+(?!/\S+\.java:\d+(?::\d+)?:\s)}
-  verbose_line = %r{
-    \[(?:error|warn)\]\s+
-    (?:Loading\s+source\s+file\s
-     |\[parsing\s+(?:started|completed)\s
-     |\[loading\s
-     |\[checking\s
-     |\[wrote\s
-     |Generating\s+\S+\.html
-    )
-  }x
-
-  # Doclint phase tracking for the trailing summary. Standard Doclet bookends the
-  # phase that produces build-failing diagnostics with these marker lines; any
-  # `error:` outside this window is source-loading noise that does not contribute
-  # to javadoc's exit code. The summary below captures only the fatal ones and
-  # re-emits them as GitHub Actions annotations so they surface in the PR check
-  # panel instead of being buried in a 4K-line log.
-  doclint_start   = %r{\bBuilding\s+tree\s+for\s+all\s+the\s+packages\s+and\s+classes\b}
-  doclint_end     = %r{\bBuilding\s+index\s+for\s+all\s+classes\b}
-  doclint_diag    = %r{\A\[warn\]\s+(?<path>\S+):(?<lineno>\d+)(?::\d+)?:\s+error:\s+(?<msg>.+?)\s*\z}
-  doclint_cont    = %r{\A\[warn\]\s(?!\S+:\d+(?::\d+)?:\s+error:)(?<content>.*?)\s*\z}
-  doclint_summary = %r{\A\[warn\]\s+(?<count>[\d,]+)\s+errors?\s*\z}
-
-  in_stub = false
-  in_doclint = false
-  fatal_diagnostics = []
-  pending_context_lines = 0  # snippet + caret lines that follow each diag header
-  reported_error_count = nil
-
-  IO.popen("#{command} 2>&1", 'r') do |pipe|
-    pipe.each_line do |line|
-      plain = line.gsub(ansi, '')
-
-      if plain =~ doclint_start
-        in_doclint = true
-      elsif in_doclint && plain =~ doclint_end
-        in_doclint = false
-        pending_context_lines = 0
-      end
-
-      if in_doclint && (m = plain.match(doclint_diag))
-        fatal_diagnostics << {
-          path: m[:path], line: m[:lineno], msg: m[:msg], context: []
-        }
-        pending_context_lines = 2
-      elsif in_doclint && pending_context_lines > 0 &&
-            (m = plain.match(doclint_cont)) && !fatal_diagnostics.empty?
-        fatal_diagnostics.last[:context] << m[:content]
-        pending_context_lines -= 1
-      end
-
-      if reported_error_count.nil? && (m = plain.match(doclint_summary))
-        reported_error_count = m[:count].delete(',').to_i
-      end
-
-      if plain =~ verbose_line
-        in_stub = false
-        # suppress -verbose progress line
-      elsif plain =~ stub_header
-        in_stub = true
-      elsif in_stub && plain =~ stub_cont
-        # continuation of a stub block; suppress
-      else
-        in_stub = false
-        $stdout.write(line)
-        $stdout.flush
-      end
-    end
-  end
-
-  unless fatal_diagnostics.empty?
-    bar = "=" * 72
-    puts ""
-    puts bar
-    puts "Fatal javadoc errors (#{fatal_diagnostics.size}):"
-    puts bar
-    fatal_diagnostics.each_with_index do |d, i|
-      puts "  #{i + 1}. #{d[:path]}:#{d[:line]}: #{d[:msg]}"
-      d[:context].each { |c| puts "       #{c}" }
-    end
-    puts bar
-    puts ""
-
-    # GitHub Actions inline annotations. `%`, `\r`, `\n` require URL-style
-    # escaping per the workflow command spec; newlines render as multiple
-    # lines inside the annotation, so the source snippet and caret display
-    # under the error message in the PR check panel.
-    project_root = SPARK_PROJECT_ROOT + '/'
-    fatal_diagnostics.each do |d|
-      rel = d[:path].start_with?(project_root) ? d[:path][project_root.length..] : d[:path]
-      full = ([d[:msg]] + d[:context]).join("\n")
-      enc = full.gsub(/[%\r\n]/, '%' => '%25', "\r" => '%0D', "\n" => '%0A')
-      puts "::error file=#{rel},line=#{d[:line]},title=javadoc::#{enc}"
-    end
-  end
-
-  if reported_error_count && reported_error_count != fatal_diagnostics.size
-    puts "::warning::Javadoc reported #{reported_error_count} errors but " \
-         "build_api_docs.rb captured #{fatal_diagnostics.size}. The doclint " \
-         "phase markers may have shifted; please update build_api_docs.rb."
-  end
-
-  raise("Unidoc generation failed") unless $?.success?
+  system(command) || raise("Unidoc generation failed")
 end
 
 def build_scala_and_java_docs
diff --git a/project/GlobalSettings.scala b/project/GlobalSettings.scala
new file mode 100644
index 0000000000000..104def4171490
--- /dev/null
+++ b/project/GlobalSettings.scala
@@ -0,0 +1,15 @@
+import sbt._
+import sbt.Keys._
+
+/**
+ * Suppresses the "N keys that are not used by any other settings/tasks" lint
+ * that fires on project load. These are a side effect of applying broad shared
+ * settings (e.g. Checkstyle, publishMavenStyle) to all projects, including
+ * assembly and other modules that don't use all of them.
+ */
+object GlobalSettingsPlugin extends AutoPlugin {
+  override def trigger = allRequirements
+  override def globalSettings: Seq[Setting[_]] = Seq(
+    lintUnusedKeysOnLoad := false
+  )
+}
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 724b42942c7e5..40723525c7bcd 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1790,7 +1790,6 @@ object Unidoc {
         "-tag", "inheritdoc",
         "--ignore-source-errors", "-notree",
         "-Xmaxerrs", "0",
-        "-verbose",
         "-Xdoclint:all", "-Xdoclint:-missing"
       )
     },

From d8b018ff000c94cffb2a606cbbf14806ab9613d7 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 5 Jul 2026 20:21:09 -0400
Subject: [PATCH 02/45] add license header

---
 project/GlobalSettings.scala | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/project/GlobalSettings.scala b/project/GlobalSettings.scala
index 104def4171490..c36f819c10261 100644
--- a/project/GlobalSettings.scala
+++ b/project/GlobalSettings.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import sbt._
 import sbt.Keys._
 

From 0943dee856b40042684b7beb34dc6bd21af5767b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 5 Jul 2026 23:14:30 -0400
Subject: [PATCH 03/45] try to trigger doc errors

---
 core/src/main/scala/org/apache/spark/Partition.scala             | 1 +
 .../main/java/org/apache/spark/sql/vectorized/ColumnarMap.java   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/Partition.scala b/core/src/main/scala/org/apache/spark/Partition.scala
index e10660793d162..add650cf99123 100644
--- a/core/src/main/scala/org/apache/spark/Partition.scala
+++ b/core/src/main/scala/org/apache/spark/Partition.scala
@@ -23,6 +23,7 @@ package org.apache.spark
 trait Partition extends Serializable {
   /**
    * Get the partition's index within its parent RDD
+   * See also [[Partition.index]].
    */
   def index: Int
 
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
index 6b3d518746dc3..5beb671eaa43c 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java
@@ -22,6 +22,7 @@
 
 /**
  * Map abstraction in {@link ColumnVector}.
+ * See also {@link org.apache.spark.deliberately.NoSuchClass}.
  */
 public final class ColumnarMap extends MapData {
   private final ColumnarArray keys;

From 4c3d880773108b76fceeb23c84bd4fb87452f741 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 11:22:36 -0400
Subject: [PATCH 04/45] fix problem matcher regex

---
 .github/problem-matchers/javadoc.json | 6 ++++--
 project/SparkBuild.scala              | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index c3707b92b1a75..6eeef27f74fae 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,10 +5,12 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^\\[warn\\]\\s+(.+?):(\\d+)(?::\\d+)?:\\s+error:\\s+(.+)$",
+          "regexp": "^\\[error\\]\\s+(?!\\S*/target/java/org/apache/spark/)(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(.+)$",
           "file": 1,
           "line": 2,
-          "message": 3
+          "column": 3,
+          "severity": 4,
+          "message": 5
         }
       ]
     }
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 40723525c7bcd..9c4e1ee762d3f 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1775,6 +1775,9 @@ object Unidoc {
           .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hadoop"))))
     },
 
+    // This is separate from `-Xmaxerrs` below. The default is 100.
+    (JavaUnidoc / unidoc / maxErrors) := 0,
+
     (JavaUnidoc / unidoc / javacOptions) := {
       Seq(
         "-windowtitle", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " JavaDoc",

From f7b6037e1a3df53b43382175ffe8ac111d4b96e6 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 12:31:36 -0400
Subject: [PATCH 05/45] -quiet

---
 project/SparkBuild.scala | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 9c4e1ee762d3f..c8ff568774ea1 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1775,7 +1775,12 @@ object Unidoc {
           .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hadoop"))))
     },
 
-    // This is separate from `-Xmaxerrs` below. The default is 100.
+    // This setting is separate from `-Xmaxerrs` below and just suppresses the _display_ of
+    // errors, not the errors themselves. The default is 100.
+    // We want _all_ errors to show because we have no way to separate real errors in our
+    // source from errors in generated Java files that are ignored. So if some errors are
+    // suppressed here, they will still fail the doc build even though they won't print
+    // an error message indicating the location of the problem.
     (JavaUnidoc / unidoc / maxErrors) := 0,
 
     (JavaUnidoc / unidoc / javacOptions) := {
@@ -1791,6 +1796,7 @@ object Unidoc {
         "-tag", "todo:X",
         "-tag", "groupname:X",
         "-tag", "inheritdoc",
+        "-quiet",
         "--ignore-source-errors", "-notree",
         "-Xmaxerrs", "0",
         "-Xdoclint:all", "-Xdoclint:-missing"

From 30e6c8b868787b22241948e8a0bc7df8b34202b7 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 12:31:55 -0400
Subject: [PATCH 06/45] escape backslashes in problem matcher regex

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 6eeef27f74fae..26de2cecffe03 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^\\[error\\]\\s+(?!\\S*/target/java/org/apache/spark/)(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(.+)$",
+          "regexp": "^\\[error\\]\\s+(?!\\S*\\/target\\/java\\/org\\/apache\\/spark\\/)(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(.+)$",
           "file": 1,
           "line": 2,
           "column": 3,

From 15122349975fc975bcc8c7c4e5c113fcafcfba77 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 14:53:28 -0400
Subject: [PATCH 07/45] tolerate ANSI color codes in problem matcher

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 26de2cecffe03..0bac4827456d8 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^\\[error\\]\\s+(?!\\S*\\/target\\/java\\/org\\/apache\\/spark\\/)(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(.+)$",
+          "regexp": "^(?:\u001b\\[[0-9;]*m)*\\[(?:\u001b\\[[0-9;]*m)*error(?:\u001b\\[[0-9;]*m)*\\](?:\u001b\\[[0-9;]*m)*\\s+(?!\\S*\\/target\\/java\\/org\\/apache\\/spark\\/)(?:\u001b\\[[0-9;]*m)*(\\S+):(\\d+):(\\d+):\\s+(?:\u001b\\[[0-9;]*m)*([^:]+):\\s+(?:\u001b\\[[0-9;]*m)*(.+?)(?:\u001b\\[[0-9;]*m)*$",
           "file": 1,
           "line": 2,
           "column": 3,

From aaec8d2342a03704b1d9a7db29f4b556c7d27a12 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 16:44:51 -0400
Subject: [PATCH 08/45] target "reference not found" specifically

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 0bac4827456d8..addead791499a 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "^(?:\u001b\\[[0-9;]*m)*\\[(?:\u001b\\[[0-9;]*m)*error(?:\u001b\\[[0-9;]*m)*\\](?:\u001b\\[[0-9;]*m)*\\s+(?!\\S*\\/target\\/java\\/org\\/apache\\/spark\\/)(?:\u001b\\[[0-9;]*m)*(\\S+):(\\d+):(\\d+):\\s+(?:\u001b\\[[0-9;]*m)*([^:]+):\\s+(?:\u001b\\[[0-9;]*m)*(.+?)(?:\u001b\\[[0-9;]*m)*$",
+          "regexp": "\\s+/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)$",
           "file": 1,
           "line": 2,
           "column": 3,

From 3ce9a8109b649b3886103169447643fe72e23f17 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 17:51:51 -0400
Subject: [PATCH 09/45] color codes shitting all over the log

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index addead791499a..a84bb0a575d01 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "\\s+/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)$",
+          "regexp": "/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,

From cd2ef20147136a0d42e6f3f4f12644ada4659d0b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 19:24:48 -0400
Subject: [PATCH 10/45] maybe we want the leading `spark/`?

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index a84bb0a575d01..fa7dac721b62c 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "/__w/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,

From b7aa58bf2a19778965f5d09b2c2a6bb0b04e0d31 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 20:59:08 -0400
Subject: [PATCH 11/45] enable actions debug

---
 .github/problem-matchers/javadoc.json | 2 +-
 .github/workflows/build_and_test.yml  | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index fa7dac721b62c..65261640c34e4 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "/__w/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "/__w/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index cd3cb107b1033..b96ff6929acbb 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1256,6 +1256,7 @@ jobs:
       PYSPARK_DRIVER_PYTHON: python3.9
       PYSPARK_PYTHON: python3.9
       GITHUB_PREV_SHA: ${{ github.event.before }}
+      ACTIONS_RUNNER_DEBUG: true
     container:
       image: ${{ needs.precondition.outputs.image_docs_url_link }}
     steps:
@@ -1346,6 +1347,7 @@ jobs:
         bundle install --retry=100
     - name: Register javadoc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
+    - run: echo "$GITHUB_WORKSPACE"
     - name: Run documentation build for branch-3.5 and branch-4.0
       if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
       run: |

From 02e0bc9ae2244efe5551992a0d4bd0f9e1222ac9 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 22:14:46 -0400
Subject: [PATCH 12/45] cache our own image builds

---
 .github/problem-matchers/javadoc.json |  2 +-
 .github/workflows/build_and_test.yml  | 25 ++++++++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 65261640c34e4..0eafdc19c01d9 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "/__w/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "/__w/spark/spark(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index b96ff6929acbb..fef2bfea335de 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -518,7 +518,10 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_url }}
           # Use the infra image cache to speed up
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ inputs.branch }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-cache:${{ inputs.branch }}
+            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ inputs.branch }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-cache:${{ inputs.branch }},mode=max
       - name: Build and push (Documentation)
         if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).docs == 'true' && hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }}
         id: docker_build_docs
@@ -529,7 +532,10 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_docs_url }}
           # Use the infra image cache to speed up
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
+            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }},mode=max
       - name: Build and push (Linter)
         if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).lint == 'true' && hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }}
         id: docker_build_lint
@@ -540,7 +546,10 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_lint_url }}
           # Use the infra image cache to speed up
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
+            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }},mode=max
       - name: Build and push (SparkR)
         if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).sparkr == 'true' && hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }}
         id: docker_build_sparkr
@@ -551,7 +560,10 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_sparkr_url }}
           # Use the infra image cache to speed up
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
+            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }},mode=max
       - name: Build and push (PySpark with ${{ env.PYSPARK_IMAGE_TO_TEST }})
         if: ${{ inputs.branch != 'branch-3.5' && (fromJson(needs.precondition.outputs.required).pyspark == 'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') && env.PYSPARK_IMAGE_TO_TEST != '' }}
         id: docker_build_pyspark
@@ -563,7 +575,10 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_pyspark_url }}
           # Use the infra image cache to speed up
-          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
+            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }},mode=max
 
 
   precompile:

From 3ada73ce8e3918d3178d3ee13f1d8553aec60e1b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 22:45:11 -0400
Subject: [PATCH 13/45] enable step debug

---
 .github/problem-matchers/javadoc.json | 2 +-
 .github/workflows/build_and_test.yml  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 0eafdc19c01d9..a84bb0a575d01 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "/__w/spark/spark(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index fef2bfea335de..44461b39c4d60 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1272,6 +1272,7 @@ jobs:
       PYSPARK_PYTHON: python3.9
       GITHUB_PREV_SHA: ${{ github.event.before }}
       ACTIONS_RUNNER_DEBUG: true
+      ACTIONS_STEP_DEBUG: true
     container:
       image: ${{ needs.precondition.outputs.image_docs_url_link }}
     steps:

From 869ef23ac671d8b980df89233315b0bd96a6ac19 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 6 Jul 2026 23:14:04 -0400
Subject: [PATCH 14/45] take the whole path?

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index a84bb0a575d01..35bf30d065937 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "/__w/spark/spark/(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,

From 7412eb207344dec770e0b557e53758562ec3ac54 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 7 Jul 2026 09:24:22 -0400
Subject: [PATCH 15/45] leading slash

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 35bf30d065937..5391db0660b14 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "(\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "(/\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,

From 60627803666ef248b54cdf8ab9a15a93cc1084a2 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 7 Jul 2026 12:17:36 -0400
Subject: [PATCH 16/45] try fixing file detection in repo

---
 .github/workflows/build_and_test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 44461b39c4d60..913d8828ed978 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1364,6 +1364,9 @@ jobs:
     - name: Register javadoc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - run: echo "$GITHUB_WORKSPACE"
+    - name: Fix git remote for problem matcher compatibility
+      if: github.repository != 'apache/spark'
+      run: git remote set-url origin "https://github.com/${{ github.repository }}"
     - name: Run documentation build for branch-3.5 and branch-4.0
       if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
       run: |

From ef58ab3bd90d07efebd4a2e05007b8a36c5d2bee Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 7 Jul 2026 14:56:13 -0400
Subject: [PATCH 17/45] try a separate docs workflow

the main build_and_test has too many custom bits that prevent problem matchers from working
---
 .github/workflows/build_and_test.yml |   2 -
 .github/workflows/docs.yml           | 106 +++++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/docs.yml

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 913d8828ed978..8edc3695d1459 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1271,8 +1271,6 @@ jobs:
       PYSPARK_DRIVER_PYTHON: python3.9
       PYSPARK_PYTHON: python3.9
       GITHUB_PREV_SHA: ${{ github.event.before }}
-      ACTIONS_RUNNER_DEBUG: true
-      ACTIONS_STEP_DEBUG: true
     container:
       image: ${{ needs.precondition.outputs.image_docs_url_link }}
     steps:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000000000..abe946c7a7a31
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Documentation
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+    steps:
+      - name: Checkout pull request merge commit
+        uses: actions/checkout@v6
+
+      - name: Install Java
+        uses: actions/setup-java@v5
+        with:
+          distribution: zulu
+          java-version: 17
+
+      - name: Cache SBT and Maven
+        uses: actions/cache@v5
+        with:
+          path: |
+            build/apache-maven-*
+            build/*.jar
+            ~/.sbt
+          key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+          restore-keys: |
+            build-${{ runner.os }}-
+
+      - name: Restore Coursier local repository
+        uses: actions/cache/restore@v5
+        with:
+          path: ~/.cache/coursier
+          key: coursier-${{ runner.os }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+          restore-keys: |
+            coursier-${{ runner.os }}-
+
+      - name: Cache Maven local repository
+        uses: actions/cache@v5
+        with:
+          path: ~/.m2/repository
+          key: docs-maven-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            docs-maven-${{ runner.os }}-
+
+      - name: Set up Ruby 3
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          working-directory: docs
+          bundler: 2.4.22
+          bundler-cache: true
+
+      - name: Install Python 3.12
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: dev/requirements.txt
+
+      - name: Install Python requirements
+        run: python3.12 -m pip install --upgrade -r dev/requirements.txt
+
+      - name: Register javadoc problem matcher
+        run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
+
+      - name: Build docs
+        run: |
+          cd docs
+          SKIP_RDOC=1 bundle exec jekyll build
+
+      - name: Remove javadoc problem matcher
+        if: always()
+        run: echo "::remove-matcher owner=javadoc-doclint::"

From fb8889b01fa9167a141567efd02226ca2bf7e9ce Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Wed, 8 Jul 2026 09:43:25 -0400
Subject: [PATCH 18/45] pin setup-ruby version

---
 .github/workflows/docs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index abe946c7a7a31..eda23411e9bec 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -75,15 +75,15 @@ jobs:
           restore-keys: |
             docs-maven-${{ runner.os }}-
 
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@4dc28cf14d77b0afa6832d9765ac422dbf0dfedd
         with:
           ruby-version: '3.4'
           working-directory: docs
           bundler: 2.4.22
           bundler-cache: true
 
-      - name: Install Python 3.12
+      - name: Install Python
         uses: actions/setup-python@v6
         with:
           python-version: '3.12'
@@ -91,7 +91,7 @@ jobs:
           cache-dependency-path: dev/requirements.txt
 
       - name: Install Python requirements
-        run: python3.12 -m pip install --upgrade -r dev/requirements.txt
+        run: python3 -m pip install --upgrade -r dev/requirements.txt
 
       - name: Register javadoc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/javadoc.json"

From 5e080fe69e8d3427e9e6d119c1423a8591cf1308 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Wed, 8 Jul 2026 10:47:37 -0400
Subject: [PATCH 19/45] setup-ruby is an approved action

https://github.com/apache/infrastructure-actions/blob/d8a5f9fd8ec5c7d228c2c1127eaa0ab2a4653798/approved_patterns.yml#L303
---
 .github/workflows/docs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index eda23411e9bec..090dc64013c98 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -76,7 +76,7 @@ jobs:
             docs-maven-${{ runner.os }}-
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@4dc28cf14d77b0afa6832d9765ac422dbf0dfedd
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.4'
           working-directory: docs

From 81f0e72cb5d2426a03b2920f77134dd272b4dac8 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 08:50:43 -0400
Subject: [PATCH 20/45] switch to `on: push`

---
 .github/workflows/docs.yml | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 090dc64013c98..bfba11ac39ddc 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -20,14 +20,16 @@
 name: Documentation
 
 on:
-  pull_request:
-    types: [opened, reopened, synchronize]
+  push:
+    branches:
+      - '**'
+      - '!branch-*.*'
 
 permissions:
   contents: read
 
 concurrency:
-  group: docs-${{ github.event.pull_request.number }}
+  group: docs-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -39,8 +41,30 @@ jobs:
       LC_ALL: C.UTF-8
       LANG: C.UTF-8
     steps:
-      - name: Checkout pull request merge commit
+      - name: Checkout Spark repository
         uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Check branch freshness
+        if: github.repository != 'apache/spark'
+        run: |
+          git fetch https://github.com/apache/spark.git master --quiet
+          echo "APACHE_SPARK_REF=$(git rev-parse FETCH_HEAD)" >> $GITHUB_ENV
+          commits_behind=$(git rev-list --count HEAD..FETCH_HEAD)
+          echo "Branch is ${commits_behind} commits behind upstream master."
+          # Spark has averaged around 10 commits per day for the past few years.
+          if [ "$commits_behind" -gt 30 ]; then
+            echo "::error::Branch is ${commits_behind} commits behind upstream master. Please update your branch."
+            exit 1
+          fi
+
+      - name: Demonstrate is-changed.py compatibility
+        if: github.repository != 'apache/spark'
+        run: |
+          echo "APACHE_SPARK_REF=$APACHE_SPARK_REF"
+          docs_changed=$(./dev/is-changed.py -m docs)
+          echo "docs module changed: $docs_changed"
 
       - name: Install Java
         uses: actions/setup-java@v5

From dead0da8d67832440c9c20947a2755d057cb4560 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 09:17:58 -0400
Subject: [PATCH 21/45] use pyproject.toml-based deps

---
 .github/workflows/docs.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index bfba11ac39ddc..557adfea9559b 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -112,10 +112,10 @@ jobs:
         with:
           python-version: '3.12'
           cache: pip
-          cache-dependency-path: dev/requirements.txt
+          cache-dependency-path: pyproject.toml
 
       - name: Install Python requirements
-        run: python3 -m pip install --upgrade -r dev/requirements.txt
+        run: python3 -m pip install --upgrade --group dev
 
       - name: Register javadoc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/javadoc.json"

From 5137596962dc2c03225767734084f3ff766aa608 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 09:31:15 -0400
Subject: [PATCH 22/45] bump pip version

---
 .github/workflows/docs.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 557adfea9559b..cfbaa0263498c 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -115,7 +115,10 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install Python requirements
-        run: python3 -m pip install --upgrade --group dev
+        run: |
+          python3 -m pip --version
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --group dev
 
       - name: Register javadoc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/javadoc.json"

From cb3132ad9d5300826f017df1814ae39f001c8401 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 09:53:49 -0400
Subject: [PATCH 23/45] back to requirements.txt

---
 .github/workflows/docs.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index cfbaa0263498c..19641f46422c0 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -112,13 +112,11 @@ jobs:
         with:
           python-version: '3.12'
           cache: pip
-          cache-dependency-path: pyproject.toml
+          cache-dependency-path: dev/requirements.txt
 
       - name: Install Python requirements
         run: |
-          python3 -m pip --version
-          python3 -m pip install --upgrade pip
-          python3 -m pip install --group dev
+          python3 -m pip install --upgrade -r dev/requirements.txt
 
       - name: Register javadoc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/javadoc.json"

From 438fb101e3fc37f43f2e56d424adf1d34bd7472f Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 13:11:55 -0400
Subject: [PATCH 24/45] relay annotations to dedicated workflow

---
 .github/workflows/docs.yml               | 16 ++++-
 .github/workflows/report_annotations.yml | 76 ++++++++++++++++++++++++
 2 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/report_annotations.yml

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 19641f46422c0..d65111a69cbca 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -124,8 +124,22 @@ jobs:
       - name: Build docs
         run: |
           cd docs
-          SKIP_RDOC=1 bundle exec jekyll build
+          SKIP_RDOC=1 bundle exec jekyll build 2>&1 | tee "$RUNNER_TEMP/docs-build.log"
 
       - name: Remove javadoc problem matcher
         if: always()
         run: echo "::remove-matcher owner=javadoc-doclint::"
+
+      - name: Extract annotations
+        if: always()
+        run: |
+          grep -E '^::(error|warning|notice) ' "$RUNNER_TEMP/docs-build.log" \
+            > "$RUNNER_TEMP/annotations.txt" || true
+
+      - name: Upload annotations
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: docs-annotations
+          path: ${{ runner.temp }}/annotations.txt
+          retention-days: 3
diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
new file mode 100644
index 0000000000000..0855db8659c48
--- /dev/null
+++ b/.github/workflows/report_annotations.yml
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Relay workflow: runs on the upstream repo after a fork's workflow completes.
+# Downloads pre-extracted annotation commands and replays them so they appear
+# as inline annotations on the upstream PR's diff.
+#
+# This is necessary because annotations from fork workflows (on: push) don't
+# appear inline on the upstream PR. This lightweight relay runs in the
+# upstream repo's context with checks:write permission, so its annotations
+# do appear inline.
+#
+# To add annotations from a new workflow:
+# 1. In the source workflow, grep ::error/::warning lines from the build log.
+# 2. Upload them as an artifact matching the pattern "*-annotations".
+# 3. Add the source workflow's name to the list below.
+
+name: Report annotations
+
+on:
+  workflow_run:
+    workflows:
+      - "Documentation"
+      # Add other workflows here as needed, e.g.:
+      # - "Build"
+      # - "Linters, licenses, and dependencies"
+    types:
+      - completed
+
+jobs:
+  annotate:
+    name: Post annotations
+    if: >
+      !contains(fromJson('["skipped", "cancelled"]'), github.event.workflow_run.conclusion)
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      checks: write
+    steps:
+      - name: Download annotations
+        id: download
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: "*-annotations"
+          merge-multiple: true
+
+      - name: Replay annotations
+        if: steps.download.outcome == 'success'
+        run: |
+          count=0
+          for f in *.txt; do
+            [ -f "$f" ] || continue
+            [ -s "$f" ] || continue
+            count=$((count + $(wc -l < "$f")))
+            cat "$f"
+          done
+          echo "Replayed ${count} annotation(s) from workflow: ${{ github.event.workflow_run.name }}"

From 8c83e15bb603ec91962414b23a73790fcdae5af9 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 14:03:59 -0400
Subject: [PATCH 25/45] pipefail

---
 .github/workflows/docs.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index d65111a69cbca..191c95634e52d 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -123,6 +123,7 @@ jobs:
 
       - name: Build docs
         run: |
+          set -o pipefail
           cd docs
           SKIP_RDOC=1 bundle exec jekyll build 2>&1 | tee "$RUNNER_TEMP/docs-build.log"
 

From 4faf9e78c7627284f9c80978ccaaba61cf3122c9 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 14:49:05 -0400
Subject: [PATCH 26/45] run problem matchers in report workflow

---
 .github/workflows/docs.yml               | 12 ++------
 .github/workflows/report_annotations.yml | 39 ++++++++++++++++--------
 2 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 191c95634e52d..ffddef2c788b3 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -131,16 +131,10 @@ jobs:
         if: always()
         run: echo "::remove-matcher owner=javadoc-doclint::"
 
-      - name: Extract annotations
-        if: always()
-        run: |
-          grep -E '^::(error|warning|notice) ' "$RUNNER_TEMP/docs-build.log" \
-            > "$RUNNER_TEMP/annotations.txt" || true
-
-      - name: Upload annotations
+      - name: Upload build log
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: docs-annotations
-          path: ${{ runner.temp }}/annotations.txt
+          name: docs-build-log
+          path: ${{ runner.temp }}/docs-build.log
           retention-days: 3
diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 0855db8659c48..33bb2fef0b0a9 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -27,8 +27,9 @@
 # do appear inline.
 #
 # To add annotations from a new workflow:
-# 1. In the source workflow, grep ::error/::warning lines from the build log.
-# 2. Upload them as an artifact matching the pattern "*-annotations".
+# 1. In the source workflow, capture the build log with tee and upload it
+#    as an artifact matching the pattern "*-build-log".
+# 2. Add problem matcher JSON files to .github/problem-matchers/.
 # 3. Add the source workflow's name to the list below.
 
 name: Report annotations
@@ -53,24 +54,38 @@ jobs:
       actions: read
       checks: write
     steps:
-      - name: Download annotations
+      - name: Checkout problem matchers
+        uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/problem-matchers
+
+      - name: Download build logs
         id: download
         uses: actions/download-artifact@v8
         continue-on-error: true
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           run-id: ${{ github.event.workflow_run.id }}
-          pattern: "*-annotations"
+          pattern: "*-build-log"
           merge-multiple: true
 
-      - name: Replay annotations
+      - name: Replay logs through problem matchers
         if: steps.download.outcome == 'success'
         run: |
-          count=0
-          for f in *.txt; do
-            [ -f "$f" ] || continue
-            [ -s "$f" ] || continue
-            count=$((count + $(wc -l < "$f")))
-            cat "$f"
+          # Register all available problem matchers.
+          for matcher in .github/problem-matchers/*.json; do
+            echo "::add-matcher::$matcher"
+          done
+
+          # Replay each build log so the runner creates annotations.
+          for log in *.log; do
+            [ -f "$log" ] || continue
+            echo "--- Replaying $log ---"
+            cat "$log"
+          done
+
+          # Clean up matchers.
+          for matcher in .github/problem-matchers/*.json; do
+            owner=$(python3 -c "import json; print(json.load(open('$matcher'))['problemMatcher'][0]['owner'])")
+            echo "::remove-matcher owner=$owner::"          
           done
-          echo "Replayed ${count} annotation(s) from workflow: ${{ github.event.workflow_run.name }}"

From 7c363a40471695c75533a33a86e7ee0e89efac42 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 16:52:56 -0400
Subject: [PATCH 27/45] is it a problem with the workspace root?

---
 .github/problem-matchers/javadoc.json    |  2 +-
 .github/workflows/docs.yml               | 10 ++++++++
 .github/workflows/report_annotations.yml | 31 +++++++++++++++++++++---
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 5391db0660b14..0f1a14dd69f7b 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "(/\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "([/a-zA-Z]\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
           "file": 1,
           "line": 2,
           "column": 3,
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index ffddef2c788b3..9b9a3e7a31a56 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -138,3 +138,13 @@ jobs:
           name: docs-build-log
           path: ${{ runner.temp }}/docs-build.log
           retention-days: 3
+
+      - name: Upload workspace root
+        if: always()
+        run: echo "$GITHUB_WORKSPACE" > "$RUNNER_TEMP/workspace-root.txt"
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: docs-workspace-root
+          path: ${{ runner.temp }}/workspace-root.txt
+          retention-days: 3
diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 33bb2fef0b0a9..5dc318d8d996a 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -60,7 +60,7 @@ jobs:
           sparse-checkout: .github/problem-matchers
 
       - name: Download build logs
-        id: download
+        id: download-logs
         uses: actions/download-artifact@v8
         continue-on-error: true
         with:
@@ -69,19 +69,42 @@ jobs:
           pattern: "*-build-log"
           merge-multiple: true
 
+      - name: Download workspace root
+        id: download-root
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: "*-workspace-root"
+          merge-multiple: true
+
       - name: Replay logs through problem matchers
-        if: steps.download.outcome == 'success'
+        if: steps.download-logs.outcome == 'success'
         run: |
+          # Read the fork's workspace root so we can strip it from paths.
+          if [ -f workspace-root.txt ]; then
+            ws_root=$(cat workspace-root.txt | tr -d '\n')
+            echo "Stripping workspace root: $ws_root"
+          else
+            ws_root=""
+          fi
+
           # Register all available problem matchers.
           for matcher in .github/problem-matchers/*.json; do
             echo "::add-matcher::$matcher"
           done
 
-          # Replay each build log so the runner creates annotations.
+          # Replay each build log with the fork's workspace prefix stripped
+          # so paths are relative to the repo root.
           for log in *.log; do
             [ -f "$log" ] || continue
             echo "--- Replaying $log ---"
-            cat "$log"
+            if [ -n "$ws_root" ]; then
+              sed "s|${ws_root}/||g" "$log"
+            else
+              cat "$log"
+            fi
           done
 
           # Clean up matchers.

From d9f8d893c2c229871947d07b697d42ac2e0a0def Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 17:43:12 -0400
Subject: [PATCH 28/45] how about a little space

---
 .github/workflows/report_annotations.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 5dc318d8d996a..55003d766bad7 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -101,7 +101,7 @@ jobs:
             [ -f "$log" ] || continue
             echo "--- Replaying $log ---"
             if [ -n "$ws_root" ]; then
-              sed "s|${ws_root}/||g" "$log"
+              sed "s|${ws_root}/| |g" "$log"
             else
               cat "$log"
             fi

From 48e8dae0a1d91005760e670174f141607a66995b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Tue, 14 Jul 2026 18:22:46 -0400
Subject: [PATCH 29/45] don't we need a full checkout?

---
 .github/workflows/report_annotations.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 55003d766bad7..2eca53c7d3c59 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -54,10 +54,10 @@ jobs:
       actions: read
       checks: write
     steps:
-      - name: Checkout problem matchers
+      - name: Checkout Spark
         uses: actions/checkout@v6
-        with:
-          sparse-checkout: .github/problem-matchers
+        # with:
+        #   sparse-checkout: .github/problem-matchers
 
       - name: Download build logs
         id: download-logs

From 274e4f0b2d576b15e4a9cd99a71e1a1927593b6f Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Thu, 16 Jul 2026 10:00:17 -0400
Subject: [PATCH 30/45] restore build_and_test to master

---
 .github/workflows/build_and_test.yml | 264 ++-------------------------
 1 file changed, 18 insertions(+), 246 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 44e732c0d320c..0bcda10e3b6c3 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -185,7 +185,6 @@ jobs:
         fi
     - name: Check envs
       id: check-envs
-      if: inputs.branch != 'branch-3.5'
       env: ${{ fromJSON(inputs.envs) }}
       run: |
         if [[ "${{ fromJson(steps.set-outputs.outputs.required).pyspark }}"  == 'true' || "${{ fromJson(steps.set-outputs.outputs.required).pyspark-pandas }}"  == 'true' ]]; then
@@ -253,17 +252,10 @@ jobs:
       run: |
         # Set the image URL for job "docs"
         # Should delete the link and directly use image_docs_url after SPARK 3.x EOL
-        if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then
-          echo "image_docs_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
-          echo "image_lint_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
-          echo "image_sparkr_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
-          echo "image_pyspark_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
-        else
-          echo "image_docs_url_link=${{ steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT
-          echo "image_lint_url_link=${{ steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT
-          echo "image_sparkr_url_link=${{ steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT
-          echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT
-        fi
+        echo "image_docs_url_link=${{ steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT
+        echo "image_lint_url_link=${{ steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT
+        echo "image_sparkr_url_link=${{ steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT
+        echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT
 
   # Build: build Spark and run the tests for specified modules.
   build:
@@ -433,10 +425,6 @@ jobs:
         export TERM=vt100
         # Hive "other tests" test needs larger metaspace size based on experiment.
         if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi
-        # SPARK-46283: should delete the following env replacement after SPARK 3.x EOL
-        if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" =~ ^branch-3 ]]; then
-          MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /}
-        fi
         if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
           export SKIP_SCALA_BUILD=true
           echo "Reusing precompiled artifact, skipping local SBT build."
@@ -508,22 +496,8 @@ jobs:
         uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
-      - name: Build and push for branch-3.5
-        if: inputs.branch == 'branch-3.5'
-        id: docker_build
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
-        with:
-          context: ./dev/infra/
-          push: true
-          tags: |
-            ${{ needs.precondition.outputs.image_url }}
-          # Use the infra image cache to speed up
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-cache:${{ inputs.branch }}
-            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ inputs.branch }}
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-cache:${{ inputs.branch }},mode=max
       - name: Build and push (Documentation)
-        if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).docs == 'true' && hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }}
+        if: ${{ fromJson(needs.precondition.outputs.required).docs == 'true' && hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }}
         id: docker_build_docs
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
@@ -532,12 +506,9 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_docs_url }}
           # Use the infra image cache to speed up
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
-            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }},mode=max
+          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }}
       - name: Build and push (Linter)
-        if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).lint == 'true' && hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }}
+        if: ${{ fromJson(needs.precondition.outputs.required).lint == 'true' && hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }}
         id: docker_build_lint
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
@@ -546,12 +517,9 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_lint_url }}
           # Use the infra image cache to speed up
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
-            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }},mode=max
+          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }}
       - name: Build and push (SparkR)
-        if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).sparkr == 'true' && hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }}
+        if: ${{ fromJson(needs.precondition.outputs.required).sparkr == 'true' && hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }}
         id: docker_build_sparkr
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
         with:
@@ -560,12 +528,9 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_sparkr_url }}
           # Use the infra image cache to speed up
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
-            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }},mode=max
+          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
       - name: Build and push (PySpark with ${{ env.PYSPARK_IMAGE_TO_TEST }})
-        if: ${{ inputs.branch != 'branch-3.5' && (fromJson(needs.precondition.outputs.required).pyspark == 'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') && env.PYSPARK_IMAGE_TO_TEST != '' }}
+        if: ${{ (fromJson(needs.precondition.outputs.required).pyspark == 'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') && env.PYSPARK_IMAGE_TO_TEST != '' }}
         id: docker_build_pyspark
         env: ${{ fromJSON(inputs.envs) }}
         uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
@@ -575,10 +540,7 @@ jobs:
           tags: |
             ${{ needs.precondition.outputs.image_pyspark_url }}
           # Use the infra image cache to speed up
-          cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
-            type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }},mode=max
+          cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-${{ env.PYSPARK_IMAGE_TO_TEST }}-cache:${{ inputs.branch }}
 
 
   precompile:
@@ -811,12 +773,7 @@ jobs:
           export SKIP_PACKAGING=false
           echo "Python Packaging Tests Enabled!"
         fi
-        if [ ! -z "$PYTHON_TO_TEST" ]; then
-          ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST"
-        else
-          # For branch-3.5 and below, it uses the default Python versions.
-          ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST"
-        fi
+        ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST"
     - name: Run tests for old client
       env:
         SPARK_TESTING: 1
@@ -1033,21 +990,11 @@ jobs:
       uses: actions/setup-python@v6
       with:
         python-version: '3.12'
-    - name: Install dependencies for Python CodeGen check (branch-3.5, branch-4.0, branch-4.1)
-      if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0' || inputs.branch == 'branch-4.1'
-      run: |
-        python3.12 -m pip install 'black==23.12.1' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
-        python3.12 -m pip list
     - name: Install dependencies for Python CodeGen check
-      if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' }}
       run: |
         python3.12 -m pip install 'ruff==0.14.8' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
         python3.12 -m pip list
-    - name: Python CodeGen check for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: ./dev/connect-check-protos.py
     - name: Python CodeGen check
-      if: inputs.branch != 'branch-3.5'
       run: ./dev/check-protos.py
 
   # Static analysis
@@ -1153,87 +1100,23 @@ jobs:
       if: hashFiles('dev/structured_logging_style.py') != ''
       shell: 'script -q -e -c "bash {0}"'
       run: |
-        if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
-          python3.9 ./dev/structured_logging_style.py
-        elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then
-          python3.12 ./dev/structured_logging_style.py
-        elif [[ "$BRANCH" == 'branch-4.1' ]]; then
-          python3.11 ./dev/structured_logging_style.py
-        else
-          python3.12 ./dev/structured_logging_style.py
-        fi
+        python3.12 ./dev/structured_logging_style.py
     - name: Java linter
       run: ./dev/lint-java
     - name: Spark connect jvm client mima check
       run: ./dev/connect-jvm-client-mima-check
-    - name: Install Python linter dependencies for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: |
-        # SPARK-45212: Copy from https://github.com/apache/spark/blob/555c8def51e5951c7bf5165a332795e9e330ec9d/.github/workflows/build_and_test.yml#L631-L638
-        # Should delete this section after SPARK 3.5 EOL.
-        python3.9 -m pip install 'flake8==3.9.0' pydata_sphinx_theme 'mypy==0.982' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' numpydoc 'jinja2<3.0.0' 'black==22.6.0'
-        python3.9 -m pip install 'pandas-stubs==1.2.0.53' ipython 'grpcio==1.56.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0'
     - name: List Python packages
       shell: 'script -q -e -c "bash {0}"'
       run: |
         lsb_release -a
-        if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
-          python3.9 --version
-          python3.9 -m pip list
-        elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then
-          python3.12 --version
-          python3.12 -m pip list
-        elif [[ "$BRANCH" == 'branch-4.1' ]]; then
-          python3.11 --version
-          python3.11 -m pip list
-        else
-          python3.12 --version
-          python3.12 -m pip list
-        fi
+        python3.12 --version
+        python3.12 -m pip list
     - name: Python linter
       shell: 'script -q -e -c "bash {0}"'
       run: |
-        if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then
-          PYTHON_EXECUTABLE=python3.9 ./dev/lint-python
-        elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then
-          PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
-        elif [[ "$BRANCH" == 'branch-4.1' ]]; then
-          PYTHON_EXECUTABLE=python3.11 ./dev/lint-python
-        else
-          PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
-        fi
-    # Should delete this section after SPARK 3.5 EOL.
-    - name: Install dependencies for Python code generation check for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: |
-        # See more in "Installation" https://docs.buf.build/installation#tarball
-        curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz
-        mkdir -p $HOME/buf
-        tar -xvzf buf-Linux-x86_64.tar.gz -C $HOME/buf --strip-components 1
-        rm buf-Linux-x86_64.tar.gz
-        python3.9 -m pip install 'protobuf==4.25.1' 'mypy-protobuf==3.3.0'
-    # Should delete this section after SPARK 3.5 EOL.
-    - name: Python code generation check for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: if test -f ./dev/connect-check-protos.py; then PATH=$PATH:$HOME/buf/bin PYTHON_EXECUTABLE=python3.9 ./dev/connect-check-protos.py; fi
-    # Should delete this section after SPARK 3.5 EOL.
-    - name: Install JavaScript linter dependencies for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: |
-        apt update
-        apt-get install -y nodejs npm
+        PYTHON_EXECUTABLE=python3.12 ./dev/lint-python
     - name: JS linter
       run: ./dev/lint-js
-    # Should delete this section after SPARK 3.5 EOL.
-    - name: Install R linter dependencies for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: |
-        apt update
-        apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev \
-          libfontconfig1-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev \
-          libtiff5-dev libjpeg-dev
-        Rscript -e "install.packages(c('remotes'), repos='https://cloud.r-project.org/')"
-        Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')"
     - name: Install R linter dependencies and SparkR
       run: ./R/install-dev.sh
     - name: R linter
@@ -1322,32 +1205,7 @@ jobs:
       with:
         distribution: zulu
         java-version: ${{ inputs.java }}
-    - name: Install dependencies for documentation generation for branch-3.5
-      if: inputs.branch == 'branch-3.5'
-      run: |
-        # pandoc is required to generate PySpark APIs as well in nbsphinx.
-        apt-get update -y
-        apt-get install -y libcurl4-openssl-dev pandoc
-        apt-get install -y ruby ruby-dev
-        Rscript -e "install.packages(c('remotes', 'testthat', 'knitr', 'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'), repos='https://cloud.r-project.org/')"
-        Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')"
-        Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')"
-        # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5
-        python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
-        python3.9 -m pip install ipython_genutils # See SPARK-38517
-        python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.22' pyarrow pandas 'plotly<6.0.0'
-        python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421
-    - name: List Python packages for branch-3.5 and branch-4.0
-      if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
-      run: python3.9 -m pip list
-    - name: List Python packages for branch-4.2 and branch-4.x (Python 3.12)
-      if: inputs.branch == 'branch-4.2' || inputs.branch == 'branch-4.x'
-      run: python3.12 -m pip list
-    - name: List Python packages for branch-4.1
-      if: inputs.branch == 'branch-4.1'
-      run: python3.11 -m pip list
     - name: List Python packages
-      if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' && inputs.branch != 'branch-4.2' && inputs.branch != 'branch-4.x' }}
       run: |
         lsb_release -a
         python3.12 -m pip list
@@ -1359,84 +1217,7 @@ jobs:
         gem install bundler -v 2.4.22
         cd docs
         bundle install --retry=100
-    - name: Register javadoc problem matcher
-      run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
-    - run: echo "$GITHUB_WORKSPACE"
-    - name: Fix git remote for problem matcher compatibility
-      if: github.repository != 'apache/spark'
-      run: git remote set-url origin "https://github.com/${{ github.repository }}"
-    - name: Run documentation build for branch-3.5 and branch-4.0
-      if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0'
-      run: |
-        # We need this link to make sure `python3` points to `python3.9` which contains the prerequisite packages.
-        ln -s "$(which python3.9)" "/usr/local/bin/python3"
-        # Build docs first with SKIP_API to ensure they are buildable without requiring any
-        # language docs to be built beforehand.
-        cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd ..
-        if [ -f "./dev/is-changed.py" ]; then
-          # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs
-          pyspark_modules=`cd dev && python3.9 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"`
-          if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi
-          if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi
-        fi
-        # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC`
-        echo "SKIP_ERRORDOC: $SKIP_ERRORDOC"
-        echo "SKIP_SCALADOC: $SKIP_SCALADOC"
-        echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC"
-        echo "SKIP_RDOC: $SKIP_RDOC"
-        echo "SKIP_SQLDOC: $SKIP_SQLDOC"
-        cd docs
-        bundle exec jekyll build
-    - name: Run documentation build for branch-4.2 and branch-4.x (Python 3.12)
-      if: inputs.branch == 'branch-4.2' || inputs.branch == 'branch-4.x'
-      run: |
-        # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages.
-        ln -s "$(which python3.12)" "/usr/local/bin/python3"
-        # Build docs first with SKIP_API to ensure they are buildable without requiring any
-        # language docs to be built beforehand.
-        cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd ..
-        if [ -f "./dev/is-changed.py" ]; then
-          # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs
-          pyspark_modules=`cd dev && python3.12 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"`
-          if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi
-          if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi
-        fi
-        export PYSPARK_DRIVER_PYTHON=python3.12
-        export PYSPARK_PYTHON=python3.12
-        # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC`
-        echo "SKIP_ERRORDOC: $SKIP_ERRORDOC"
-        echo "SKIP_SCALADOC: $SKIP_SCALADOC"
-        echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC"
-        echo "SKIP_RDOC: $SKIP_RDOC"
-        echo "SKIP_SQLDOC: $SKIP_SQLDOC"
-        cd docs
-        bundle exec jekyll build
-    - name: Run documentation build for branch-4.1
-      if: inputs.branch == 'branch-4.1'
-      run: |
-        # We need this link to make sure `python3` points to `python3.11` which contains the prerequisite packages.
-        ln -s "$(which python3.11)" "/usr/local/bin/python3"
-        # Build docs first with SKIP_API to ensure they are buildable without requiring any
-        # language docs to be built beforehand.
-        cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd ..
-        if [ -f "./dev/is-changed.py" ]; then
-          # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs
-          pyspark_modules=`cd dev && python3.11 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"`
-          if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi
-          if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi
-        fi
-        export PYSPARK_DRIVER_PYTHON=python3.11
-        export PYSPARK_PYTHON=python3.11
-        # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC`
-        echo "SKIP_ERRORDOC: $SKIP_ERRORDOC"
-        echo "SKIP_SCALADOC: $SKIP_SCALADOC"
-        echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC"
-        echo "SKIP_RDOC: $SKIP_RDOC"
-        echo "SKIP_SQLDOC: $SKIP_SQLDOC"
-        cd docs
-        bundle exec jekyll build
     - name: Run documentation build
-      if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' && inputs.branch != 'branch-4.2' && inputs.branch != 'branch-4.x' }}
       run: |
         # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages.
         ln -s "$(which python3.12)" "/usr/local/bin/python3"
@@ -1459,8 +1240,6 @@ jobs:
         echo "SKIP_SQLDOC: $SKIP_SQLDOC"
         cd docs
         bundle exec jekyll build
-    - name: Remove problem matcher
-      run: echo "::remove-matcher owner=javadoc-doclint::"
     - name: Tar documentation
       if: github.repository != 'apache/spark'
       run: tar cjf site.tar.bz2 docs/_site
@@ -1573,7 +1352,6 @@ jobs:
           spark.sql.autoBroadcastJoinThreshold=-1
           spark.sql.join.forceApplyShuffledHashJoin=true
     - name: Run TPC-DS queries on collated data
-      if: inputs.branch != 'branch-3.5'
       run: |
         SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSCollationQueryTestSuite"
     - name: Upload test results to report
@@ -1777,13 +1555,7 @@ jobs:
           export PVC_TESTS_VM_PATH=$PVC_TMP_DIR
           minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0 --uid=185 &
           kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
-          if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then
-            kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true
-          elif [[ "${{ inputs.branch }}" == 'branch-4.0' ]]; then
-            kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.11.0/installer/volcano-development.yaml || true
-          else
-            kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml || true
-          fi
+          kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml || true
           eval $(minikube docker-env)
           build/sbt -Phadoop-3 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test"
       - name: Upload Spark on K8S integration tests log files

From acee82ecaf629ddaf3e8f27a382ab0c3b22fa610 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Thu, 16 Jul 2026 10:00:46 -0400
Subject: [PATCH 31/45] remove suppression of lintUnusedKeysOnLoad

that's for another PR
---
 project/GlobalSettings.scala | 32 --------------------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 project/GlobalSettings.scala

diff --git a/project/GlobalSettings.scala b/project/GlobalSettings.scala
deleted file mode 100644
index c36f819c10261..0000000000000
--- a/project/GlobalSettings.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt._
-import sbt.Keys._
-
-/**
- * Suppresses the "N keys that are not used by any other settings/tasks" lint
- * that fires on project load. These are a side effect of applying broad shared
- * settings (e.g. Checkstyle, publishMavenStyle) to all projects, including
- * assembly and other modules that don't use all of them.
- */
-object GlobalSettingsPlugin extends AutoPlugin {
-  override def trigger = allRequirements
-  override def globalSettings: Seq[Setting[_]] = Seq(
-    lintUnusedKeysOnLoad := false
-  )
-}

From e49c8ae4b356080b01f8a44493db8869fb398345 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 20:37:28 -0400
Subject: [PATCH 32/45] match errors only and ignore /target/ paths

---
 .github/problem-matchers/javadoc.json | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 0f1a14dd69f7b..894e7bf89b101 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,12 +5,11 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "([/a-zA-Z]\\S+):(\\d+):(\\d+):\\s+([^:]+):\\s+(reference not found)",
+          "regexp": "(?:^|(?<=\\s))([/a-zA-Z](?:(?!/target/)\\S)*):(\\d+):(\\d+):\\s+error:\\s+(.+)",
           "file": 1,
           "line": 2,
           "column": 3,
-          "severity": 4,
-          "message": 5
+          "message": 4
         }
       ]
     }

From ceecc607558f3eb8cb4737a1c925760d041c032b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 20:40:23 -0400
Subject: [PATCH 33/45] try merging docs back into main build_and_test workflow

---
 .github/workflows/build_and_test.yml     |  15 ++-
 .github/workflows/docs.yml               | 150 -----------------------
 .github/workflows/report_annotations.yml |  37 ++----
 3 files changed, 21 insertions(+), 181 deletions(-)
 delete mode 100644 .github/workflows/docs.yml

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 0bcda10e3b6c3..1e4689e9134aa 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1217,6 +1217,8 @@ jobs:
         gem install bundler -v 2.4.22
         cd docs
         bundle install --retry=100
+    - name: Register javadoc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build
       run: |
         # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages.
@@ -1239,7 +1241,18 @@ jobs:
         echo "SKIP_RDOC: $SKIP_RDOC"
         echo "SKIP_SQLDOC: $SKIP_SQLDOC"
         cd docs
-        bundle exec jekyll build
+        set -o pipefail
+        bundle exec jekyll build 2>&1 | tee /tmp/docs-build.log
+    - name: Remove javadoc problem matcher
+      if: always()
+      run: echo "::remove-matcher owner=javadoc-doclint::"
+    - name: Upload build log
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: docs-build-log
+        path: /tmp/docs-build.log
+        retention-days: 3
     - name: Tar documentation
       if: github.repository != 'apache/spark'
       run: tar cjf site.tar.bz2 docs/_site
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
deleted file mode 100644
index 9b9a3e7a31a56..0000000000000
--- a/.github/workflows/docs.yml
+++ /dev/null
@@ -1,150 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: Documentation
-
-on:
-  push:
-    branches:
-      - '**'
-      - '!branch-*.*'
-
-permissions:
-  contents: read
-
-concurrency:
-  group: docs-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    env:
-      LC_ALL: C.UTF-8
-      LANG: C.UTF-8
-    steps:
-      - name: Checkout Spark repository
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Check branch freshness
-        if: github.repository != 'apache/spark'
-        run: |
-          git fetch https://github.com/apache/spark.git master --quiet
-          echo "APACHE_SPARK_REF=$(git rev-parse FETCH_HEAD)" >> $GITHUB_ENV
-          commits_behind=$(git rev-list --count HEAD..FETCH_HEAD)
-          echo "Branch is ${commits_behind} commits behind upstream master."
-          # Spark has averaged around 10 commits per day for the past few years.
-          if [ "$commits_behind" -gt 30 ]; then
-            echo "::error::Branch is ${commits_behind} commits behind upstream master. Please update your branch."
-            exit 1
-          fi
-
-      - name: Demonstrate is-changed.py compatibility
-        if: github.repository != 'apache/spark'
-        run: |
-          echo "APACHE_SPARK_REF=$APACHE_SPARK_REF"
-          docs_changed=$(./dev/is-changed.py -m docs)
-          echo "docs module changed: $docs_changed"
-
-      - name: Install Java
-        uses: actions/setup-java@v5
-        with:
-          distribution: zulu
-          java-version: 17
-
-      - name: Cache SBT and Maven
-        uses: actions/cache@v5
-        with:
-          path: |
-            build/apache-maven-*
-            build/*.jar
-            ~/.sbt
-          key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
-          restore-keys: |
-            build-${{ runner.os }}-
-
-      - name: Restore Coursier local repository
-        uses: actions/cache/restore@v5
-        with:
-          path: ~/.cache/coursier
-          key: coursier-${{ runner.os }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
-          restore-keys: |
-            coursier-${{ runner.os }}-
-
-      - name: Cache Maven local repository
-        uses: actions/cache@v5
-        with:
-          path: ~/.m2/repository
-          key: docs-maven-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            docs-maven-${{ runner.os }}-
-
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.4'
-          working-directory: docs
-          bundler: 2.4.22
-          bundler-cache: true
-
-      - name: Install Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: '3.12'
-          cache: pip
-          cache-dependency-path: dev/requirements.txt
-
-      - name: Install Python requirements
-        run: |
-          python3 -m pip install --upgrade -r dev/requirements.txt
-
-      - name: Register javadoc problem matcher
-        run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
-
-      - name: Build docs
-        run: |
-          set -o pipefail
-          cd docs
-          SKIP_RDOC=1 bundle exec jekyll build 2>&1 | tee "$RUNNER_TEMP/docs-build.log"
-
-      - name: Remove javadoc problem matcher
-        if: always()
-        run: echo "::remove-matcher owner=javadoc-doclint::"
-
-      - name: Upload build log
-        if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: docs-build-log
-          path: ${{ runner.temp }}/docs-build.log
-          retention-days: 3
-
-      - name: Upload workspace root
-        if: always()
-        run: echo "$GITHUB_WORKSPACE" > "$RUNNER_TEMP/workspace-root.txt"
-      - uses: actions/upload-artifact@v7
-        if: always()
-        with:
-          name: docs-workspace-root
-          path: ${{ runner.temp }}/workspace-root.txt
-          retention-days: 3
diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 2eca53c7d3c59..3620dfc2adb79 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -37,9 +37,11 @@ name: Report annotations
 on:
   workflow_run:
     workflows:
-      - "Documentation"
+      # These must match the `name:` field of the source workflow exactly.
+      # There is no way to reference a workflow by filename.
+      - "Build and test"
       # Add other workflows here as needed, e.g.:
-      # - "Build"
+      # - "Build and test using Maven"
       # - "Linters, licenses, and dependencies"
     types:
       - completed
@@ -56,8 +58,6 @@ jobs:
     steps:
       - name: Checkout Spark
         uses: actions/checkout@v6
-        # with:
-        #   sparse-checkout: .github/problem-matchers
 
       - name: Download build logs
         id: download-logs
@@ -69,46 +69,23 @@ jobs:
           pattern: "*-build-log"
           merge-multiple: true
 
-      - name: Download workspace root
-        id: download-root
-        uses: actions/download-artifact@v8
-        continue-on-error: true
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          run-id: ${{ github.event.workflow_run.id }}
-          pattern: "*-workspace-root"
-          merge-multiple: true
-
       - name: Replay logs through problem matchers
         if: steps.download-logs.outcome == 'success'
         run: |
-          # Read the fork's workspace root so we can strip it from paths.
-          if [ -f workspace-root.txt ]; then
-            ws_root=$(cat workspace-root.txt | tr -d '\n')
-            echo "Stripping workspace root: $ws_root"
-          else
-            ws_root=""
-          fi
-
           # Register all available problem matchers.
           for matcher in .github/problem-matchers/*.json; do
             echo "::add-matcher::$matcher"
           done
 
-          # Replay each build log with the fork's workspace prefix stripped
-          # so paths are relative to the repo root.
+          # Replay each build log so the runner creates annotations.
           for log in *.log; do
             [ -f "$log" ] || continue
             echo "--- Replaying $log ---"
-            if [ -n "$ws_root" ]; then
-              sed "s|${ws_root}/| |g" "$log"
-            else
-              cat "$log"
-            fi
+            cat "$log"
           done
 
           # Clean up matchers.
           for matcher in .github/problem-matchers/*.json; do
             owner=$(python3 -c "import json; print(json.load(open('$matcher'))['problemMatcher'][0]['owner'])")
-            echo "::remove-matcher owner=$owner::"          
+            echo "::remove-matcher owner=$owner::"
           done

From 3785585bd3dc4dee669b35b118cada9fd649cf97 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 20:55:24 -0400
Subject: [PATCH 34/45] pipefail requires bash?

---
 .github/workflows/build_and_test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 1e4689e9134aa..aa0f94e7c24f8 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1220,6 +1220,7 @@ jobs:
     - name: Register javadoc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build
+      shell: bash
       run: |
         # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages.
         ln -s "$(which python3.12)" "/usr/local/bin/python3"

From 2d0a4e669aee9bba73bb67fd295d359cbf2bd2d4 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 21:16:01 -0400
Subject: [PATCH 35/45] restore workspace root cleanup

---
 .github/workflows/build_and_test.yml     | 10 ++++++++
 .github/workflows/report_annotations.yml | 32 +++++++++++++++++++++---
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index aa0f94e7c24f8..f63d1080068d0 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1254,6 +1254,16 @@ jobs:
         name: docs-build-log
         path: /tmp/docs-build.log
         retention-days: 3
+    - name: Upload workspace root
+      if: always()
+      run: echo "$GITHUB_WORKSPACE" > /tmp/workspace-root.txt
+    - name: Upload workspace root artifact
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: docs-workspace-root
+        path: /tmp/workspace-root.txt
+        retention-days: 3
     - name: Tar documentation
       if: github.repository != 'apache/spark'
       run: tar cjf site.tar.bz2 docs/_site
diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 3620dfc2adb79..a6a65cb84621f 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -29,8 +29,10 @@
 # To add annotations from a new workflow:
 # 1. In the source workflow, capture the build log with tee and upload it
 #    as an artifact matching the pattern "*-build-log".
-# 2. Add problem matcher JSON files to .github/problem-matchers/.
-# 3. Add the source workflow's name to the list below.
+# 2. Upload $GITHUB_WORKSPACE as an artifact matching "*-workspace-root"
+#    so the relay can strip the runner's absolute path prefix from log entries.
+# 3. Add problem matcher JSON files to .github/problem-matchers/.
+# 4. Add the source workflow's name to the list below.
 
 name: Report annotations
 
@@ -69,19 +71,41 @@ jobs:
           pattern: "*-build-log"
           merge-multiple: true
 
+      - name: Download workspace roots
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: "*-workspace-root"
+          merge-multiple: true
+
       - name: Replay logs through problem matchers
         if: steps.download-logs.outcome == 'success'
         run: |
+          # Read the fork's workspace root and strip it from paths so GitHub
+          # can map annotations to repo-relative files in the PR diff.
+          # (Absolute paths vary by runner: /home/runner/work/..., /__w/..., etc.)
+          ws_root=""
+          if [ -f workspace-root.txt ]; then
+            ws_root=$(tr -d '\n' < workspace-root.txt)
+            echo "Stripping workspace root: $ws_root"
+          fi
+
           # Register all available problem matchers.
           for matcher in .github/problem-matchers/*.json; do
             echo "::add-matcher::$matcher"
           done
 
-          # Replay each build log so the runner creates annotations.
+          # Replay each build log with absolute paths stripped to repo-relative.
           for log in *.log; do
             [ -f "$log" ] || continue
             echo "--- Replaying $log ---"
-            cat "$log"
+            if [ -n "$ws_root" ]; then
+              sed "s|${ws_root}/||g" "$log"
+            else
+              cat "$log"
+            fi
           done
 
           # Clean up matchers.

From 49dab2d4126de241336ac73266809e67bd725f04 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 22:28:44 -0400
Subject: [PATCH 36/45] ansi codes again

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 894e7bf89b101..ebc368e42a23f 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "(?:^|(?<=\\s))([/a-zA-Z](?:(?!/target/)\\S)*):(\\d+):(\\d+):\\s+error:\\s+(.+)",
+          "regexp": "(/(?:(?!/target/)\\S)*):(\\d+):(\\d+):\\s+error:\\s+(.+)",
           "file": 1,
           "line": 2,
           "column": 3,

From 9dc70feb29bf85fc14d2c332cfa06316f1a9a50a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sat, 18 Jul 2026 23:50:00 -0400
Subject: [PATCH 37/45] fix the negative lookahead

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index ebc368e42a23f..00f3e059dc533 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "(/(?:(?!/target/)\\S)*):(\\d+):(\\d+):\\s+error:\\s+(.+)",
+          "regexp": "\\]\\s+[^\\s/]*(/(?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+\\w+:\\s+(.+)",
           "file": 1,
           "line": 2,
           "column": 3,

From 540a6a9ad94c688c20ef16d367c7bf04bee1f858 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 19 Jul 2026 08:44:43 -0400
Subject: [PATCH 38/45] match errors specifically

---
 .github/problem-matchers/javadoc.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 00f3e059dc533..9396b8cdd27c6 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "\\]\\s+[^\\s/]*(/(?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+\\w+:\\s+(.+)",
+          "regexp": "\\]\\s+[^\\s/]*(/(?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+error:\\s+(.+)",
           "file": 1,
           "line": 2,
           "column": 3,

From b49a38f5b703c1d56eedfbc0baf3f652c4f5794b Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 19 Jul 2026 08:55:17 -0400
Subject: [PATCH 39/45] match relative path only

---
 .github/problem-matchers/javadoc.json |  2 +-
 .github/workflows/build_and_test.yml  | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json
index 9396b8cdd27c6..94a776477ec11 100644
--- a/.github/problem-matchers/javadoc.json
+++ b/.github/problem-matchers/javadoc.json
@@ -5,7 +5,7 @@
       "severity": "error",
       "pattern": [
         {
-          "regexp": "\\]\\s+[^\\s/]*(/(?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+error:\\s+(.+)",
+          "regexp": "\\]\\s+\\S*/spark/spark/((?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+error:\\s+(.+)",
           "file": 1,
           "line": 2,
           "column": 3,
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index f63d1080068d0..9e28c8320f6b2 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1217,6 +1217,7 @@ jobs:
         gem install bundler -v 2.4.22
         cd docs
         bundle install --retry=100
+    - run: echo "$GITHUB_WORKSPACE"
     - name: Register javadoc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build
@@ -1243,7 +1244,8 @@ jobs:
         echo "SKIP_SQLDOC: $SKIP_SQLDOC"
         cd docs
         set -o pipefail
-        bundle exec jekyll build 2>&1 | tee /tmp/docs-build.log
+        bundle exec jekyll build 2>&1 \
+          | tee "$GITHUB_WORKSPACE/docs-build.log"
     - name: Remove javadoc problem matcher
       if: always()
       run: echo "::remove-matcher owner=javadoc-doclint::"
@@ -1252,17 +1254,17 @@ jobs:
       uses: actions/upload-artifact@v7
       with:
         name: docs-build-log
-        path: /tmp/docs-build.log
+        path: ${{ github.workspace }}/docs-build.log
         retention-days: 3
     - name: Upload workspace root
       if: always()
-      run: echo "$GITHUB_WORKSPACE" > /tmp/workspace-root.txt
+      run: echo "$GITHUB_WORKSPACE" > "$GITHUB_WORKSPACE/workspace-root.txt"
     - name: Upload workspace root artifact
       if: always()
       uses: actions/upload-artifact@v7
       with:
         name: docs-workspace-root
-        path: /tmp/workspace-root.txt
+        path: ${{ github.workspace }}/workspace-root.txt
         retention-days: 3
     - name: Tar documentation
       if: github.repository != 'apache/spark'

From 667cf6468b2931dda31e8c20f5510d7fcf69e59d Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 19 Jul 2026 09:38:15 -0400
Subject: [PATCH 40/45] try fixing repo lookup

---
 .github/workflows/build_and_test.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 9e28c8320f6b2..e34df6d7271cf 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1218,6 +1218,15 @@ jobs:
         cd docs
         bundle install --retry=100
     - run: echo "$GITHUB_WORKSPACE"
+    - name: Fix remote URL for problem matcher (fork runs only)
+      # The checkout above always uses repository: apache/spark, so .git/config
+      # contains url = https://github.com/apache/spark. When running from a fork,
+      # GITHUB_REPOSITORY is e.g. nchammas/spark, but the runner verifies file
+      # annotations by looking for GITHUB_REPOSITORY in .git/config -- it won't
+      # find it and silently drops every file path from annotations. Re-pointing
+      # origin to the fork URL fixes the mismatch without changing the working tree.
+      if: github.repository != 'apache/spark'
+      run: git remote set-url origin https://github.com/${{ github.repository }}
     - name: Register javadoc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build

From 66b337c6f1c0bb5df673fa677fb69a859e2dba50 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 19 Jul 2026 10:07:22 -0400
Subject: [PATCH 41/45] fix trigger for report job

---
 .github/workflows/report_annotations.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index a6a65cb84621f..79cd63ee4d8d3 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -39,12 +39,12 @@ name: Report annotations
 on:
   workflow_run:
     workflows:
-      # These must match the `name:` field of the source workflow exactly.
-      # There is no way to reference a workflow by filename.
-      - "Build and test"
-      # Add other workflows here as needed, e.g.:
+      # These must match the `name:` field of the top-level (caller) workflow
+      # exactly. build_and_test.yml is a reusable workflow (workflow_call only)
+      # so it never appears as a standalone run; the caller is what fires here.
+      - "Build"
+      # Add other callers here as needed, e.g.:
       # - "Build and test using Maven"
-      # - "Linters, licenses, and dependencies"
     types:
       - completed
 

From 4f113436e3712f69dce1d46c12a3115c1b1a937a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Sun, 19 Jul 2026 20:58:10 -0400
Subject: [PATCH 42/45] debug info in relay

---
 .github/workflows/report_annotations.yml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml
index 79cd63ee4d8d3..baea64d4e4c92 100644
--- a/.github/workflows/report_annotations.yml
+++ b/.github/workflows/report_annotations.yml
@@ -80,6 +80,29 @@ jobs:
           pattern: "*-workspace-root"
           merge-multiple: true
 
+      - name: Print workspace info
+        run: |
+          echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE"
+          echo "Downloaded files:"
+          ls -la
+          echo "workspace-root.txt contents:"
+          cat workspace-root.txt 2>/dev/null || echo "(not found)"
+
+      - name: Replay logs through problem matchers (raw, no path stripping)
+        if: steps.download-logs.outcome == 'success'
+        run: |
+          # Debug replay: emit lines exactly as-is so we can see what the
+          # problem matcher receives before any path rewriting is applied.
+          for matcher in .github/problem-matchers/*.json; do
+            echo "::add-matcher::$matcher"
+          done
+
+          for log in *.log; do
+            [ -f "$log" ] || continue
+            echo "--- Replaying (raw) $log ---"
+            cat "$log"
+          done
+
       - name: Replay logs through problem matchers
         if: steps.download-logs.outcome == 'success'
         run: |

From f82510bdb67fa7b0c49a21cb55f45b96dfd816e2 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 20 Jul 2026 00:38:06 -0400
Subject: [PATCH 43/45] trigger build


From 1ae353ee496c2cd4b7d17613992190427bc7d59a Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 20 Jul 2026 09:12:43 -0400
Subject: [PATCH 44/45] pull proper github merge ref

---
 .github/workflows/build_and_test.yml | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index e34df6d7271cf..be737e7a1bf30 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1168,11 +1168,28 @@ jobs:
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
     - name: Sync the current branch with the latest in Apache Spark
       if: github.repository != 'apache/spark'
+      env:
+        # needed so that gh api is not rate limited
+        GH_TOKEN: ${{ github.token }}
       run: |
         echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
-        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
-        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
-        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+        BRANCH="${GITHUB_REF#refs/heads/}"
+        PR_NUMBER=$(gh pr list \
+          --repo apache/spark \
+          --head "${{ github.repository_owner }}:${BRANCH}" \
+          --state open \
+          --json number \
+          --jq '.[0].number' 2>/dev/null || true)
+        if [ -n "$PR_NUMBER" ]; then
+          echo "Found PR #${PR_NUMBER}; using exact merge ref refs/pull/${PR_NUMBER}/merge"
+          git fetch https://github.com/apache/spark.git "refs/pull/${PR_NUMBER}/merge"
+          git checkout FETCH_HEAD
+        else
+          echo "No open PR found against apache/spark; falling back to squash merge"
+          git fetch https://github.com/$GITHUB_REPOSITORY.git "${BRANCH}"
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+        fi
     # Cache local repositories. Note that GitHub Actions cache has a 10G limit.
     - name: Cache SBT and Maven
       uses: actions/cache@v5

From f080ff24dc07936fe6731ea95d2fae5fb403d067 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 20 Jul 2026 09:31:46 -0400
Subject: [PATCH 45/45] try curl instead of gh

---
 .github/workflows/build_and_test.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index be737e7a1bf30..78932fb7059a4 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1174,12 +1174,12 @@ jobs:
       run: |
         echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
         BRANCH="${GITHUB_REF#refs/heads/}"
-        PR_NUMBER=$(gh pr list \
-          --repo apache/spark \
-          --head "${{ github.repository_owner }}:${BRANCH}" \
-          --state open \
-          --json number \
-          --jq '.[0].number' 2>/dev/null || true)
+        PR_NUMBER=$(curl -s \
+          -H "Authorization: Bearer ${GH_TOKEN}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/apache/spark/pulls?state=open&head=${{ github.repository_owner }}:${BRANCH}&per_page=1" \
+          | python3 -c "import json,sys; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" \
+          2>/dev/null || true)
         if [ -n "$PR_NUMBER" ]; then
           echo "Found PR #${PR_NUMBER}; using exact merge ref refs/pull/${PR_NUMBER}/merge"
           git fetch https://github.com/apache/spark.git "refs/pull/${PR_NUMBER}/merge"