[DOCS][INFRA] Use GitHub Actions problem matcher to simplify Unidoc log filtering

.github/problem-matchers/javadoc.json

@@ -0,0 +1,17 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "javadoc-doclint",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "\\]\\s+\\S*/spark/spark/((?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+error:\\s+(.+)",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "message": 4
+        }
+      ]
+    }
+  ]
+}

.github/workflows/build_and_test.yml

@@ -1168,11 +1168,28 @@ jobs:
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
     - name: Sync the current branch with the latest in Apache Spark
       if: github.repository != 'apache/spark'
+      env:
+        # needed so that gh api is not rate limited
+        GH_TOKEN: ${{ github.token }}
       run: |
         echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
-        git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
-        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
-        git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+        BRANCH="${GITHUB_REF#refs/heads/}"
+        PR_NUMBER=$(curl -s \
+          -H "Authorization: Bearer ${GH_TOKEN}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/apache/spark/pulls?state=open&head=${{ github.repository_owner }}:${BRANCH}&per_page=1" \
+          | python3 -c "import json,sys; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" \
+          2>/dev/null || true)
+        if [ -n "$PR_NUMBER" ]; then
+          echo "Found PR #${PR_NUMBER}; using exact merge ref refs/pull/${PR_NUMBER}/merge"
+          git fetch https://github.com/apache/spark.git "refs/pull/${PR_NUMBER}/merge"
+          git checkout FETCH_HEAD
+        else
+          echo "No open PR found against apache/spark; falling back to squash merge"
+          git fetch https://github.com/$GITHUB_REPOSITORY.git "${BRANCH}"
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+        fi
     # Cache local repositories. Note that GitHub Actions cache has a 10G limit.
     - name: Cache SBT and Maven
       uses: actions/cache@v5
@@ -1217,7 +1234,20 @@ jobs:
         gem install bundler -v 2.4.22
         cd docs
         bundle install --retry=100
+    - run: echo "$GITHUB_WORKSPACE"
+    - name: Fix remote URL for problem matcher (fork runs only)
+      # The checkout above always uses repository: apache/spark, so .git/config
+      # contains url = https://github.com/apache/spark. When running from a fork,
+      # GITHUB_REPOSITORY is e.g. nchammas/spark, but the runner verifies file
+      # annotations by looking for GITHUB_REPOSITORY in .git/config -- it won't
+      # find it and silently drops every file path from annotations. Re-pointing
+      # origin to the fork URL fixes the mismatch without changing the working tree.
+      if: github.repository != 'apache/spark'
+      run: git remote set-url origin https://github.com/${{ github.repository }}
+    - name: Register javadoc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/javadoc.json"
     - name: Run documentation build
+      shell: bash
       run: |
         # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages.
         ln -s "$(which python3.12)" "/usr/local/bin/python3"
@@ -1239,7 +1269,29 @@ jobs:
         echo "SKIP_RDOC: $SKIP_RDOC"
         echo "SKIP_SQLDOC: $SKIP_SQLDOC"
         cd docs
-        bundle exec jekyll build
+        set -o pipefail
+        bundle exec jekyll build 2>&1 \
+          | tee "$GITHUB_WORKSPACE/docs-build.log"
+    - name: Remove javadoc problem matcher
+      if: always()
+      run: echo "::remove-matcher owner=javadoc-doclint::"
+    - name: Upload build log
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: docs-build-log
+        path: ${{ github.workspace }}/docs-build.log
+        retention-days: 3
+    - name: Upload workspace root
+      if: always()
+      run: echo "$GITHUB_WORKSPACE" > "$GITHUB_WORKSPACE/workspace-root.txt"
+    - name: Upload workspace root artifact
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: docs-workspace-root
+        path: ${{ github.workspace }}/workspace-root.txt
+        retention-days: 3
     - name: Tar documentation
       if: github.repository != 'apache/spark'
       run: tar cjf site.tar.bz2 docs/_site

.github/workflows/report_annotations.yml

@@ -0,0 +1,138 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Relay workflow: runs on the upstream repo after a fork's workflow completes.
+# Downloads pre-extracted annotation commands and replays them so they appear
+# as inline annotations on the upstream PR's diff.
+#
+# This is necessary because annotations from fork workflows (on: push) don't
+# appear inline on the upstream PR. This lightweight relay runs in the
+# upstream repo's context with checks:write permission, so its annotations
+# do appear inline.
+#
+# To add annotations from a new workflow:
+# 1. In the source workflow, capture the build log with tee and upload it
+#    as an artifact matching the pattern "*-build-log".
+# 2. Upload $GITHUB_WORKSPACE as an artifact matching "*-workspace-root"
+#    so the relay can strip the runner's absolute path prefix from log entries.
+# 3. Add problem matcher JSON files to .github/problem-matchers/.
+# 4. Add the source workflow's name to the list below.
+
+name: Report annotations
+
+on:
+  workflow_run:
+    workflows:
+      # These must match the `name:` field of the top-level (caller) workflow
+      # exactly. build_and_test.yml is a reusable workflow (workflow_call only)
+      # so it never appears as a standalone run; the caller is what fires here.
+      - "Build"
+      # Add other callers here as needed, e.g.:
+      # - "Build and test using Maven"
+    types:
+      - completed
+
+jobs:
+  annotate:
+    name: Post annotations
+    if: >
+      !contains(fromJson('["skipped", "cancelled"]'), github.event.workflow_run.conclusion)
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      checks: write
+    steps:
+      - name: Checkout Spark
+        uses: actions/checkout@v6
+
+      - name: Download build logs
+        id: download-logs
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: "*-build-log"
+          merge-multiple: true
+
+      - name: Download workspace roots
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: "*-workspace-root"
+          merge-multiple: true
+
+      - name: Print workspace info
+        run: |
+          echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE"
+          echo "Downloaded files:"
+          ls -la
+          echo "workspace-root.txt contents:"
+          cat workspace-root.txt 2>/dev/null || echo "(not found)"
+
+      - name: Replay logs through problem matchers (raw, no path stripping)
+        if: steps.download-logs.outcome == 'success'
+        run: |
+          # Debug replay: emit lines exactly as-is so we can see what the
+          # problem matcher receives before any path rewriting is applied.
+          for matcher in .github/problem-matchers/*.json; do
+            echo "::add-matcher::$matcher"
+          done
+
+          for log in *.log; do
+            [ -f "$log" ] || continue
+            echo "--- Replaying (raw) $log ---"
+            cat "$log"
+          done
+
+      - name: Replay logs through problem matchers
+        if: steps.download-logs.outcome == 'success'
+        run: |
+          # Read the fork's workspace root and strip it from paths so GitHub
+          # can map annotations to repo-relative files in the PR diff.
+          # (Absolute paths vary by runner: /home/runner/work/..., /__w/..., etc.)
+          ws_root=""
+          if [ -f workspace-root.txt ]; then
+            ws_root=$(tr -d '\n' < workspace-root.txt)
+            echo "Stripping workspace root: $ws_root"
+          fi
+
+          # Register all available problem matchers.
+          for matcher in .github/problem-matchers/*.json; do
+            echo "::add-matcher::$matcher"
+          done
+
+          # Replay each build log with absolute paths stripped to repo-relative.
+          for log in *.log; do
+            [ -f "$log" ] || continue
+            echo "--- Replaying $log ---"
+            if [ -n "$ws_root" ]; then
+              sed "s|${ws_root}/||g" "$log"
+            else
+              cat "$log"
+            fi
+          done
+
+          # Clean up matchers.
+          for matcher in .github/problem-matchers/*.json; do
+            owner=$(python3 -c "import json; print(json.load(open('$matcher'))['problemMatcher'][0]['owner'])")
+            echo "::remove-matcher owner=$owner::"
+          done

core/src/main/scala/org/apache/spark/Partition.scala

@@ -23,6 +23,7 @@ package org.apache.spark
 trait Partition extends Serializable {
   /**
    * Get the partition's index within its parent RDD
+   * See also [[Partition.index]].
    */
   def index: Int
 

docs/_plugins/build_api_docs.rb

@@ -131,147 +131,7 @@ def build_spark_scala_and_java_docs_if_necessary
 
   command = "build/sbt -Pkinesis-asl unidoc"
   puts "Running '#{command}'..."
-
-  # Two filter passes on the unidoc output, plus an additive fatal-error summary:
-  #
-  # 1. Genjavadoc-stub diagnostic blocks (~28 `[error]` lines on stubs under
-  #    `target/java/`, plus 3-5 continuation lines each). Inert because
-  #    `--ignore-source-errors` is set; matched by message text so legitimate
-  #    doclint diagnostics on stub paths still pass through.
-  #
-  # 2. `-verbose` progress lines (~13K total): `Loading source file ...`,
-  #    `[parsing started/completed ...]`, `[loading /path/X.class]`,
-  #    `Generating .../X.html`. These are dominant in the log when `-verbose`
-  #    is set (which it is in `JavaUnidoc / unidoc / javacOptions` to surface
-  #    per-file `error: reference not found` diagnostics) but carry no signal
-  #    of their own. Suppressing them brings the visible log from ~17K to ~5K
-  #    lines on a typical run while leaving every diagnostic untouched.
-  #
-  # 3. Fatal-error summary (additive, drops no log lines). The filtered log is
-  #    still ~4K lines and most `error:` text in it is non-fatal source-loading
-  #    chatter, so the build-failing diagnostics are hard to spot. After the
-  #    pipe closes, we print a `Fatal javadoc errors (N): ...` block and emit
-  #    `::error file=,line=::` GitHub Actions annotations so they surface in the
-  #    PR check panel. Captured strictly within the Standard Doclet phase
-  #    bracketed by `Building tree for all the packages and classes...` and
-  #    `Building index for all classes...`, which is where doclint diagnostics
-  #    are emitted -- this matches what javadoc counts toward exit code 1.
-  #    Self-checked against javadoc's own `N errors` summary line; a mismatch
-  #    emits a `::warning::` so future phase-marker drift is visible.
-  ansi = /\e\[[0-9;]*[A-Za-z]/
-  stub_header = %r{
-    \[(?:error|warn)\]\s+
-    \S*?/target/java/\S+\.java:\d+(?::\d+)?:\s+
-    error:\s+
-    (?:cannot\s+find\s+symbol
-     |illegal\s+combination\s+of\s+modifiers
-     |non-static\s+type\s+variable\b
-     |.*?\s+is\s+not\s+public\s+in\s+\S+;\s+cannot\s+be\s+accessed\s+from\s+outside\s+package)
-  }x
-  stub_cont = %r{\A\s*\[(?:error|warn)\]\s+(?!/\S+\.java:\d+(?::\d+)?:\s)}
-  verbose_line = %r{
-    \[(?:error|warn)\]\s+
-    (?:Loading\s+source\s+file\s
-     |\[parsing\s+(?:started|completed)\s
-     |\[loading\s
-     |\[checking\s
-     |\[wrote\s
-     |Generating\s+\S+\.html
-    )
-  }x
-
-  # Doclint phase tracking for the trailing summary. Standard Doclet bookends the
-  # phase that produces build-failing diagnostics with these marker lines; any
-  # `error:` outside this window is source-loading noise that does not contribute
-  # to javadoc's exit code. The summary below captures only the fatal ones and
-  # re-emits them as GitHub Actions annotations so they surface in the PR check
-  # panel instead of being buried in a 4K-line log.
-  doclint_start   = %r{\bBuilding\s+tree\s+for\s+all\s+the\s+packages\s+and\s+classes\b}
-  doclint_end     = %r{\bBuilding\s+index\s+for\s+all\s+classes\b}
-  doclint_diag    = %r{\A\[warn\]\s+(?<path>\S+):(?<lineno>\d+)(?::\d+)?:\s+error:\s+(?<msg>.+?)\s*\z}
-  doclint_cont    = %r{\A\[warn\]\s(?!\S+:\d+(?::\d+)?:\s+error:)(?<content>.*?)\s*\z}
-  doclint_summary = %r{\A\[warn\]\s+(?<count>[\d,]+)\s+errors?\s*\z}
-
-  in_stub = false
-  in_doclint = false
-  fatal_diagnostics = []
-  pending_context_lines = 0  # snippet + caret lines that follow each diag header
-  reported_error_count = nil
-
-  IO.popen("#{command} 2>&1", 'r') do |pipe|
-    pipe.each_line do |line|
-      plain = line.gsub(ansi, '')
-
-      if plain =~ doclint_start
-        in_doclint = true
-      elsif in_doclint && plain =~ doclint_end
-        in_doclint = false
-        pending_context_lines = 0
-      end
-
-      if in_doclint && (m = plain.match(doclint_diag))
-        fatal_diagnostics << {
-          path: m[:path], line: m[:lineno], msg: m[:msg], context: []
-        }
-        pending_context_lines = 2
-      elsif in_doclint && pending_context_lines > 0 &&
-            (m = plain.match(doclint_cont)) && !fatal_diagnostics.empty?
-        fatal_diagnostics.last[:context] << m[:content]
-        pending_context_lines -= 1
-      end
-
-      if reported_error_count.nil? && (m = plain.match(doclint_summary))
-        reported_error_count = m[:count].delete(',').to_i
-      end
-
-      if plain =~ verbose_line
-        in_stub = false
-        # suppress -verbose progress line
-      elsif plain =~ stub_header
-        in_stub = true
-      elsif in_stub && plain =~ stub_cont
-        # continuation of a stub block; suppress
-      else
-        in_stub = false
-        $stdout.write(line)
-        $stdout.flush
-      end
-    end
-  end
-
-  unless fatal_diagnostics.empty?
-    bar = "=" * 72
-    puts ""
-    puts bar
-    puts "Fatal javadoc errors (#{fatal_diagnostics.size}):"
-    puts bar
-    fatal_diagnostics.each_with_index do |d, i|
-      puts "  #{i + 1}. #{d[:path]}:#{d[:line]}: #{d[:msg]}"
-      d[:context].each { |c| puts "       #{c}" }
-    end
-    puts bar
-    puts ""
-
-    # GitHub Actions inline annotations. `%`, `\r`, `\n` require URL-style
-    # escaping per the workflow command spec; newlines render as multiple
-    # lines inside the annotation, so the source snippet and caret display
-    # under the error message in the PR check panel.
-    project_root = SPARK_PROJECT_ROOT + '/'
-    fatal_diagnostics.each do |d|
-      rel = d[:path].start_with?(project_root) ? d[:path][project_root.length..] : d[:path]
-      full = ([d[:msg]] + d[:context]).join("\n")
-      enc = full.gsub(/[%\r\n]/, '%' => '%25', "\r" => '%0D', "\n" => '%0A')
-      puts "::error file=#{rel},line=#{d[:line]},title=javadoc::#{enc}"
-    end
-  end
-
-  if reported_error_count && reported_error_count != fatal_diagnostics.size
-    puts "::warning::Javadoc reported #{reported_error_count} errors but " \
-         "build_api_docs.rb captured #{fatal_diagnostics.size}. The doclint " \
-         "phase markers may have shifted; please update build_api_docs.rb."
-  end
-
-  raise("Unidoc generation failed") unless $?.success?
+  system(command) || raise("Unidoc generation failed")
 end
 
 def build_scala_and_java_docs