diff --git a/.github/problem-matchers/javadoc.json b/.github/problem-matchers/javadoc.json new file mode 100644 index 0000000000000..94a776477ec11 --- /dev/null +++ b/.github/problem-matchers/javadoc.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "javadoc-doclint", + "severity": "error", + "pattern": [ + { + "regexp": "\\]\\s+\\S*/spark/spark/((?!(?:[^:]*/)?target/)[^:\\s]+):(\\d+):(\\d+):\\s+error:\\s+(.+)", + "file": 1, + "line": 2, + "column": 3, + "message": 4 + } + ] + } + ] +} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0bcda10e3b6c3..78932fb7059a4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1168,11 +1168,28 @@ jobs: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Sync the current branch with the latest in Apache Spark if: github.repository != 'apache/spark' + env: + # needed so that gh api is not rate limited + GH_TOKEN: ${{ github.token }} run: | echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV - git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD - git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + BRANCH="${GITHUB_REF#refs/heads/}" + PR_NUMBER=$(curl -s \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/apache/spark/pulls?state=open&head=${{ github.repository_owner }}:${BRANCH}&per_page=1" \ + | python3 -c "import json,sys; prs=json.load(sys.stdin); print(prs[0]['number'] if prs else '')" \ + 2>/dev/null || true) + if [ -n "$PR_NUMBER" ]; then + echo "Found PR #${PR_NUMBER}; using exact merge ref refs/pull/${PR_NUMBER}/merge" + git fetch https://github.com/apache/spark.git "refs/pull/${PR_NUMBER}/merge" + git checkout FETCH_HEAD + else + echo "No open PR found against apache/spark; falling back to squash merge" + git fetch https://github.com/$GITHUB_REPOSITORY.git "${BRANCH}" + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + fi # Cache local repositories. Note that GitHub Actions cache has a 10G limit. - name: Cache SBT and Maven uses: actions/cache@v5 @@ -1217,7 +1234,20 @@ jobs: gem install bundler -v 2.4.22 cd docs bundle install --retry=100 + - run: echo "$GITHUB_WORKSPACE" + - name: Fix remote URL for problem matcher (fork runs only) + # The checkout above always uses repository: apache/spark, so .git/config + # contains url = https://github.com/apache/spark. When running from a fork, + # GITHUB_REPOSITORY is e.g. nchammas/spark, but the runner verifies file + # annotations by looking for GITHUB_REPOSITORY in .git/config -- it won't + # find it and silently drops every file path from annotations. Re-pointing + # origin to the fork URL fixes the mismatch without changing the working tree. + if: github.repository != 'apache/spark' + run: git remote set-url origin https://github.com/${{ github.repository }} + - name: Register javadoc problem matcher + run: echo "::add-matcher::.github/problem-matchers/javadoc.json" - name: Run documentation build + shell: bash run: | # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages. ln -s "$(which python3.12)" "/usr/local/bin/python3" @@ -1239,7 +1269,29 @@ jobs: echo "SKIP_RDOC: $SKIP_RDOC" echo "SKIP_SQLDOC: $SKIP_SQLDOC" cd docs - bundle exec jekyll build + set -o pipefail + bundle exec jekyll build 2>&1 \ + | tee "$GITHUB_WORKSPACE/docs-build.log" + - name: Remove javadoc problem matcher + if: always() + run: echo "::remove-matcher owner=javadoc-doclint::" + - name: Upload build log + if: always() + uses: actions/upload-artifact@v7 + with: + name: docs-build-log + path: ${{ github.workspace }}/docs-build.log + retention-days: 3 + - name: Upload workspace root + if: always() + run: echo "$GITHUB_WORKSPACE" > "$GITHUB_WORKSPACE/workspace-root.txt" + - name: Upload workspace root artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: docs-workspace-root + path: ${{ github.workspace }}/workspace-root.txt + retention-days: 3 - name: Tar documentation if: github.repository != 'apache/spark' run: tar cjf site.tar.bz2 docs/_site diff --git a/.github/workflows/report_annotations.yml b/.github/workflows/report_annotations.yml new file mode 100644 index 0000000000000..baea64d4e4c92 --- /dev/null +++ b/.github/workflows/report_annotations.yml @@ -0,0 +1,138 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Relay workflow: runs on the upstream repo after a fork's workflow completes. +# Downloads pre-extracted annotation commands and replays them so they appear +# as inline annotations on the upstream PR's diff. +# +# This is necessary because annotations from fork workflows (on: push) don't +# appear inline on the upstream PR. This lightweight relay runs in the +# upstream repo's context with checks:write permission, so its annotations +# do appear inline. +# +# To add annotations from a new workflow: +# 1. In the source workflow, capture the build log with tee and upload it +# as an artifact matching the pattern "*-build-log". +# 2. Upload $GITHUB_WORKSPACE as an artifact matching "*-workspace-root" +# so the relay can strip the runner's absolute path prefix from log entries. +# 3. Add problem matcher JSON files to .github/problem-matchers/. +# 4. Add the source workflow's name to the list below. + +name: Report annotations + +on: + workflow_run: + workflows: + # These must match the `name:` field of the top-level (caller) workflow + # exactly. build_and_test.yml is a reusable workflow (workflow_call only) + # so it never appears as a standalone run; the caller is what fires here. + - "Build" + # Add other callers here as needed, e.g.: + # - "Build and test using Maven" + types: + - completed + +jobs: + annotate: + name: Post annotations + if: > + !contains(fromJson('["skipped", "cancelled"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + permissions: + actions: read + checks: write + steps: + - name: Checkout Spark + uses: actions/checkout@v6 + + - name: Download build logs + id: download-logs + uses: actions/download-artifact@v8 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + pattern: "*-build-log" + merge-multiple: true + + - name: Download workspace roots + uses: actions/download-artifact@v8 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + pattern: "*-workspace-root" + merge-multiple: true + + - name: Print workspace info + run: | + echo "GITHUB_WORKSPACE: $GITHUB_WORKSPACE" + echo "Downloaded files:" + ls -la + echo "workspace-root.txt contents:" + cat workspace-root.txt 2>/dev/null || echo "(not found)" + + - name: Replay logs through problem matchers (raw, no path stripping) + if: steps.download-logs.outcome == 'success' + run: | + # Debug replay: emit lines exactly as-is so we can see what the + # problem matcher receives before any path rewriting is applied. + for matcher in .github/problem-matchers/*.json; do + echo "::add-matcher::$matcher" + done + + for log in *.log; do + [ -f "$log" ] || continue + echo "--- Replaying (raw) $log ---" + cat "$log" + done + + - name: Replay logs through problem matchers + if: steps.download-logs.outcome == 'success' + run: | + # Read the fork's workspace root and strip it from paths so GitHub + # can map annotations to repo-relative files in the PR diff. + # (Absolute paths vary by runner: /home/runner/work/..., /__w/..., etc.) + ws_root="" + if [ -f workspace-root.txt ]; then + ws_root=$(tr -d '\n' < workspace-root.txt) + echo "Stripping workspace root: $ws_root" + fi + + # Register all available problem matchers. + for matcher in .github/problem-matchers/*.json; do + echo "::add-matcher::$matcher" + done + + # Replay each build log with absolute paths stripped to repo-relative. + for log in *.log; do + [ -f "$log" ] || continue + echo "--- Replaying $log ---" + if [ -n "$ws_root" ]; then + sed "s|${ws_root}/||g" "$log" + else + cat "$log" + fi + done + + # Clean up matchers. + for matcher in .github/problem-matchers/*.json; do + owner=$(python3 -c "import json; print(json.load(open('$matcher'))['problemMatcher'][0]['owner'])") + echo "::remove-matcher owner=$owner::" + done diff --git a/core/src/main/scala/org/apache/spark/Partition.scala b/core/src/main/scala/org/apache/spark/Partition.scala index e10660793d162..add650cf99123 100644 --- a/core/src/main/scala/org/apache/spark/Partition.scala +++ b/core/src/main/scala/org/apache/spark/Partition.scala @@ -23,6 +23,7 @@ package org.apache.spark trait Partition extends Serializable { /** * Get the partition's index within its parent RDD + * See also [[Partition.index]]. */ def index: Int diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb index 429cef5aa026c..d48f7f0c1e700 100644 --- a/docs/_plugins/build_api_docs.rb +++ b/docs/_plugins/build_api_docs.rb @@ -131,147 +131,7 @@ def build_spark_scala_and_java_docs_if_necessary command = "build/sbt -Pkinesis-asl unidoc" puts "Running '#{command}'..." - - # Two filter passes on the unidoc output, plus an additive fatal-error summary: - # - # 1. Genjavadoc-stub diagnostic blocks (~28 `[error]` lines on stubs under - # `target/java/`, plus 3-5 continuation lines each). Inert because - # `--ignore-source-errors` is set; matched by message text so legitimate - # doclint diagnostics on stub paths still pass through. - # - # 2. `-verbose` progress lines (~13K total): `Loading source file ...`, - # `[parsing started/completed ...]`, `[loading /path/X.class]`, - # `Generating .../X.html`. These are dominant in the log when `-verbose` - # is set (which it is in `JavaUnidoc / unidoc / javacOptions` to surface - # per-file `error: reference not found` diagnostics) but carry no signal - # of their own. Suppressing them brings the visible log from ~17K to ~5K - # lines on a typical run while leaving every diagnostic untouched. - # - # 3. Fatal-error summary (additive, drops no log lines). The filtered log is - # still ~4K lines and most `error:` text in it is non-fatal source-loading - # chatter, so the build-failing diagnostics are hard to spot. After the - # pipe closes, we print a `Fatal javadoc errors (N): ...` block and emit - # `::error file=,line=::` GitHub Actions annotations so they surface in the - # PR check panel. Captured strictly within the Standard Doclet phase - # bracketed by `Building tree for all the packages and classes...` and - # `Building index for all classes...`, which is where doclint diagnostics - # are emitted -- this matches what javadoc counts toward exit code 1. - # Self-checked against javadoc's own `N errors` summary line; a mismatch - # emits a `::warning::` so future phase-marker drift is visible. - ansi = /\e\[[0-9;]*[A-Za-z]/ - stub_header = %r{ - \[(?:error|warn)\]\s+ - \S*?/target/java/\S+\.java:\d+(?::\d+)?:\s+ - error:\s+ - (?:cannot\s+find\s+symbol - |illegal\s+combination\s+of\s+modifiers - |non-static\s+type\s+variable\b - |.*?\s+is\s+not\s+public\s+in\s+\S+;\s+cannot\s+be\s+accessed\s+from\s+outside\s+package) - }x - stub_cont = %r{\A\s*\[(?:error|warn)\]\s+(?!/\S+\.java:\d+(?::\d+)?:\s)} - verbose_line = %r{ - \[(?:error|warn)\]\s+ - (?:Loading\s+source\s+file\s - |\[parsing\s+(?:started|completed)\s - |\[loading\s - |\[checking\s - |\[wrote\s - |Generating\s+\S+\.html - ) - }x - - # Doclint phase tracking for the trailing summary. Standard Doclet bookends the - # phase that produces build-failing diagnostics with these marker lines; any - # `error:` outside this window is source-loading noise that does not contribute - # to javadoc's exit code. The summary below captures only the fatal ones and - # re-emits them as GitHub Actions annotations so they surface in the PR check - # panel instead of being buried in a 4K-line log. - doclint_start = %r{\bBuilding\s+tree\s+for\s+all\s+the\s+packages\s+and\s+classes\b} - doclint_end = %r{\bBuilding\s+index\s+for\s+all\s+classes\b} - doclint_diag = %r{\A\[warn\]\s+(?\S+):(?\d+)(?::\d+)?:\s+error:\s+(?.+?)\s*\z} - doclint_cont = %r{\A\[warn\]\s(?!\S+:\d+(?::\d+)?:\s+error:)(?.*?)\s*\z} - doclint_summary = %r{\A\[warn\]\s+(?[\d,]+)\s+errors?\s*\z} - - in_stub = false - in_doclint = false - fatal_diagnostics = [] - pending_context_lines = 0 # snippet + caret lines that follow each diag header - reported_error_count = nil - - IO.popen("#{command} 2>&1", 'r') do |pipe| - pipe.each_line do |line| - plain = line.gsub(ansi, '') - - if plain =~ doclint_start - in_doclint = true - elsif in_doclint && plain =~ doclint_end - in_doclint = false - pending_context_lines = 0 - end - - if in_doclint && (m = plain.match(doclint_diag)) - fatal_diagnostics << { - path: m[:path], line: m[:lineno], msg: m[:msg], context: [] - } - pending_context_lines = 2 - elsif in_doclint && pending_context_lines > 0 && - (m = plain.match(doclint_cont)) && !fatal_diagnostics.empty? - fatal_diagnostics.last[:context] << m[:content] - pending_context_lines -= 1 - end - - if reported_error_count.nil? && (m = plain.match(doclint_summary)) - reported_error_count = m[:count].delete(',').to_i - end - - if plain =~ verbose_line - in_stub = false - # suppress -verbose progress line - elsif plain =~ stub_header - in_stub = true - elsif in_stub && plain =~ stub_cont - # continuation of a stub block; suppress - else - in_stub = false - $stdout.write(line) - $stdout.flush - end - end - end - - unless fatal_diagnostics.empty? - bar = "=" * 72 - puts "" - puts bar - puts "Fatal javadoc errors (#{fatal_diagnostics.size}):" - puts bar - fatal_diagnostics.each_with_index do |d, i| - puts " #{i + 1}. #{d[:path]}:#{d[:line]}: #{d[:msg]}" - d[:context].each { |c| puts " #{c}" } - end - puts bar - puts "" - - # GitHub Actions inline annotations. `%`, `\r`, `\n` require URL-style - # escaping per the workflow command spec; newlines render as multiple - # lines inside the annotation, so the source snippet and caret display - # under the error message in the PR check panel. - project_root = SPARK_PROJECT_ROOT + '/' - fatal_diagnostics.each do |d| - rel = d[:path].start_with?(project_root) ? d[:path][project_root.length..] : d[:path] - full = ([d[:msg]] + d[:context]).join("\n") - enc = full.gsub(/[%\r\n]/, '%' => '%25', "\r" => '%0D', "\n" => '%0A') - puts "::error file=#{rel},line=#{d[:line]},title=javadoc::#{enc}" - end - end - - if reported_error_count && reported_error_count != fatal_diagnostics.size - puts "::warning::Javadoc reported #{reported_error_count} errors but " \ - "build_api_docs.rb captured #{fatal_diagnostics.size}. The doclint " \ - "phase markers may have shifted; please update build_api_docs.rb." - end - - raise("Unidoc generation failed") unless $?.success? + system(command) || raise("Unidoc generation failed") end def build_scala_and_java_docs diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 53412c20d8313..ae862339c2869 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1775,6 +1775,14 @@ object Unidoc { .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hadoop")))) }, + // This setting is separate from `-Xmaxerrs` below and just suppresses the _display_ of + // errors, not the errors themselves. The default is 100. + // We want _all_ errors to show because we have no way to separate real errors in our + // source from errors in generated Java files that are ignored. So if some errors are + // suppressed here, they will still fail the doc build even though they won't print + // an error message indicating the location of the problem. + (JavaUnidoc / unidoc / maxErrors) := 0, + (JavaUnidoc / unidoc / javacOptions) := { Seq( "-windowtitle", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " JavaDoc", @@ -1788,9 +1796,9 @@ object Unidoc { "-tag", "todo:X", "-tag", "groupname:X", "-tag", "inheritdoc", + "-quiet", "--ignore-source-errors", "-notree", "-Xmaxerrs", "0", - "-verbose", "-Xdoclint:all", "-Xdoclint:-missing" ) }, diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java index 6b3d518746dc3..5beb671eaa43c 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarMap.java @@ -22,6 +22,7 @@ /** * Map abstraction in {@link ColumnVector}. + * See also {@link org.apache.spark.deliberately.NoSuchClass}. */ public final class ColumnarMap extends MapData { private final ColumnarArray keys;