From 71ba533696790c0d4ff7f9042abd8a6ee8a3626f Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 13 Sep 2026 22:59:03 +0800 Subject: [PATCH] chore: drop support for JDK 11 Require JDK 17 or later to build Comet. The Spark 3.4 and 3.5 profiles now target Java 17, the jdk11 and jdk17 profiles are removed, and a Maven enforcer rule rejects older JDKs. CI jobs that ran Spark 3.4 on JDK 11 move to JDK 17, and the user and contributor docs are updated. Closes #4844. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 6 +-- .github/workflows/pr_build_linux.yml | 12 ++--- .../benchmarking_micro_ec2.md | 15 ++---- docs/source/contributor-guide/development.md | 6 +-- .../contributor-guide/iceberg-spark-tests.md | 4 +- .../latest/compatibility/spark-versions.md | 9 +--- docs/source/user-guide/latest/installation.md | 10 ++-- pom.xml | 48 +++++++------------ spark/pom.xml | 4 +- 9 files changed, 44 insertions(+), 70 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd3a0aa99cc..9904c2760b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: uses: actions/setup-java@v6 with: distribution: temurin - java-version: 11 + java-version: 17 - name: Apache RAT license check run: ./mvnw -B -N apache-rat:check @@ -261,7 +261,7 @@ jobs: with: spark-short: '3.4' spark-full: '3.4.3' - java: 11 + java: 17 spark_3_5: name: Spark SQL Tests (Spark 3.5) @@ -317,7 +317,7 @@ jobs: iceberg-full: '1.8.1' spark-short: '3.4' spark-full: '3.4.3' - java: 11 + java: 17 iceberg_1_9: name: Iceberg Spark SQL Tests (Iceberg 1.9) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 6814d75769b..bc5d633925e 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -98,12 +98,12 @@ jobs: container: image: amd64/rust env: - JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} + JAVA_TOOL_OPTIONS: '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' strategy: matrix: profile: - - name: "Spark 3.4, JDK 11, Scala 2.12" - java_version: "11" + - name: "Spark 3.4, JDK 17, Scala 2.12" + java_version: "17" maven_opts: "-Pspark-3.4 -Pscala-2.12" - name: "Spark 3.5, JDK 17, Scala 2.12" java_version: "17" @@ -347,8 +347,8 @@ jobs: # the goal with these profiles is to get coverage of all Java, Scala, and Spark # versions without testing all possible combinations, which would be overkill profile: - - name: "Spark 3.4, JDK 11, Scala 2.12" - java_version: "11" + - name: "Spark 3.4, JDK 17, Scala 2.12" + java_version: "17" maven_opts: "-Pspark-3.4 -Pscala-2.12" - name: "Spark 3.5, JDK 17, Scala 2.13" @@ -514,7 +514,7 @@ jobs: container: image: amd64/rust env: - JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} + JAVA_TOOL_OPTIONS: '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' steps: - uses: actions/checkout@v7 diff --git a/docs/source/contributor-guide/benchmarking_micro_ec2.md b/docs/source/contributor-guide/benchmarking_micro_ec2.md index 2d68f36b969..a848b2d849d 100644 --- a/docs/source/contributor-guide/benchmarking_micro_ec2.md +++ b/docs/source/contributor-guide/benchmarking_micro_ec2.md @@ -163,9 +163,8 @@ java -version javac -version ``` -JDK 17 or later is required. Comet targets Java 17 bytecode whenever the JDK in use is 17 or -newer, through the `jdk17` profile in the pom, so no profile needs to be passed by hand. `run.py` -detects `JAVA_HOME` when it is not set, preferring the newest JDK under `/usr/lib/jvm`, and refuses +JDK 17 or later is required. Comet targets Java 17 bytecode, so no JDK profile needs to be passed +by hand. `run.py` detects `JAVA_HOME` when it is not set, preferring the newest JDK under `/usr/lib/jvm`, and refuses to run with anything older than 17. ### Maven @@ -352,21 +351,15 @@ sudo dnf install -y gcc gcc-c++ make **`JAVA_HOME could not be determined`.** Install a JDK and export `JAVA_HOME`, or pass a checkout that has one configured. -**`Class java.lang.Record not found - continuing with a stub`.** The Scala compiler is targeting the -Java 11 API, which has no `Record`, while the Spark 4.x sources need Java 17. This happens on a JDK -older than 17, where the `jdk17` profile does not activate and `java.version` stays at its default -of 11. Use JDK 17 or later: +**`Comet requires JDK 17 or later to build`.** The Maven enforcer plugin rejects JDKs older than 17. +Point `JAVA_HOME` at JDK 17 or later and rerun the build: ```shell export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto export PATH=$JAVA_HOME/bin:$PATH -./mvnw clean make release ``` -Run `./mvnw clean` first: classes compiled against the wrong API stay in `target/` and break the -next build. - **`cargo: command not found` after `setup`.** `rustup` installs into `~/.cargo/bin`. Run `source "$HOME/.cargo/env"`, or start a new shell. diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index b04a3fdd424..104e7ceb346 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -407,10 +407,8 @@ from a clean IntelliJ configuration: PROFILES="-Pspark-4.0" make release ``` - The `spark-4.0` profile sets Scala 2.13 and Java 17 properties. If you need to be explicit, use - `PROFILES="-Pspark-4.0 -Pscala-2.13 -Pjdk17" make release`. - - The Maven profile is named `jdk17` in this project. + The `spark-4.0` profile sets the Scala 2.13 properties, and every profile targets Java 17. If you + need to be explicit, use `PROFILES="-Pspark-4.0 -Pscala-2.13" make release`. If the native build previously used a different JDK, clear Cargo's cached JNI link path before rebuilding: diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index 4aef43b102c..65230eb497c 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -99,8 +99,8 @@ diff must be generated against its own tag. ## Running Tests in CI The `iceberg_spark_test_.yml` workflows apply these diffs and run the three Gradle targets above -against each Iceberg version. Iceberg 1.8.1 runs against Spark 3.4.3 with Java 11; Iceberg 1.9.1 and 1.10.0 -run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.3 with Java 17. Iceberg 1.11 +against each Iceberg version, all with Java 17. Iceberg 1.8.1 runs against Spark 3.4.3; Iceberg 1.9.1 and 1.10.0 +run against Spark 3.5.9; Iceberg 1.11.0 runs against Spark 4.1.3. Iceberg 1.11 (the only version testing Spark 4.1) runs on every pull request and in the merge queue; the older versions (1.8, 1.9, 1.10) run only in the merge queue, or on a pull request labeled `run-iceberg-tests`. All caller workflows delegate to `iceberg_spark_test_reusable.yml`, which holds the build and test job logic. See diff --git a/docs/source/user-guide/latest/compatibility/spark-versions.md b/docs/source/user-guide/latest/compatibility/spark-versions.md index 090a1df85e8..292561c1ee9 100644 --- a/docs/source/user-guide/latest/compatibility/spark-versions.md +++ b/docs/source/user-guide/latest/compatibility/spark-versions.md @@ -28,7 +28,7 @@ of the versioning policy. ## Spark 3.4 -Spark 3.4.3 is supported with Java 11/17 and Scala 2.12/2.13. +Spark 3.4.3 is supported with Java 17 and Scala 2.12/2.13. ```{warning} Spark 3.4 support is deprecated as of the 1.0.0 release and will be removed in the 1.1.0 release. @@ -47,12 +47,7 @@ Spark 3.4 support is deprecated as of the 1.0.0 release and will be removed in t ## Spark 3.5 -Spark 3.5.9 is supported with Java 11/17 and Scala 2.12/2.13. - -```{warning} -JDK 11 support is deprecated as of the 1.0.0 release and will be removed in the 1.1.0 release. -We recommend moving to JDK 17 or later. -``` +Spark 3.5.9 is supported with Java 17 and Scala 2.12/2.13. ### Known Limitations diff --git a/docs/source/user-guide/latest/installation.md b/docs/source/user-guide/latest/installation.md index 7fc4a81d897..a8d05955f04 100644 --- a/docs/source/user-guide/latest/installation.md +++ b/docs/source/user-guide/latest/installation.md @@ -45,15 +45,17 @@ in the [Compatibility Guide] for more information, such as known limitations per We recommend only using Comet with Spark versions where we currently have both Comet and Spark tests enabled in CI. Other versions may work well enough for development and evaluation purposes. +Comet requires JDK 17 or later. JDK 11 is no longer supported as of the 1.1.0 release. + ```{warning} -JDK 11 and Spark 3.4 support are deprecated as of the 1.0.0 release and will be removed in the 1.1.0 release. -We recommend moving to JDK 17 or later and Spark 3.5 or later. +Spark 3.4 support is deprecated as of the 1.0.0 release and will be removed in the 1.1.0 release. +We recommend moving to Spark 3.5 or later. ``` | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI | | ------------- | ------------ | ------------- | ----------------- | --------------------- | -| 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes | -| 3.5.9 | 11/17 | 2.12/2.13 | Yes | Yes | +| 3.4.3 | 17 | 2.12/2.13 | Yes | Yes | +| 3.5.9 | 17 | 2.12/2.13 | Yes | Yes | | 4.0.4 | 17/21 | 2.13 | Yes | Yes | | 4.1.3 | 17/21 | 2.13 | Yes | Yes | diff --git a/pom.xml b/pom.xml index f4b2be220ec..6766979652d 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ under the License. UTF-8 UTF-8 - 11 + 17 - [17,) - - - 17 - ${java.version} - ${java.version} - - - semanticdb @@ -1210,6 +1182,20 @@ under the License. maven-enforcer-plugin ${maven-enforcer-plugin.version} + + require-java-17 + + enforce + + + + + [17,) + Comet requires JDK 17 or later to build. + + + + no-duplicate-declared-dependencies diff --git a/spark/pom.xml b/spark/pom.xml index 8dc632d5b92..9a92f0c3b00 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -267,7 +267,7 @@ under the License. 1.5.2 test - + org.eclipse.jetty jetty-server @@ -299,7 +299,7 @@ under the License. 1.8.1 test - + org.eclipse.jetty jetty-server