Build: Test Parquet 1.18.0

build.gradle

@@ -117,6 +117,9 @@ allprojects {
   repositories {
     mavenCentral()
     mavenLocal()
+    maven {
+      url = uri("https://repository.apache.org/content/groups/staging/")
+    }
   }
 }
 

data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java

@@ -200,7 +200,9 @@ public void createOrcInputFile() throws IOException {
         record.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0
         record.setField(
             "_no_stats_parquet",
-            TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats
+            (i % 10 == 0)
+                ? null
+                : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats
         // in Parquet, but will produce stats for ORC
         record.setField("_required", "req"); // required, always non-null
         record.setField("_all_nulls", null); // never non-null
@@ -240,7 +242,9 @@ private void createParquetInputFile() throws IOException {
       builder.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0
       builder.setField(
           "_no_stats_parquet",
-          TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats
+          (i % 10 == 0)
+              ? null
+              : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats
       // in Parquet
       builder.setField("_required", "req"); // required, always non-null
       builder.setField("_all_nulls", null); // never non-null
@@ -341,7 +345,7 @@ public void testFloatWithNan() {
     shouldRead = shouldRead(lessThan("some_nans", 3.0));
     assertThat(shouldRead).isTrue();
 
-    shouldRead = shouldRead(lessThanOrEqual("some_nans", 1.0));
+    shouldRead = shouldRead(lessThanOrEqual("some_nans", 2.0));
     assertThat(shouldRead).isTrue();
 
     shouldRead = shouldRead(equal("some_nans", 2.0));
@@ -361,7 +365,7 @@ public void testDoubleWithNan() {
     shouldRead = shouldRead(lessThan("some_double_nans", 3.0));
     assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue();
 
-    shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 1.0));
+    shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 2.0));
     assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue();
 
     shouldRead = shouldRead(equal("some_double_nans", 2.0));

flink/v1.20/flink-runtime/runtime-deps.txt

@@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
 org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20

flink/v2.0/flink-runtime/runtime-deps.txt

@@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
 org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20

flink/v2.1/flink-runtime/runtime-deps.txt

@@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1
 org.jspecify:jspecify:1.0
 org.locationtech.jts:jts-core:1.20

gradle/libs.versions.toml

@@ -81,7 +81,7 @@ nessie = "0.108.3"
 netty-buffer = "4.2.16.Final"
 object-client-bundle = "3.3.2"
 orc = "1.9.9"
-parquet = "1.17.1"
+parquet = "1.18.0"
 roaringbitmap = "1.6.18"
 scala-collection-compat = "2.14.0"
 slf4j = "2.0.18"

kafka-connect/kafka-connect-runtime/runtime-deps.txt

@@ -171,14 +171,14 @@ org.apache.httpcomponents:httpclient:4.5
 org.apache.httpcomponents:httpcore:4.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.bouncycastle:bcprov-jdk18on:1.85
 org.checkerframework:checker-compat-qual:2.5
 org.checkerframework:checker-qual:3.49

parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java

@@ -451,12 +451,21 @@ public void testVariantFloatNaN() throws IOException {
     assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L));
     assertThat(metrics.nanValueCounts()).isEqualTo(Map.of());
 
-    assertThat(metrics.lowerBounds())
-        .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L)));
-    assertThat(metrics.upperBounds())
-        .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L)));
+    // NaN is ignored when computing the shredded value bounds, so both bounds are the float value
+    assertThat(metrics.lowerBounds().get(1))
+        .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L));
+    assertThat(metrics.lowerBounds().get(2))
+        .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+        .isEqualTo(Variants.of(1.0F));
+    assertThat(metrics.upperBounds().get(1))
+        .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L));
+    assertThat(metrics.upperBounds().get(2))
+        .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+        .isEqualTo(Variants.of(1.0F));
 
-    assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get()));
+    assertThat(metrics)
+        .extracting("originalTypes")
+        .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get()));
   }
 
   @Test
@@ -473,12 +482,21 @@ public void testVariantDoubleNaN() throws IOException {
     assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L));
     assertThat(metrics.nanValueCounts()).isEqualTo(Map.of());
 
-    assertThat(metrics.lowerBounds())
-        .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L)));
-    assertThat(metrics.upperBounds())
-        .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L)));
+    // NaN is ignored when computing the shredded value bounds, so both bounds are the double value
+    assertThat(metrics.lowerBounds().get(1))
+        .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L));
+    assertThat(metrics.lowerBounds().get(2))
+        .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+        .isEqualTo(Variants.of(1.0D));
+    assertThat(metrics.upperBounds().get(1))
+        .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L));
+    assertThat(metrics.upperBounds().get(2))
+        .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+        .isEqualTo(Variants.of(1.0D));
 
-    assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get()));
+    assertThat(metrics)
+        .extracting("originalTypes")
+        .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get()));
   }
 
   @Test

spark/v3.5/spark-runtime/runtime-deps.txt

@@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.collections:eclipse-collections-api:11.1
 org.eclipse.collections:eclipse-collections:11.1
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1

spark/v4.0/spark-runtime/runtime-deps.txt

@@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.collections:eclipse-collections-api:11.1
 org.eclipse.collections:eclipse-collections:11.1
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1

spark/v4.1/spark-runtime/runtime-deps.txt

@@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4
 org.apache.httpcomponents.core5:httpcore5:5.4
 org.apache.orc:orc-core:1.9
 org.apache.orc:orc-shims:1.9
-org.apache.parquet:parquet-avro:1.17
-org.apache.parquet:parquet-column:1.17
-org.apache.parquet:parquet-common:1.17
-org.apache.parquet:parquet-encoding:1.17
-org.apache.parquet:parquet-format-structures:1.17
-org.apache.parquet:parquet-hadoop:1.17
-org.apache.parquet:parquet-jackson:1.17
-org.apache.parquet:parquet-variant:1.17
+org.apache.parquet:parquet-avro:1.18
+org.apache.parquet:parquet-column:1.18
+org.apache.parquet:parquet-common:1.18
+org.apache.parquet:parquet-encoding:1.18
+org.apache.parquet:parquet-format-structures:1.18
+org.apache.parquet:parquet-hadoop:1.18
+org.apache.parquet:parquet-jackson:1.18
+org.apache.parquet:parquet-variant:1.18
 org.eclipse.collections:eclipse-collections-api:11.1
 org.eclipse.collections:eclipse-collections:11.1
 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1