diff --git a/build.gradle b/build.gradle index 04146aa4a13a..04fc8d250c17 100644 --- a/build.gradle +++ b/build.gradle @@ -117,6 +117,9 @@ allprojects { repositories { mavenCentral() mavenLocal() + maven { + url = uri("https://repository.apache.org/content/groups/staging/") + } } } diff --git a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java index edb01d7e77ef..5572b6869708 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java +++ b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java @@ -200,7 +200,9 @@ public void createOrcInputFile() throws IOException { record.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0 record.setField( "_no_stats_parquet", - TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats + (i % 10 == 0) + ? null + : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats // in Parquet, but will produce stats for ORC record.setField("_required", "req"); // required, always non-null record.setField("_all_nulls", null); // never non-null @@ -240,7 +242,9 @@ private void createParquetInputFile() throws IOException { builder.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0 builder.setField( "_no_stats_parquet", - TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats + (i % 10 == 0) + ? null + : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats // in Parquet builder.setField("_required", "req"); // required, always non-null builder.setField("_all_nulls", null); // never non-null @@ -341,7 +345,7 @@ public void testFloatWithNan() { shouldRead = shouldRead(lessThan("some_nans", 3.0)); assertThat(shouldRead).isTrue(); - shouldRead = shouldRead(lessThanOrEqual("some_nans", 1.0)); + shouldRead = shouldRead(lessThanOrEqual("some_nans", 2.0)); assertThat(shouldRead).isTrue(); shouldRead = shouldRead(equal("some_nans", 2.0)); @@ -361,7 +365,7 @@ public void testDoubleWithNan() { shouldRead = shouldRead(lessThan("some_double_nans", 3.0)); assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue(); - shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 1.0)); + shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 2.0)); assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue(); shouldRead = shouldRead(equal("some_double_nans", 2.0)); diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index 2f053ee4414a..1207157a6f96 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index 2f053ee4414a..1207157a6f96 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index 2f053ee4414a..1207157a6f96 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9dc6b937129d..36f26cc51293 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -81,7 +81,7 @@ nessie = "0.108.3" netty-buffer = "4.2.16.Final" object-client-bundle = "3.3.2" orc = "1.9.9" -parquet = "1.17.1" +parquet = "1.18.0" roaringbitmap = "1.6.18" scala-collection-compat = "2.14.0" slf4j = "2.0.18" diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index bfed4c9b1aba..c5189003a775 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -171,14 +171,14 @@ org.apache.httpcomponents:httpclient:4.5 org.apache.httpcomponents:httpcore:4.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.bouncycastle:bcprov-jdk18on:1.85 org.checkerframework:checker-compat-qual:2.5 org.checkerframework:checker-qual:3.49 diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java index 4392452a14ea..0128ae3652b0 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java @@ -451,12 +451,21 @@ public void testVariantFloatNaN() throws IOException { assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L)); assertThat(metrics.nanValueCounts()).isEqualTo(Map.of()); - assertThat(metrics.lowerBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L))); - assertThat(metrics.upperBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L))); + // NaN is ignored when computing the shredded value bounds, so both bounds are the float value + assertThat(metrics.lowerBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L)); + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0F)); + assertThat(metrics.upperBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L)); + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0F)); - assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get())); + assertThat(metrics) + .extracting("originalTypes") + .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } @Test @@ -473,12 +482,21 @@ public void testVariantDoubleNaN() throws IOException { assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L)); assertThat(metrics.nanValueCounts()).isEqualTo(Map.of()); - assertThat(metrics.lowerBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L))); - assertThat(metrics.upperBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L))); + // NaN is ignored when computing the shredded value bounds, so both bounds are the double value + assertThat(metrics.lowerBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L)); + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0D)); + assertThat(metrics.upperBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L)); + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0D)); - assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get())); + assertThat(metrics) + .extracting("originalTypes") + .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } @Test diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index a9ea5bc4377d..187311b391f7 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index a9ea5bc4377d..187311b391f7 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index 74cd1230387b..098b90c10f2a 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1