From 6d61dce2b16ef1dd7cafbd2de8b1d381c4d0f30e Mon Sep 17 00:00:00 2001 From: Vinoo Ganesh Date: Sat, 12 Sep 2026 23:22:11 -0400 Subject: [PATCH] Update inlined parquet.thrift to parquet-format 2.14.0 Brings the IDL up from 2.13.0, picking up the FILE logical type, INT96 chronological ordering and the ALP encoding. FILE now exists in the LogicalType union, so the converter would have thrown on it instead of falling into the unrecognised-type branch that ignores it. It is ignored explicitly now, keeping the physical type as before. ALP is in the format enum but has no Java implementation yet, so testEnumEquivalence skips it until one lands. --- .../src/main/thrift/parquet-format.version | 2 +- .../src/main/thrift/parquet.thrift | 58 ++++++++++++++++--- .../converter/ParquetMetadataConverter.java | 4 ++ .../TestParquetMetadataConverter.java | 15 +++++ 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/parquet-format-structures/src/main/thrift/parquet-format.version b/parquet-format-structures/src/main/thrift/parquet-format.version index 774d7578dc..762e7a62e8 100644 --- a/parquet-format-structures/src/main/thrift/parquet-format.version +++ b/parquet-format-structures/src/main/thrift/parquet-format.version @@ -1,2 +1,2 @@ # Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh. -parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0 +parquet-format.commit=04d56f291ff963e98bc37ab8100e2fc133ff583c diff --git a/parquet-format-structures/src/main/thrift/parquet.thrift b/parquet-format-structures/src/main/thrift/parquet.thrift index fe259d61bc..bb582017bf 100644 --- a/parquet-format-structures/src/main/thrift/parquet.thrift +++ b/parquet-format-structures/src/main/thrift/parquet.thrift @@ -468,6 +468,17 @@ struct GeographyType { 2: optional EdgeInterpolationAlgorithm algorithm; } +/** + * File logical type annotation + * + * Annotates a group that represents a reference to a file, or to a range of + * bytes that may be stored inline or in an external file. + * + * See LogicalTypes.md for details. + */ +struct FileType { +} + /** * LogicalType annotations to replace ConvertedType. * @@ -501,6 +512,7 @@ union LogicalType { 16: VariantType VARIANT // no compatible ConvertedType 17: GeometryType GEOMETRY // no compatible ConvertedType 18: GeographyType GEOGRAPHY // no compatible ConvertedType + 19: FileType FILE // no compatible ConvertedType } /** @@ -636,6 +648,14 @@ enum Encoding { Support for INT32, INT64 and FIXED_LEN_BYTE_ARRAY added in 2.11. */ BYTE_STREAM_SPLIT = 9; + + /** Adaptive Lossless floating-Point (ALP) encoding for FLOAT and DOUBLE. + Losslessly converts decimal-like floating-point values to integers via + decimal scaling, then applies Frame of Reference (FOR) encoding and + bit-packing; values that cannot be converted losslessly are stored as + exceptions. See Encodings.md for the detailed specification. + */ + ALP = 10; } /** @@ -1061,6 +1081,9 @@ struct TypeDefinedOrder {} /** Empty struct to signal IEEE 754 total order for floating point types */ struct IEEE754TotalOrder {} +/** Empty struct to signal chronological ordering of physical type INT96 */ +struct Int96TimestampOrder {} + /** * Union to specify the order used for the min_value and max_value fields for a * column. This union takes the role of an enhanced enum that allows rich @@ -1071,6 +1094,8 @@ struct IEEE754TotalOrder {} * physical type (if there is no logical type). * * IEEE754TotalOrder - the floating point column uses IEEE 754 total order. * + * * Int96TimestampOrder - the INT96 column uses chronological timestamp order. + * * If the reader does not support the value of this union, min and max stats * for this column should be ignored. */ @@ -1103,25 +1128,29 @@ union ColumnOrder { * VARIANT - undefined * GEOMETRY - undefined * GEOGRAPHY - undefined + * FILE - undefined * * In the absence of logical types, the sort order is determined by the physical type: * BOOLEAN - false, true * INT32 - signed comparison * INT64 - signed comparison - * INT96 (only used for legacy timestamps) - undefined(+) + * INT96 (only used for legacy timestamps) - depends on sort order (+) * FLOAT - signed comparison of the represented value (*) * DOUBLE - signed comparison of the represented value (*) * BYTE_ARRAY - unsigned byte-wise comparison * FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison * * (+) While the INT96 type has been deprecated, at the time of writing it is - * still used in many legacy systems. If a Parquet implementation chooses - * to write statistics for INT96 columns, it is recommended to order them - * according to the legacy rules: - * - compare the last 4 bytes (days) as a little-endian 32-bit signed integer - * - if equal last 4 bytes, compare the first 8 bytes as a little-endian - * 64-bit signed integer (nanos) - * See https://github.com/apache/parquet-format/issues/502 for more details + * still used in many legacy systems. It is optional for writers to emit + * statistics for INT96 columns. Writers that emit stats for such columns + * should use the INT96_TIMESTAMP_ORDER for this type and order the values + * according to the legacy rules: + * - compare the last 4 bytes (days) as a little-endian 32-bit signed integer + * - if equal last 4 bytes, compare the first 8 bytes as a little-endian + * 64-bit signed integer (nanos) + * If TYPE_ORDER is used for an INT96 column, readers should ignore all statistics + * (`min`/`max` fields in `Statistics` and `min_values`/`max_values` fields in + * `ColumnIndex`) for that column. * * (*) Because TYPE_ORDER is ambiguous for floating point types due to * underspecified handling of NaN and -0/+0, it is recommended that writers @@ -1195,6 +1224,12 @@ union ColumnOrder { * or max_values indicates that all non-null values are NaN. */ 2: IEEE754TotalOrder IEEE_754_TOTAL_ORDER; + + /* + * The INT96 timestamp type is ordered chronologically. Only columns of + * physical type INT96 may use this ordering. + */ + 3: Int96TimestampOrder INT96_TIMESTAMP_ORDER; } struct PageLocation { @@ -1278,6 +1313,13 @@ struct ColumnIndex { * - If the order of this column is IEEE754_TOTAL_ORDER, then min_values[i] * and max_values[i] of that page must be set to the smallest and largest * NaN values as defined by IEEE 754 total order. + * + * For columns of physical type INT96, the writer must do the following: + * - If the order of this column is not INT96_TIMESTAMP_ORDER, then a column + * index must not be written for this column chunk. + * - If the order of this column is INT96_TIMESTAMP_ORDER, the min_values[i] + * and max_values[i] of that page must be set to the smallest and largest + * values as defined by the INT96 chronological timestamp ordering. */ 2: required list min_values 3: required list max_values diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index f6ee73bbc0..cc7805c317 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -1409,6 +1409,10 @@ LogicalTypeAnnotation getLogicalTypeAnnotation(LogicalType type) { case VARIANT: VariantType variant = type.getVARIANT(); return LogicalTypeAnnotation.variantType(variant.getSpecification_version()); + case FILE: + // Present in the format but not mapped to a LogicalTypeAnnotation yet. Ignore it to + // preserve the physical type, as an unrecognised logical type would be. + return null; default: throw new RuntimeException("Unknown logical type " + type); } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index f5222de828..784a837e3e 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -101,6 +101,7 @@ import org.apache.parquet.format.DecimalType; import org.apache.parquet.format.FieldRepetitionType; import org.apache.parquet.format.FileMetaData; +import org.apache.parquet.format.FileType; import org.apache.parquet.format.GeographyType; import org.apache.parquet.format.GeometryType; import org.apache.parquet.format.GeospatialStatistics; @@ -542,6 +543,15 @@ public void testLogicalToConvertedTypeConversion() { .isEqualTo(ConvertedType.MAP_KEY_VALUE); } + @Test + public void testFileLogicalTypeIsIgnoredRatherThanFailing() { + ParquetMetadataConverter converter = new ParquetMetadataConverter(); + // FILE has no LogicalTypeAnnotation yet, so it must degrade to the physical type the way an + // unrecognised logical type does, rather than throwing. + assertThat(converter.getLogicalTypeAnnotation(LogicalType.FILE(new FileType()))) + .isNull(); + } + @Test public void testEnumEquivalence() { ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); @@ -550,6 +560,11 @@ public void testEnumEquivalence() { .isEqualTo(encoding); } for (org.apache.parquet.format.Encoding encoding : org.apache.parquet.format.Encoding.values()) { + // ALP is in the format spec but is not implemented on the Java side yet, so it has no + // org.apache.parquet.column.Encoding to round trip through. Remove this once it does. + if (encoding == org.apache.parquet.format.Encoding.ALP) { + continue; + } assertThat(parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding))) .isEqualTo(encoding); }