diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q index 12f4be2536b9..8cc471d3c973 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q @@ -164,6 +164,27 @@ drop table ice_tbl_level; set hive.iceberg.stats.collect.partlevel=true; +-- HIVE-30131: a decimal column answers MIN, MAX and COUNT from its statistics, folded across the +-- partitions by the handler; 40.60 is stored as 40.6 and comes back padded to the column's scale +create external table ice_dec_stats (d decimal(5,2), p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2'); + +insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b'); +analyze table ice_dec_stats compute statistics for columns; + +explain +select min(d), max(d), count(d) from ice_dec_stats; + +select min(d), max(d), count(d) from ice_dec_stats; + +explain +select min(d), max(d), count(d) from ice_dec_stats where p = 'b'; + +select min(d), max(d), count(d) from ice_dec_stats where p = 'b'; + +drop table ice_dec_stats; + -- with the statistics kept by the metastore there are no per-partition numbers to answer from set hive.iceberg.stats.source=metastore; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out index 3293f88e1d25..14551387ec18 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out @@ -851,6 +851,108 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@ice_tbl_level POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_tbl_level +PREHOOK: query: create external table ice_dec_stats (d decimal(5,2), p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_dec_stats +POSTHOOK: query: create external table ice_dec_stats (d decimal(5,2), p string) + partitioned by spec (p) +stored by iceberg tblproperties ('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_dec_stats +PREHOOK: query: insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_dec_stats +POSTHOOK: query: insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_dec_stats +PREHOOK: query: analyze table ice_dec_stats compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: default@ice_dec_stats +PREHOOK: Output: default@ice_dec_stats@p=a +PREHOOK: Output: default@ice_dec_stats@p=b +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: analyze table ice_dec_stats compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: default@ice_dec_stats +POSTHOOK: Output: default@ice_dec_stats@p=a +POSTHOOK: Output: default@ice_dec_stats@p=b +POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain +select min(d), max(d), count(d) from ice_dec_stats +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select min(d), max(d), count(d) from ice_dec_stats +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select min(d), max(d), count(d) from ice_dec_stats +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(d), max(d), count(d) from ice_dec_stats +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +-10.40 40.60 3 +PREHOOK: query: explain +select min(d), max(d), count(d) from ice_dec_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain +select min(d), max(d), count(d) from ice_dec_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select min(d), max(d), count(d) from ice_dec_stats where p = 'b' +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select min(d), max(d), count(d) from ice_dec_stats where p = 'b' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: hdfs://### HDFS PATH ### +40.60 40.60 1 +PREHOOK: query: drop table ice_dec_stats +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice_dec_stats +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_dec_stats +POSTHOOK: query: drop table ice_dec_stats +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice_dec_stats +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_dec_stats PREHOOK: query: create external table ice_part_stats_hms (id bigint, p string) partitioned by spec (p) stored by iceberg tblproperties ('format-version'='2') diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index ffbd4383c9db..219a6a21067e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -29,10 +29,13 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Date; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Decimal; +import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -81,12 +84,13 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -166,6 +170,7 @@ public MetaDataProcessor (ParseContext pctx) { enum StatType{ Integer, Double, + Decimal, String, Boolean, Binary, @@ -205,28 +210,27 @@ enum GbyKeyType { NULL, CONSTANT, OTHER } - private StatType getType(String origType) { - if (serdeConstants.IntegralTypes.contains(origType)) { - return StatType.Integer; - } else if (origType.equals(serdeConstants.DOUBLE_TYPE_NAME) || - origType.equals(serdeConstants.FLOAT_TYPE_NAME)) { - return StatType.Double; - } else if (origType.equals(serdeConstants.BINARY_TYPE_NAME)) { - return StatType.Binary; - } else if (origType.equals(serdeConstants.BOOLEAN_TYPE_NAME)) { - return StatType.Boolean; - } else if (origType.equals(serdeConstants.STRING_TYPE_NAME)) { - return StatType.String; - } else if (origType.equals(serdeConstants.DATE_TYPE_NAME)) { - return StatType.Date; + private StatType getType(TypeInfo typeInfo) { + if (!(typeInfo instanceof PrimitiveTypeInfo pti)) { + return StatType.Unsupported; } - return StatType.Unsupported; + return switch (pti.getPrimitiveCategory()) { + case BYTE, SHORT, INT, LONG -> StatType.Integer; + case FLOAT, DOUBLE -> StatType.Double; + case DECIMAL -> StatType.Decimal; + case BINARY -> StatType.Binary; + case BOOLEAN -> StatType.Boolean; + case STRING -> StatType.String; + case DATE -> StatType.Date; + default -> StatType.Unsupported; + }; } private Long getNullCountFor(StatType type, ColumnStatisticsData statData) { return switch (type) { case Integer -> statData.getLongStats().getNumNulls(); case Double -> statData.getDoubleStats().getNumNulls(); + case Decimal -> statData.getDecimalStats().getNumNulls(); case String -> statData.getStringStats().getNumNulls(); case Boolean -> statData.getBooleanStats().getNumNulls(); case Binary -> statData.getBinaryStats().getNumNulls(); @@ -246,6 +250,7 @@ private static ColumnStatisticsData emptyColStats(StatType type) { return switch (type) { case Integer -> ColumnStatisticsData.longStats(new LongColumnStatsData()); case Double -> ColumnStatisticsData.doubleStats(new DoubleColumnStatsData()); + case Decimal -> ColumnStatisticsData.decimalStats(new DecimalColumnStatsData()); case String -> ColumnStatisticsData.stringStats(new StringColumnStatsData()); case Boolean -> ColumnStatisticsData.booleanStats(new BooleanColumnStatsData()); case Binary -> ColumnStatisticsData.binaryStats(new BinaryColumnStatsData()); @@ -473,7 +478,7 @@ else if (udaf instanceof GenericUDAFCount) { ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get( ((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()); String colName = colDesc.getColumn(); - StatType type = getType(colDesc.getTypeString()); + StatType type = getType(colDesc.getTypeInfo()); ColumnStatisticsData statData = scanColStats.statsFor(colName, type); if (statData == null) { @@ -501,6 +506,12 @@ else if (udaf instanceof GenericUDAFCount) { Date bound = high ? dstats.getHighValue() : dstats.getLowValue(); oneRow.add(isSet ? DateSubType.DAYS.cast(bound.getDaysSinceEpoch()) : null); } + case Decimal -> { + DecimalColumnStatsData dstats = statData.getDecimalStats(); + boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue(); + Decimal bound = high ? dstats.getHighValue() : dstats.getLowValue(); + oneRow.add(isSet ? DecimalUtils.getHiveDecimal(bound) : null); + } default -> { Logger.debug("Unsupported type: {} encountered in metadata optimizer for column: {}", colDesc.getTypeString(), colName); @@ -773,7 +784,7 @@ private Long countFor(AggregationDesc aggr, Map exprMap, l // count(col): the rows where it is set ExprNodeColumnDesc desc = (ExprNodeColumnDesc) param; String colName = desc.getColumn(); - StatType type = getType(desc.getTypeString()); + StatType type = getType(desc.getTypeInfo()); ColumnStatisticsData statData = scanColStats.statsFor(colName, type); if (statData == null) { diff --git a/ql/src/test/queries/clientpositive/metadata_only_queries.q b/ql/src/test/queries/clientpositive/metadata_only_queries.q index 23cea92c036b..a2d33f13cbb6 100644 --- a/ql/src/test/queries/clientpositive/metadata_only_queries.q +++ b/ql/src/test/queries/clientpositive/metadata_only_queries.q @@ -93,6 +93,28 @@ explain select min(i), '1' as one, max(i), min(b), max(b), min(f), max(f), 3+4.0 as three, min(d), max(d) from stats_tbl_part; select min(i), '1' as one, max(i), min(b), max(b), min(f), max(f), 3+4.0 as three, min(d), max(d) from stats_tbl_part; +-- HIVE-30131: MIN, MAX and COUNT on decimal columns are answered from column statistics, +-- including a column whose values are all NULL (n). +-- The statistics keep 40.60 as 40.6, so the expected 40.60 checks that the result is padded back +-- to the column's scale. +create table stats_dec(d decimal(5,2), n decimal(7,3)); +insert into stats_dec values (-10.40, null), (40.60, null), (1.50, null), (null, null); +analyze table stats_dec compute statistics for columns d,n; + +explain +select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec; +select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec; + +-- and over partitions, where each bound is the aggregate of the per-partition statistics +create table stats_dec_part(d decimal(5,2)) partitioned by (p string); +insert into stats_dec_part partition(p='a') values (-10.40), (1.50); +insert into stats_dec_part partition(p='b') values (40.60), (null); +analyze table stats_dec_part partition(p) compute statistics for columns d; + +explain +select min(d), max(d), count(d) from stats_dec_part; +select min(d), max(d), count(d) from stats_dec_part; + explain select count(ts) from stats_tbl_part; explain select count('1') from stats_tbl group by '1'; diff --git a/ql/src/test/queries/clientpositive/vector_decimal_udf.q b/ql/src/test/queries/clientpositive/vector_decimal_udf.q index b3288ee042c8..65867252de61 100644 --- a/ql/src/test/queries/clientpositive/vector_decimal_udf.q +++ b/ql/src/test/queries/clientpositive/vector_decimal_udf.q @@ -2,6 +2,7 @@ set hive.mapred.mode=nonstrict; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=minimal; +set hive.compute.query.using.stats=false; -- SORT_QUERY_RESULTS diff --git a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out index 1b965503c748..ad3139f6c16b 100644 --- a/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out +++ b/ql/src/test/results/clientpositive/llap/metadata_only_queries.q.out @@ -696,6 +696,136 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@stats_tbl_part #### A masked pattern was here #### 65536 1 65791 4294967296 4294967551 0.01 99.98 7 0.01 50.0 +PREHOOK: query: create table stats_dec(d decimal(5,2), n decimal(7,3)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_dec +POSTHOOK: query: create table stats_dec(d decimal(5,2), n decimal(7,3)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_dec +PREHOOK: query: insert into stats_dec values (-10.40, null), (40.60, null), (1.50, null), (null, null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_dec +POSTHOOK: query: insert into stats_dec values (-10.40, null), (40.60, null), (1.50, null), (null, null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_dec +POSTHOOK: Lineage: stats_dec.d SCRIPT [] +POSTHOOK: Lineage: stats_dec.n EXPRESSION [] +PREHOOK: query: analyze table stats_dec compute statistics for columns d,n +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@stats_dec +PREHOOK: Output: default@stats_dec +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_dec compute statistics for columns d,n +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@stats_dec +POSTHOOK: Output: default@stats_dec +#### A masked pattern was here #### +PREHOOK: query: explain +select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_dec +#### A masked pattern was here #### +POSTHOOK: query: explain +select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_dec +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_dec +#### A masked pattern was here #### +POSTHOOK: query: select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_dec +#### A masked pattern was here #### +-10.40 40.60 3 NULL NULL 0 +PREHOOK: query: create table stats_dec_part(d decimal(5,2)) partitioned by (p string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@stats_dec_part +POSTHOOK: query: create table stats_dec_part(d decimal(5,2)) partitioned by (p string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@stats_dec_part +PREHOOK: query: insert into stats_dec_part partition(p='a') values (-10.40), (1.50) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_dec_part@p=a +POSTHOOK: query: insert into stats_dec_part partition(p='a') values (-10.40), (1.50) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_dec_part@p=a +POSTHOOK: Lineage: stats_dec_part PARTITION(p=a).d SCRIPT [] +PREHOOK: query: insert into stats_dec_part partition(p='b') values (40.60), (null) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@stats_dec_part@p=b +POSTHOOK: query: insert into stats_dec_part partition(p='b') values (40.60), (null) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@stats_dec_part@p=b +POSTHOOK: Lineage: stats_dec_part PARTITION(p=b).d SCRIPT [] +PREHOOK: query: analyze table stats_dec_part partition(p) compute statistics for columns d +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@stats_dec_part +PREHOOK: Input: default@stats_dec_part@p=a +PREHOOK: Input: default@stats_dec_part@p=b +PREHOOK: Output: default@stats_dec_part +PREHOOK: Output: default@stats_dec_part@p=a +PREHOOK: Output: default@stats_dec_part@p=b +#### A masked pattern was here #### +POSTHOOK: query: analyze table stats_dec_part partition(p) compute statistics for columns d +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@stats_dec_part +POSTHOOK: Input: default@stats_dec_part@p=a +POSTHOOK: Input: default@stats_dec_part@p=b +POSTHOOK: Output: default@stats_dec_part +POSTHOOK: Output: default@stats_dec_part@p=a +POSTHOOK: Output: default@stats_dec_part@p=b +#### A masked pattern was here #### +PREHOOK: query: explain +select min(d), max(d), count(d) from stats_dec_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_dec_part +#### A masked pattern was here #### +POSTHOOK: query: explain +select min(d), max(d), count(d) from stats_dec_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_dec_part +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select min(d), max(d), count(d) from stats_dec_part +PREHOOK: type: QUERY +PREHOOK: Input: default@stats_dec_part +#### A masked pattern was here #### +POSTHOOK: query: select min(d), max(d), count(d) from stats_dec_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@stats_dec_part +#### A masked pattern was here #### +-10.40 40.60 3 PREHOOK: query: explain select count(ts) from stats_tbl_part PREHOOK: type: QUERY PREHOOK: Input: default@stats_tbl_part