Skip to content

HIVE-30131: StatsOptimizer: Answer MIN, MAX and COUNT on a DECIMAL column from column statistics #6844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
deniskuzZ wants to merge 1 commit into
base: master
Choose a base branch
from
+304 −18
Open

HIVE-30131: StatsOptimizer: Answer MIN, MAX and COUNT on a DECIMAL column from column statistics #6844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/iceberg_part_colstats.q
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,27 @@ drop table ice_tbl_level;

set hive.iceberg.stats.collect.partlevel=true;

-- HIVE-30131: a decimal column answers MIN, MAX and COUNT from its statistics, folded across the
-- partitions by the handler; 40.60 is stored as 40.6 and comes back padded to the column's scale
create external table ice_dec_stats (d decimal(5,2), p string)
partitioned by spec (p)
stored by iceberg tblproperties ('format-version'='2');

insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b');
analyze table ice_dec_stats compute statistics for columns;

explain
select min(d), max(d), count(d) from ice_dec_stats;

select min(d), max(d), count(d) from ice_dec_stats;

explain
select min(d), max(d), count(d) from ice_dec_stats where p = 'b';

select min(d), max(d), count(d) from ice_dec_stats where p = 'b';

drop table ice_dec_stats;

-- with the statistics kept by the metastore there are no per-partition numbers to answer from
set hive.iceberg.stats.source=metastore;

Expand Down
102 changes: 102 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/iceberg_part_colstats.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -851,6 +851,108 @@ POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@ice_tbl_level
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_tbl_level
PREHOOK: query: create external table ice_dec_stats (d decimal(5,2), p string)
partitioned by spec (p)
stored by iceberg tblproperties ('format-version'='2')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@ice_dec_stats
POSTHOOK: query: create external table ice_dec_stats (d decimal(5,2), p string)
partitioned by spec (p)
stored by iceberg tblproperties ('format-version'='2')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_dec_stats
PREHOOK: query: insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_dec_stats
POSTHOOK: query: insert into ice_dec_stats values (-10.40, 'a'), (1.50, 'a'), (40.60, 'b'), (null, 'b')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_dec_stats
PREHOOK: query: analyze table ice_dec_stats compute statistics for columns
PREHOOK: type: ANALYZE_TABLE
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: default@ice_dec_stats
PREHOOK: Output: default@ice_dec_stats@p=a
PREHOOK: Output: default@ice_dec_stats@p=b
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: analyze table ice_dec_stats compute statistics for columns
POSTHOOK: type: ANALYZE_TABLE
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: default@ice_dec_stats
POSTHOOK: Output: default@ice_dec_stats@p=a
POSTHOOK: Output: default@ice_dec_stats@p=b
POSTHOOK: Output: hdfs://### HDFS PATH ###
PREHOOK: query: explain
select min(d), max(d), count(d) from ice_dec_stats
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: explain
select min(d), max(d), count(d) from ice_dec_stats
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: hdfs://### HDFS PATH ###
STAGE DEPENDENCIES:
Stage-0 is a root stage

STAGE PLANS:
Stage: Stage-0
Fetch Operator
limit: 1
Processor Tree:
ListSink

PREHOOK: query: select min(d), max(d), count(d) from ice_dec_stats
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select min(d), max(d), count(d) from ice_dec_stats
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: hdfs://### HDFS PATH ###
-10.40 40.60 3
PREHOOK: query: explain
select min(d), max(d), count(d) from ice_dec_stats where p = 'b'
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: explain
select min(d), max(d), count(d) from ice_dec_stats where p = 'b'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: hdfs://### HDFS PATH ###
STAGE DEPENDENCIES:
Stage-0 is a root stage

STAGE PLANS:
Stage: Stage-0
Fetch Operator
limit: 1
Processor Tree:
ListSink

PREHOOK: query: select min(d), max(d), count(d) from ice_dec_stats where p = 'b'
PREHOOK: type: QUERY
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select min(d), max(d), count(d) from ice_dec_stats where p = 'b'
POSTHOOK: type: QUERY
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: hdfs://### HDFS PATH ###
40.60 40.60 1
PREHOOK: query: drop table ice_dec_stats
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@ice_dec_stats
PREHOOK: Output: database:default
PREHOOK: Output: default@ice_dec_stats
POSTHOOK: query: drop table ice_dec_stats
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@ice_dec_stats
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_dec_stats
PREHOOK: query: create external table ice_part_stats_hms (id bigint, p string)
partitioned by spec (p)
stored by iceberg tblproperties ('format-version'='2')
Expand Down
47 changes: 29 additions & 18 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
Expand Down Expand Up @@ -81,12 +84,13 @@
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMin;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -166,6 +170,7 @@ public MetaDataProcessor (ParseContext pctx) {
enum StatType{
Integer,
Double,
Decimal,
String,
Boolean,
Binary,
Expand Down Expand Up @@ -205,28 +210,27 @@ enum GbyKeyType {
NULL, CONSTANT, OTHER
}

private StatType getType(String origType) {
if (serdeConstants.IntegralTypes.contains(origType)) {
return StatType.Integer;
} else if (origType.equals(serdeConstants.DOUBLE_TYPE_NAME) ||
origType.equals(serdeConstants.FLOAT_TYPE_NAME)) {
return StatType.Double;
} else if (origType.equals(serdeConstants.BINARY_TYPE_NAME)) {
return StatType.Binary;
} else if (origType.equals(serdeConstants.BOOLEAN_TYPE_NAME)) {
return StatType.Boolean;
} else if (origType.equals(serdeConstants.STRING_TYPE_NAME)) {
return StatType.String;
} else if (origType.equals(serdeConstants.DATE_TYPE_NAME)) {
return StatType.Date;
private StatType getType(TypeInfo typeInfo) {
if (!(typeInfo instanceof PrimitiveTypeInfo pti)) {
return StatType.Unsupported;
}
return StatType.Unsupported;
return switch (pti.getPrimitiveCategory()) {
case BYTE, SHORT, INT, LONG -> StatType.Integer;
case FLOAT, DOUBLE -> StatType.Double;
case DECIMAL -> StatType.Decimal;
case BINARY -> StatType.Binary;
case BOOLEAN -> StatType.Boolean;
case STRING -> StatType.String;
case DATE -> StatType.Date;
default -> StatType.Unsupported;
};
}

private Long getNullCountFor(StatType type, ColumnStatisticsData statData) {
return switch (type) {
case Integer -> statData.getLongStats().getNumNulls();
case Double -> statData.getDoubleStats().getNumNulls();
case Decimal -> statData.getDecimalStats().getNumNulls();
case String -> statData.getStringStats().getNumNulls();
case Boolean -> statData.getBooleanStats().getNumNulls();
case Binary -> statData.getBinaryStats().getNumNulls();
Expand All @@ -246,6 +250,7 @@ private static ColumnStatisticsData emptyColStats(StatType type) {
return switch (type) {
case Integer -> ColumnStatisticsData.longStats(new LongColumnStatsData());
case Double -> ColumnStatisticsData.doubleStats(new DoubleColumnStatsData());
case Decimal -> ColumnStatisticsData.decimalStats(new DecimalColumnStatsData());
case String -> ColumnStatisticsData.stringStats(new StringColumnStatsData());
case Boolean -> ColumnStatisticsData.booleanStats(new BooleanColumnStatsData());
case Binary -> ColumnStatisticsData.binaryStats(new BinaryColumnStatsData());
Expand Down Expand Up @@ -473,7 +478,7 @@ else if (udaf instanceof GenericUDAFCount) {
ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(
((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
String colName = colDesc.getColumn();
StatType type = getType(colDesc.getTypeString());
StatType type = getType(colDesc.getTypeInfo());

ColumnStatisticsData statData = scanColStats.statsFor(colName, type);
if (statData == null) {
Expand Down Expand Up @@ -501,6 +506,12 @@ else if (udaf instanceof GenericUDAFCount) {
Date bound = high ? dstats.getHighValue() : dstats.getLowValue();
oneRow.add(isSet ? DateSubType.DAYS.cast(bound.getDaysSinceEpoch()) : null);
}
case Decimal -> {
DecimalColumnStatsData dstats = statData.getDecimalStats();
boolean isSet = high ? dstats.isSetHighValue() : dstats.isSetLowValue();
Decimal bound = high ? dstats.getHighValue() : dstats.getLowValue();
oneRow.add(isSet ? DecimalUtils.getHiveDecimal(bound) : null);
}
default -> {
Logger.debug("Unsupported type: {} encountered in metadata optimizer for column: {}",
colDesc.getTypeString(), colName);
Expand Down Expand Up @@ -773,7 +784,7 @@ private Long countFor(AggregationDesc aggr, Map<String, ExprNodeDesc> exprMap, l
// count(col): the rows where it is set
ExprNodeColumnDesc desc = (ExprNodeColumnDesc) param;
String colName = desc.getColumn();
StatType type = getType(desc.getTypeString());
StatType type = getType(desc.getTypeInfo());

ColumnStatisticsData statData = scanColStats.statsFor(colName, type);
if (statData == null) {
Expand Down
21 changes: 21 additions & 0 deletions ql/src/test/queries/clientpositive/metadata_only_queries.q
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,27 @@ explain
select min(i), '1' as one, max(i), min(b), max(b), min(f), max(f), 3+4.0 as three, min(d), max(d) from stats_tbl_part;
select min(i), '1' as one, max(i), min(b), max(b), min(f), max(f), 3+4.0 as three, min(d), max(d) from stats_tbl_part;

-- HIVE-30131: the bounds a decimal column reports, including a column that is entirely NULL.
-- 40.60 is stored normalised to scale 1, so reading it back padded to the column's scale exercises
-- the object inspector rather than just the statistics.
create table stats_dec(d decimal(5,2), n decimal(7,3));
insert into stats_dec values (-10.40, null), (40.60, null), (1.50, null), (null, null);
analyze table stats_dec compute statistics for columns d,n;

explain
select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec;
select min(d), max(d), count(d), min(n), max(n), count(n) from stats_dec;

-- and over partitions, where each bound is the aggregate of the per-partition statistics
create table stats_dec_part(d decimal(5,2)) partitioned by (p string);
insert into stats_dec_part partition(p='a') values (-10.40), (1.50);
insert into stats_dec_part partition(p='b') values (40.60), (null);
analyze table stats_dec_part partition(p) compute statistics for columns d;

explain
select min(d), max(d), count(d) from stats_dec_part;
select min(d), max(d), count(d) from stats_dec_part;

explain select count(ts) from stats_tbl_part;

explain select count('1') from stats_tbl group by '1';
Expand Down
1 change: 1 addition & 0 deletions ql/src/test/queries/clientpositive/vector_decimal_udf.q
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
SET hive.vectorized.execution.enabled=true;
set hive.fetch.task.conversion=minimal;
set hive.compute.query.using.stats=false;

@Aggarwal-Raghav Aggarwal-Raghav Oct 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on CI run 7 I'm assuming, this was explicitly turned off to preserve the vectorized Tez execution coverage and new q file (metadata_only_queries.q and iceberg_part_colstats.q.) will test this PR?

@deniskuzZ deniskuzZ Oct 12, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, exactly. Otherwise, vector_decimal_udf.q was answered from stats, skipping execution


-- SORT_QUERY_RESULTS

Expand Down
Loading
Loading