Skip to content

HIVE-29301: Missing histogram in DESCRIBE FORMATTED after prior column DESCRIBE #6670

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cyanzheng2926 wants to merge 2 commits into
base: master
Choose a base branch
from
Open

HIVE-29301: Missing histogram in DESCRIBE FORMATTED after prior column DESCRIBE #6670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
203d0be
HIVE-29301: Missing histogram info in DESCRIBE FORMATTED when executi…
cyanzheng2926 Aug 3, 2026
0a45af5
HIVE-29301: updated golden to execute with TestMiniLlapLocalCliDriver
cyanzheng2926 Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.orc.OrcConf;
import org.slf4j.Logger;
Expand Down Expand Up @@ -77,6 +81,12 @@ public class SetProcessor implements CommandProcessor {
private static final Pattern TIME_ZONE_PATTERN =
Pattern.compile("^time(\\s)+zone\\s", Pattern.CASE_INSENSITIVE);

private static final Set<String> EMBEDDED_MS_STATS_FETCH_KEYS = Sets.newHashSet(
MetastoreConf.ConfVars.STATS_FETCH_KLL.getVarname(),
MetastoreConf.ConfVars.STATS_FETCH_KLL.getHiveName(),
MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR.getVarname(),
MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR.getHiveName());

public static boolean getBoolean(String value) {
if (value.equals("on") || value.equals("true")) {
return true;
Expand Down Expand Up @@ -273,9 +283,28 @@ public Map<String, String> getHiveVariable() {
if (register) {
ss.getOverriddenConfigurations().put(key, value);
}
syncEmbeddedMetastoreConfIfNeeded(conf, key);
return result;
}

/**
* Session SET updates HiveConf in place, but embedded ObjectStore may still read a stale
* Configuration reference. Rebind handler and ObjectStore when stats fetch flags change.
*/
private static void syncEmbeddedMetastoreConfIfNeeded(HiveConf conf, String key) {
if (!EMBEDDED_MS_STATS_FETCH_KEYS.contains(key)) {
return;
}
try {
IMetaStoreClient msc = Hive.get(conf).getMSC();
if (msc.isLocalMetaStore()) {
msc.syncEmbeddedHandlerConf(conf);
}
} catch (HiveException | MetaException e) {
LOG.warn("Failed to sync embedded metastore configuration for {}", key, e);
}
}

private SortedMap<String,String> propertiesToSortedMap(Properties p){
SortedMap<String,String> sortedPropMap = new TreeMap<String,String>();
for (Map.Entry<Object, Object> entry : p.entrySet() ){
Expand Down
15 changes: 15 additions & 0 deletions ql/src/test/queries/clientpositive/stats_histogram_order_broken.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
set hive.stats.kll.enable=true;
set metastore.stats.fetch.bitvector=true;

CREATE TABLE tab1 AS (SELECT 1 as key);

DESCRIBE FORMATTED tab1 key;
set metastore.stats.fetch.kll=true;

CREATE TABLE tab2 AS (SELECT 1 as key);

ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS;
DESCRIBE FORMATTED tab2 key;

DROP TABLE tab1;
DROP TABLE tab2;
15 changes: 15 additions & 0 deletions ql/src/test/queries/clientpositive/stats_histogram_order_working.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
set hive.stats.kll.enable=true;
set metastore.stats.fetch.bitvector=true;

CREATE TABLE tab1 AS (SELECT 1 as key);

set metastore.stats.fetch.kll=true;
DESCRIBE FORMATTED tab1 key;

CREATE TABLE tab2 AS (SELECT 1 as key);

ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS;
DESCRIBE FORMATTED tab2 key;

DROP TABLE tab1;
DROP TABLE tab2;
91 changes: 91 additions & 0 deletions ql/src/test/results/clientpositive/llap/stats_histogram_order_broken.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
PREHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key)
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@tab1
POSTHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key)
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab1
POSTHOOK: Lineage: tab1.key SIMPLE []
PREHOOK: query: DESCRIBE FORMATTED tab1 key
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@tab1
POSTHOOK: query: DESCRIBE FORMATTED tab1 key
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tab1
col_name key
data_type int
min 1
max 1
num_nulls 0
distinct_count 1
avg_col_len
max_col_len
num_trues
num_falses
bit_vector HL
comment from deserializer
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
PREHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key)
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@tab2
POSTHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key)
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab2
POSTHOOK: Lineage: tab2.key SIMPLE []
PREHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS
PREHOOK: type: ANALYZE_TABLE
PREHOOK: Input: default@tab2
PREHOOK: Output: default@tab2
#### A masked pattern was here ####
POSTHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS
POSTHOOK: type: ANALYZE_TABLE
POSTHOOK: Input: default@tab2
POSTHOOK: Output: default@tab2
#### A masked pattern was here ####
PREHOOK: query: DESCRIBE FORMATTED tab2 key
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@tab2
POSTHOOK: query: DESCRIBE FORMATTED tab2 key
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tab2
col_name key
data_type int
min 1
max 1
num_nulls 0
distinct_count 1
avg_col_len
max_col_len
num_trues
num_falses
bit_vector HL
histogram Q1: 1, Q2: 1, Q3: 1
comment from deserializer
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
PREHOOK: query: DROP TABLE tab1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tab1
PREHOOK: Output: database:default
PREHOOK: Output: default@tab1
POSTHOOK: query: DROP TABLE tab1
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tab1
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab1
PREHOOK: query: DROP TABLE tab2
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tab2
PREHOOK: Output: database:default
PREHOOK: Output: default@tab2
POSTHOOK: query: DROP TABLE tab2
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tab2
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab2
92 changes: 92 additions & 0 deletions ql/src/test/results/clientpositive/llap/stats_histogram_order_working.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
PREHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key)
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@tab1
POSTHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key)
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab1
POSTHOOK: Lineage: tab1.key SIMPLE []
PREHOOK: query: DESCRIBE FORMATTED tab1 key
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@tab1
POSTHOOK: query: DESCRIBE FORMATTED tab1 key
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tab1
col_name key
data_type int
min 1
max 1
num_nulls 0
distinct_count 1
avg_col_len
max_col_len
num_trues
num_falses
bit_vector HL
histogram Q1: 1, Q2: 1, Q3: 1
comment from deserializer
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
PREHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key)
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: database:default
PREHOOK: Output: default@tab2
POSTHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key)
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab2
POSTHOOK: Lineage: tab2.key SIMPLE []
PREHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS
PREHOOK: type: ANALYZE_TABLE
PREHOOK: Input: default@tab2
PREHOOK: Output: default@tab2
#### A masked pattern was here ####
POSTHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS
POSTHOOK: type: ANALYZE_TABLE
POSTHOOK: Input: default@tab2
POSTHOOK: Output: default@tab2
#### A masked pattern was here ####
PREHOOK: query: DESCRIBE FORMATTED tab2 key
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@tab2
POSTHOOK: query: DESCRIBE FORMATTED tab2 key
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tab2
col_name key
data_type int
min 1
max 1
num_nulls 0
distinct_count 1
avg_col_len
max_col_len
num_trues
num_falses
bit_vector HL
histogram Q1: 1, Q2: 1, Q3: 1
comment from deserializer
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}}
PREHOOK: query: DROP TABLE tab1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tab1
PREHOOK: Output: database:default
PREHOOK: Output: default@tab1
POSTHOOK: query: DROP TABLE tab1
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tab1
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab1
PREHOOK: query: DROP TABLE tab2
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@tab2
PREHOOK: Output: database:default
PREHOOK: Output: default@tab2
POSTHOOK: query: DROP TABLE tab2
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@tab2
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tab2
9 changes: 9 additions & 0 deletions ...ore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ default boolean isLocalMetaStore() {
return false;
}

/**
* Rebind embedded metastore handler configuration after session-level SET.
* Default no-op; only embedded clients override this.
* Remote metastore clients must not perform any work here.
*/
@InterfaceAudience.Private
default void syncEmbeddedHandlerConf(Configuration conf) {
}

/**
* Tries to reconnect this MetaStoreClient to the MetaStore.
*/
Expand Down
5 changes: 5 additions & 0 deletions ...-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ public boolean isLocalMetaStore() {
return delegate.isLocalMetaStore();
}

@Override
public void syncEmbeddedHandlerConf(Configuration conf) {
delegate.syncEmbeddedHandlerConf(conf);
}

@Override
public void reconnect() throws MetaException {
delegate.reconnect();
Expand Down
15 changes: 15 additions & 0 deletions ...ient/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
import org.apache.hadoop.hive.metastore.MetaStorePlainSaslHelper;
import org.apache.hadoop.hive.metastore.PartitionDropOptions;
Expand Down Expand Up @@ -432,6 +433,20 @@ public boolean isLocalMetaStore() {
return localMetaStore;
}

@Override
public void syncEmbeddedHandlerConf(Configuration conf) {
if (!localMetaStore || client == null) {
return;
}
try {
java.lang.reflect.Method setConfMethod =
client.getClass().getMethod("setConf", Configuration.class);
setConfMethod.invoke(client, conf);
} catch (ReflectiveOperationException e) {
LOG.warn("Failed to sync embedded metastore handler configuration", e);
}
}

@Override
public boolean isCompatibleWith(Configuration conf) {
// Make a copy of currentMetaVars, there is a race condition that
Expand Down
Loading