diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 87547116fed1..1f0cc6931020 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -37,8 +37,12 @@ import org.apache.hadoop.hive.conf.HiveVariableSource; import org.apache.hadoop.hive.conf.VariableSubstitution; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Schema; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.orc.OrcConf; import org.slf4j.Logger; @@ -77,6 +81,12 @@ public class SetProcessor implements CommandProcessor { private static final Pattern TIME_ZONE_PATTERN = Pattern.compile("^time(\\s)+zone\\s", Pattern.CASE_INSENSITIVE); + private static final Set EMBEDDED_MS_STATS_FETCH_KEYS = Sets.newHashSet( + MetastoreConf.ConfVars.STATS_FETCH_KLL.getVarname(), + MetastoreConf.ConfVars.STATS_FETCH_KLL.getHiveName(), + MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR.getVarname(), + MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR.getHiveName()); + public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { return true; @@ -273,9 +283,28 @@ public Map getHiveVariable() { if (register) { ss.getOverriddenConfigurations().put(key, value); } + syncEmbeddedMetastoreConfIfNeeded(conf, key); return result; } + /** + * Session SET updates HiveConf in place, but embedded ObjectStore may still read a stale + * Configuration reference. Rebind handler and ObjectStore when stats fetch flags change. + */ + private static void syncEmbeddedMetastoreConfIfNeeded(HiveConf conf, String key) { + if (!EMBEDDED_MS_STATS_FETCH_KEYS.contains(key)) { + return; + } + try { + IMetaStoreClient msc = Hive.get(conf).getMSC(); + if (msc.isLocalMetaStore()) { + msc.syncEmbeddedHandlerConf(conf); + } + } catch (HiveException | MetaException e) { + LOG.warn("Failed to sync embedded metastore configuration for {}", key, e); + } + } + private SortedMap propertiesToSortedMap(Properties p){ SortedMap sortedPropMap = new TreeMap(); for (Map.Entry entry : p.entrySet() ){ diff --git a/ql/src/test/queries/clientpositive/stats_histogram_order_broken.q b/ql/src/test/queries/clientpositive/stats_histogram_order_broken.q new file mode 100644 index 000000000000..11e77ad12c61 --- /dev/null +++ b/ql/src/test/queries/clientpositive/stats_histogram_order_broken.q @@ -0,0 +1,15 @@ +set hive.stats.kll.enable=true; +set metastore.stats.fetch.bitvector=true; + +CREATE TABLE tab1 AS (SELECT 1 as key); + +DESCRIBE FORMATTED tab1 key; +set metastore.stats.fetch.kll=true; + +CREATE TABLE tab2 AS (SELECT 1 as key); + +ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS; +DESCRIBE FORMATTED tab2 key; + +DROP TABLE tab1; +DROP TABLE tab2; diff --git a/ql/src/test/queries/clientpositive/stats_histogram_order_working.q b/ql/src/test/queries/clientpositive/stats_histogram_order_working.q new file mode 100644 index 000000000000..50d9696dad7e --- /dev/null +++ b/ql/src/test/queries/clientpositive/stats_histogram_order_working.q @@ -0,0 +1,15 @@ +set hive.stats.kll.enable=true; +set metastore.stats.fetch.bitvector=true; + +CREATE TABLE tab1 AS (SELECT 1 as key); + +set metastore.stats.fetch.kll=true; +DESCRIBE FORMATTED tab1 key; + +CREATE TABLE tab2 AS (SELECT 1 as key); + +ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS; +DESCRIBE FORMATTED tab2 key; + +DROP TABLE tab1; +DROP TABLE tab2; diff --git a/ql/src/test/results/clientpositive/llap/stats_histogram_order_broken.q.out b/ql/src/test/results/clientpositive/llap/stats_histogram_order_broken.q.out new file mode 100644 index 000000000000..985ad0f8bf1c --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/stats_histogram_order_broken.q.out @@ -0,0 +1,91 @@ +PREHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@tab1 +POSTHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab1 +POSTHOOK: Lineage: tab1.key SIMPLE [] +PREHOOK: query: DESCRIBE FORMATTED tab1 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab1 +POSTHOOK: query: DESCRIBE FORMATTED tab1 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab1 +col_name key +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} +PREHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@tab2 +POSTHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab2 +POSTHOOK: Lineage: tab2.key SIMPLE [] +PREHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@tab2 +PREHOOK: Output: default@tab2 +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@tab2 +POSTHOOK: Output: default@tab2 +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE FORMATTED tab2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab2 +POSTHOOK: query: DESCRIBE FORMATTED tab2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab2 +col_name key +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +histogram Q1: 1, Q2: 1, Q3: 1 +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} +PREHOOK: query: DROP TABLE tab1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tab1 +PREHOOK: Output: database:default +PREHOOK: Output: default@tab1 +POSTHOOK: query: DROP TABLE tab1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tab1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab1 +PREHOOK: query: DROP TABLE tab2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tab2 +PREHOOK: Output: database:default +PREHOOK: Output: default@tab2 +POSTHOOK: query: DROP TABLE tab2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tab2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab2 diff --git a/ql/src/test/results/clientpositive/llap/stats_histogram_order_working.q.out b/ql/src/test/results/clientpositive/llap/stats_histogram_order_working.q.out new file mode 100644 index 000000000000..98fb3cbfd6ce --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/stats_histogram_order_working.q.out @@ -0,0 +1,92 @@ +PREHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@tab1 +POSTHOOK: query: CREATE TABLE tab1 AS (SELECT 1 as key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab1 +POSTHOOK: Lineage: tab1.key SIMPLE [] +PREHOOK: query: DESCRIBE FORMATTED tab1 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab1 +POSTHOOK: query: DESCRIBE FORMATTED tab1 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab1 +col_name key +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +histogram Q1: 1, Q2: 1, Q3: 1 +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} +PREHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: database:default +PREHOOK: Output: default@tab2 +POSTHOOK: query: CREATE TABLE tab2 AS (SELECT 1 as key) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab2 +POSTHOOK: Lineage: tab2.key SIMPLE [] +PREHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@tab2 +PREHOOK: Output: default@tab2 +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE tab2 COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@tab2 +POSTHOOK: Output: default@tab2 +#### A masked pattern was here #### +PREHOOK: query: DESCRIBE FORMATTED tab2 key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab2 +POSTHOOK: query: DESCRIBE FORMATTED tab2 key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab2 +col_name key +data_type int +min 1 +max 1 +num_nulls 0 +distinct_count 1 +avg_col_len +max_col_len +num_trues +num_falses +bit_vector HL +histogram Q1: 1, Q2: 1, Q3: 1 +comment from deserializer +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\"}} +PREHOOK: query: DROP TABLE tab1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tab1 +PREHOOK: Output: database:default +PREHOOK: Output: default@tab1 +POSTHOOK: query: DROP TABLE tab1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tab1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab1 +PREHOOK: query: DROP TABLE tab2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@tab2 +PREHOOK: Output: database:default +PREHOOK: Output: default@tab2 +POSTHOOK: query: DROP TABLE tab2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@tab2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab2 diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 5b7b174c26e7..6604b2b647b1 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -73,6 +73,15 @@ default boolean isLocalMetaStore() { return false; } + /** + * Rebind embedded metastore handler configuration after session-level SET. + * Default no-op; only embedded clients override this. + * Remote metastore clients must not perform any work here. + */ + @InterfaceAudience.Private + default void syncEmbeddedHandlerConf(Configuration conf) { + } + /** * Tries to reconnect this MetaStoreClient to the MetaStore. */ diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java index f8f3c9ab8459..f05adef31f1b 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java @@ -60,6 +60,11 @@ public boolean isLocalMetaStore() { return delegate.isLocalMetaStore(); } + @Override + public void syncEmbeddedHandlerConf(Configuration conf) { + delegate.syncEmbeddedHandlerConf(conf); + } + @Override public void reconnect() throws MetaException { delegate.reconnect(); diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java index 4cce91d84b19..a0a079ca4238 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; import org.apache.hadoop.hive.metastore.MetaStorePlainSaslHelper; import org.apache.hadoop.hive.metastore.PartitionDropOptions; @@ -432,6 +433,20 @@ public boolean isLocalMetaStore() { return localMetaStore; } + @Override + public void syncEmbeddedHandlerConf(Configuration conf) { + if (!localMetaStore || client == null) { + return; + } + try { + java.lang.reflect.Method setConfMethod = + client.getClass().getMethod("setConf", Configuration.class); + setConfMethod.invoke(client, conf); + } catch (ReflectiveOperationException e) { + LOG.warn("Failed to sync embedded metastore handler configuration", e); + } + } + @Override public boolean isCompatibleWith(Configuration conf) { // Make a copy of currentMetaVars, there is a race condition that