From f224c688e23f26f1d44fca784cb2330c5edada62 Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 18 Jun 2026 22:15:18 +0800 Subject: [PATCH 01/22] more to come about shrinking the ObjectStore --- .../hadoop/hive/metastore/ObjectStore.java | 2749 +---------------- .../hadoop/hive/metastore/RawStore.java | 13 +- .../metastore/iface/ColStatsStore.java | 50 + .../metastore/iface/ConstraintStore.java | 8 + .../metastore/impl/ColStatsStoreImpl.java | 1321 ++++++++ .../metastore/impl/ConstraintStoreImpl.java | 1399 +++++++++ .../metastore/impl/TableStoreImpl.java | 76 +- 7 files changed, 2886 insertions(+), 2730 deletions(-) create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 17ad48698ffb..4a130d81bdf3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -22,15 +22,10 @@ import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPACTOR_USE_CUSTOM_POOL; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers; import java.io.IOException; import java.net.InetAddress; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; -import java.sql.Statement; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -50,11 +45,8 @@ import java.util.Set; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -64,48 +56,29 @@ import javax.jdo.PersistenceManager; import javax.jdo.Query; import javax.jdo.Transaction; -import javax.jdo.datastore.JDOConnection; import javax.jdo.identity.IntIdentity; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; -import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql.SqlFilterForPushdown; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AllTableConstraintsRequest; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.AddPackageRequest; -import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; import org.apache.hadoop.hive.metastore.api.DropPackageRequest; import org.apache.hadoop.hive.metastore.api.DatabaseType; import org.apache.hadoop.hive.metastore.api.DataConnector; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; import org.apache.hadoop.hive.metastore.api.GetPackageRequest; -import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; -import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec; import org.apache.hadoop.hive.metastore.api.ISchema; import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.InvalidInputException; @@ -115,14 +88,10 @@ import org.apache.hadoop.hive.metastore.api.ListStoredProcedureRequest; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Package; import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PartitionFilterMode; -import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest; import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; import org.apache.hadoop.hive.metastore.api.QueryState; import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; @@ -130,13 +99,6 @@ import org.apache.hadoop.hive.metastore.api.ReplicationMetricList; import org.apache.hadoop.hive.metastore.api.GetReplicationMetricsRequest; import org.apache.hadoop.hive.metastore.api.ReplicationMetrics; -import org.apache.hadoop.hive.metastore.api.SQLAllTableConstraints; -import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; -import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.ScheduledQuery; import org.apache.hadoop.hive.metastore.api.ScheduledQueryKey; import org.apache.hadoop.hive.metastore.api.ScheduledQueryMaintenanceRequest; @@ -158,7 +120,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableParamsUpdate; import org.apache.hadoop.hive.metastore.api.Type; -import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import org.apache.hadoop.hive.metastore.api.WMMapping; import org.apache.hadoop.hive.metastore.api.WMNullablePool; @@ -178,7 +139,6 @@ import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MCatalog; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; -import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MCreationMetadata; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDataConnector; @@ -194,7 +154,6 @@ import org.apache.hadoop.hive.metastore.model.MOrder; import org.apache.hadoop.hive.metastore.model.MPackage; import org.apache.hadoop.hive.metastore.model.MPartition; -import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; import org.apache.hadoop.hive.metastore.model.MPartitionEvent; import org.apache.hadoop.hive.metastore.model.MResourceUri; import org.apache.hadoop.hive.metastore.model.MRuntimeStat; @@ -206,8 +165,6 @@ import org.apache.hadoop.hive.metastore.model.MStoredProc; import org.apache.hadoop.hive.metastore.model.MStringList; import org.apache.hadoop.hive.metastore.model.MTable; -import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; -import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; import org.apache.hadoop.hive.metastore.model.MVersionTable; import org.apache.hadoop.hive.metastore.model.MWMMapping; @@ -228,7 +185,6 @@ import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.RetryingExecutor; import org.datanucleus.ExecutionContext; import org.datanucleus.api.jdo.JDOPersistenceManager; import org.datanucleus.api.jdo.JDOTransaction; @@ -244,13 +200,10 @@ import com.cronutils.model.time.ExecutionTime; import com.cronutils.parser.CronParser; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; - /** * This class is the interface between the application logic and the database * store that contains the objects. Refrain putting any logic in mode.M* objects @@ -1321,202 +1274,6 @@ public boolean dropType(String typeName) { return success; } - @Override - public SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) - throws InvalidObjectException, MetaException { - boolean success = false; - try { - openTransaction(); - createTable(tbl); - // Add constraints. - // We need not do a deep retrieval of the Table Column Descriptor while persisting the - // constraints since this transaction involving create table is not yet committed. - if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) { - constraints.setForeignKeys(addForeignKeys(constraints.getForeignKeys(), false, constraints.getPrimaryKeys(), - constraints.getUniqueConstraints())); - } - if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) { - constraints.setPrimaryKeys(addPrimaryKeys(constraints.getPrimaryKeys(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) { - constraints.setUniqueConstraints(addUniqueConstraints(constraints.getUniqueConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) { - constraints.setNotNullConstraints(addNotNullConstraints(constraints.getNotNullConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) { - constraints.setDefaultConstraints(addDefaultConstraints(constraints.getDefaultConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) { - constraints.setCheckConstraints(addCheckConstraints(constraints.getCheckConstraints(), false)); - } - success = commitTransaction(); - return constraints; - } finally { - rollbackAndCleanup(success, null); - } - } - - /** - * Convert PrivilegeGrantInfo from privMap to MTablePrivilege, and add all of - * them to the toPersistPrivObjs. These privilege objects will be persisted as - * part of createTable. - */ - public static void putPersistentPrivObjects(MTable mtbl, List toPersistPrivObjs, - int now, Map> privMap, PrincipalType type, String authorizer) { - if (privMap != null) { - for (Map.Entry> entry : privMap - .entrySet()) { - String principalName = entry.getKey(); - List privs = entry.getValue(); - for (int i = 0; i < privs.size(); i++) { - PrivilegeGrantInfo priv = privs.get(i); - if (priv == null) { - continue; - } - MTablePrivilege mTblSec = new MTablePrivilege( - principalName, type.toString(), mtbl, priv.getPrivilege(), - now, priv.getGrantor(), priv.getGrantorType().toString(), priv - .isGrantOption(), authorizer); - toPersistPrivObjs.add(mTblSec); - } - } - } - } - - private List listAllTableConstraintsWithOptionalConstraintName( - String catName, String dbName, String tableName, String constraintname) { - catName = normalizeIdentifier(catName); - dbName = normalizeIdentifier(dbName); - tableName = normalizeIdentifier(tableName); - constraintname = constraintname!=null?normalizeIdentifier(constraintname):null; - List mConstraints = null; - List constraintNames = new ArrayList<>(); - - try (QueryWrapper queryForConstraintName = new QueryWrapper(pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where " - + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname && " + - "parentTable.database.catalogName == pcatname) || " - + "(childTable != null && childTable.tableName == ctblname &&" + - "childTable.database.name == cdbname && childTable.database.catalogName == ccatname)) " + - (constraintname != null ? " && constraintName == constraintname" : ""))); - QueryWrapper queryForMConstraint = new QueryWrapper(pm.newQuery(MConstraint.class))) { - - queryForConstraintName.declareParameters("java.lang.String ptblname, java.lang.String pdbname," - + "java.lang.String pcatname, java.lang.String ctblname, java.lang.String cdbname," + - "java.lang.String ccatname" + - (constraintname != null ? ", java.lang.String constraintname" : "")); - Collection constraintNamesColl = - constraintname != null ? - ((Collection) queryForConstraintName. - executeWithArray(tableName, dbName, catName, tableName, dbName, catName, constraintname)): - ((Collection) queryForConstraintName. - executeWithArray(tableName, dbName, catName, tableName, dbName, catName)); - for (Iterator i = constraintNamesColl.iterator(); i.hasNext();) { - String currName = (String) i.next(); - constraintNames.add(currName); - } - - queryForMConstraint.setFilter("param.contains(constraintName)"); - queryForMConstraint.declareParameters("java.util.Collection param"); - Collection constraints = (Collection)queryForMConstraint.execute(constraintNames); - mConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - mConstraints.add(currConstraint); - } - } - return mConstraints; - } - - @Override - public List getTableNamesWithStats() throws MetaException, NoSuchObjectException { - return new GetListHelper(null, null, null, true, false) { - @Override - protected List getSqlResult( - GetHelper> ctx) throws MetaException { - return directSql.getTableNamesWithStats(); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) throws MetaException { - throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? - } - }.run(false); - } - - @Override - public Map> getPartitionColsWithStats(String catName, String dbName, String tableName) - throws MetaException, NoSuchObjectException { - return new GetHelper>>(catName, dbName, null, true, false) { - @Override - protected Map> getSqlResult( - GetHelper>> ctx) throws MetaException { - try { - return directSql.getColAndPartNamesWithStats(catName, dbName, tableName); - } catch (Throwable ex) { - LOG.error("DirectSQL failed", ex); - throw new MetaException(ex.getMessage()); - } - } - - @Override - protected Map> getJdoResult( - GetHelper>> ctx) throws MetaException { - throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? - } - - @Override - protected String describeResult() { - return results.size() + " partitions"; - } - }.run(false); - } - - @Override - public List getAllTableNamesForStats() throws MetaException, NoSuchObjectException { - return new GetListHelper(null, null, null, true, false) { - @Override - protected List getSqlResult( - GetHelper> ctx) throws MetaException { - return directSql.getAllTableNamesForStats(); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) throws MetaException { - boolean commited = false; - Query query = null; - List result = new ArrayList<>(); - openTransaction(); - try { - String paramStr = "", whereStr = ""; - for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) { - if (i != 0) { - paramStr += ", "; - whereStr += "||"; - } - paramStr += "java.lang.String tt" + i; - whereStr += " tableType == tt" + i; - } - query = pm.newQuery(MTable.class, whereStr); - query.declareParameters(paramStr); - Collection tbls = (Collection) query.executeWithArray( - query, MetaStoreDirectSql.STATS_TABLE_TYPES); - pm.retrieveAll(tbls); - for (MTable tbl : tbls) { - result.add(new TableName( - tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName())); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return result; - } - }.run(false); - } - public static StringBuilder appendPatternCondition(StringBuilder builder, String fieldName, String elements, List parameters) { elements = normalizeIdentifier(elements); @@ -1554,59 +1311,6 @@ private static StringBuilder appendCondition(StringBuilder builder, return builder; } - class AttachedMTableInfo { - MTable mtbl; - MColumnDescriptor mcd; - - public AttachedMTableInfo() {} - - public AttachedMTableInfo(MTable mtbl, MColumnDescriptor mcd) { - this.mtbl = mtbl; - this.mcd = mcd; - } - } - - private AttachedMTableInfo getMTable(String catName, String db, String table, - boolean retrieveCD) { - AttachedMTableInfo nmtbl = new AttachedMTableInfo(); - MTable mtbl = null; - boolean commited = false; - Query query = null; - try { - openTransaction(); - catName = normalizeIdentifier(Optional.ofNullable(catName).orElse(getDefaultCatalog(conf))); - db = normalizeIdentifier(db); - table = normalizeIdentifier(table); - query = pm.newQuery(MTable.class, - "tableName == table && database.name == db && database.catalogName == catname"); - query.declareParameters( - "java.lang.String table, java.lang.String db, java.lang.String catname"); - query.setUnique(true); - if (LOG.isDebugEnabled()) { - LOG.debug("Executing getMTable for {}", - TableName.getQualified(catName, db, table)); - } - mtbl = (MTable) query.execute(table, db, catName); - pm.retrieve(mtbl); - // Retrieving CD can be expensive and unnecessary, so do it only when required. - if (mtbl != null && retrieveCD) { - pm.retrieve(mtbl.getSd()); - pm.retrieveAll(mtbl.getSd().getCD()); - nmtbl.mcd = mtbl.getSd().getCD(); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - nmtbl.mtbl = mtbl; - return nmtbl; - } - - private MTable getMTable(String catName, String db, String table) { - AttachedMTableInfo nmtbl = getMTable(catName, db, table, false); - return nmtbl.mtbl; - } - /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */ private static List convertList(List dnList) { return (dnList == null) ? null : Lists.newArrayList(dnList); @@ -2132,49 +1836,6 @@ public static List convertToParts(String catName, String dbName, Stri return parts; } - public static Pair> getPartQueryWithParams( - PersistenceManager pm, - String catName, String dbName, String tblName, - List partNames) { - Query query = pm.newQuery(); - Map params = new HashMap<>(); - String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); - query.setFilter(filterStr); - LOG.debug(" JDOQL filter is {}", filterStr); - query.declareParameters(makeParameterDeclarationString(params)); - return Pair.of(query, params); - } - - public static String getJDOFilterStrForPartitionNames(String catName, String dbName, String tblName, - List partNames, Map params) { - StringBuilder sb = new StringBuilder( - "table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); - params.put("t1", normalizeIdentifier(tblName)); - params.put("t2", normalizeIdentifier(dbName)); - params.put("t3", normalizeIdentifier(catName)); - int n = 0; - for (Iterator itr = partNames.iterator(); itr.hasNext(); ) { - String pn = "p" + n; - n++; - String part = itr.next(); - params.put(pn, part); - sb.append("partitionName == ").append(pn); - sb.append(" || "); - } - sb.setLength(sb.length() - 4); // remove the last " || " - sb.append(')'); - return sb.toString(); - } - - public static String makeParameterDeclarationString(Map params) { - //Create the parameter declaration string - StringBuilder paramDecl = new StringBuilder(); - for (String key : params.keySet()) { - paramDecl.append(", java.lang.String ").append(key); - } - return paramDecl.toString(); - } - /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */ @VisibleForTesting public abstract class GetHelper { @@ -2407,18 +2068,6 @@ protected String describeResult() { } } - private abstract class GetStatHelper extends GetHelper { - public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql, - boolean allowJdo, String writeIdList) throws MetaException { - super(catalogName, dbName, tblName, allowSql, allowJdo); - } - - @Override - protected String describeResult() { - return "statistics for " + (results == null ? 0 : results.getStatsObjSize()) + " columns"; - } - } - private Table ensureGetTable(String catName, String dbName, String tblName) throws NoSuchObjectException, MetaException { return convertToTable(ensureGetMTable(catName, dbName, tblName), conf); @@ -2455,71 +2104,6 @@ public static String verifyStatsChangeCtx(String fullTableName, Map cols, String col) { - if (cols == null) { - return null; - } - for (MFieldSchema mfs : cols) { - if (mfs.getName().equalsIgnoreCase(col)) { - return mfs; - } - } - return null; - } - - private static int getColumnIndexFromTableColumns(List cols, String col) { - if (cols == null) { - return -1; - } - for (int i = 0; i < cols.size(); i++) { - MFieldSchema mfs = cols.get(i); - if (mfs.getName().equalsIgnoreCase(col)) { - return i; - } - } - return -1; - } - - private boolean constraintNameAlreadyExists(MTable table, String constraintName) { - boolean commited = false; - Query constraintExistsQuery = null; - String constraintNameIfExists = null; - try { - openTransaction(); - constraintName = normalizeIdentifier(constraintName); - constraintExistsQuery = pm.newQuery(MConstraint.class, - "parentTable == parentTableP && constraintName == constraintNameP"); - constraintExistsQuery.declareParameters("MTable parentTableP, java.lang.String constraintNameP"); - constraintExistsQuery.setUnique(true); - constraintExistsQuery.setResult("constraintName"); - constraintNameIfExists = (String) constraintExistsQuery.executeWithArray(table, constraintName); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, constraintExistsQuery); - } - return constraintNameIfExists != null && !constraintNameIfExists.isEmpty(); - } - - private String generateConstraintName(MTable table, String... parameters) throws MetaException { - int hashcode = ArrayUtils.toString(parameters).hashCode() & 0xfffffff; - int counter = 0; - final int MAX_RETRIES = 10; - while (counter < MAX_RETRIES) { - String currName = (parameters.length == 0 ? "constraint_" : parameters[parameters.length-1]) + - "_" + hashcode + "_" + System.currentTimeMillis() + "_" + (counter++); - if (!constraintNameAlreadyExists(table, currName)) { - return currName; - } - } - throw new MetaException("Error while trying to generate the constraint name for " + ArrayUtils.toString(parameters)); - } - - @Override - public List addForeignKeys( - List fks) throws InvalidObjectException, MetaException { - return addForeignKeys(fks, true, null, null); - } - @Override public String getMetastoreDbUuid() throws MetaException { String ret = getGuidFromDB(); @@ -2736,1677 +2320,41 @@ public T fetchProperties(String key, java.util.function.Function Map selectProperties(String key, java.util.function.Function transform) { - boolean success = false; - Query query = null; - Map results = null; - try { - if (openTransaction()) { - Collection properties; - if (key == null || key.isEmpty()) { - query = pm.newQuery(MMetastoreDBProperties.class); - properties = (Collection) query.execute(); - } else { - query = pm.newQuery(MMetastoreDBProperties.class, "this.propertyKey.startsWith(key)"); - query.declareParameters(PTYPARAM_STR_KEY); - properties = (Collection) query.execute(key); - } - pm.retrieveAll(properties); - if (!properties.isEmpty()) { - results = new TreeMap(); - for(MMetastoreDBProperties ptys : properties) { - T t = (T) (transform != null? transform.apply(ptys) : ptys); - if (t != null) { - results.put(ptys.getPropertykey(), t); - } - } - } - success = commitTransaction(); - } - } finally { - rollbackAndCleanup(success, query); - } - return results; - } - - //TODO: clean up this method - private List addForeignKeys(List foreignKeys, boolean retrieveCD, - List primaryKeys, List uniqueConstraints) - throws InvalidObjectException, MetaException { - if (CollectionUtils.isNotEmpty(foreignKeys)) { - List mpkfks = new ArrayList<>(); - String currentConstraintName = null; - String catName = null; - // We start iterating through the foreign keys. This list might contain more than a single - // foreign key, and each foreign key might contain multiple columns. The outer loop retrieves - // the information that is common for a single key (table information) while the inner loop - // checks / adds information about each column. - for (int i = 0; i < foreignKeys.size(); i++) { - if (catName == null) { - catName = normalizeIdentifier(foreignKeys.get(i).isSetCatName() ? foreignKeys.get(i).getCatName() : - getDefaultCatalog(conf)); - } else { - String tmpCatName = normalizeIdentifier(foreignKeys.get(i).isSetCatName() ? - foreignKeys.get(i).getCatName() : getDefaultCatalog(conf)); - if (!catName.equals(tmpCatName)) { - throw new InvalidObjectException("Foreign keys cannot span catalogs"); - } - } - final String fkTableDB = normalizeIdentifier(foreignKeys.get(i).getFktable_db()); - final String fkTableName = normalizeIdentifier(foreignKeys.get(i).getFktable_name()); - // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. - // For instance, this is the case when we are creating the table. - final AttachedMTableInfo nChildTable = getMTable(catName, fkTableDB, fkTableName, retrieveCD); - final MTable childTable = nChildTable.mtbl; - if (childTable == null) { - throw new InvalidObjectException("Child table not found: " + fkTableName); - } - MColumnDescriptor childCD = retrieveCD ? nChildTable.mcd : childTable.getSd().getCD(); - final List childCols = childCD == null || childCD.getCols() == null ? - new ArrayList<>() : new ArrayList<>(childCD.getCols()); - if (childTable.getPartitionKeys() != null) { - childCols.addAll(childTable.getPartitionKeys()); - } - - final String pkTableDB = normalizeIdentifier(foreignKeys.get(i).getPktable_db()); - final String pkTableName = normalizeIdentifier(foreignKeys.get(i).getPktable_name()); - // For primary keys, we retrieve the column descriptors if retrieveCD is true (which means - // it is an alter table statement) or if it is a create table statement but we are - // referencing another table instead of self for the primary key. - final AttachedMTableInfo nParentTable; - final MTable parentTable; - MColumnDescriptor parentCD; - final List parentCols; - final List existingTablePrimaryKeys; - final List existingTableUniqueConstraints; - final boolean sameTable = fkTableDB.equals(pkTableDB) && fkTableName.equals(pkTableName); - if (sameTable) { - nParentTable = nChildTable; - parentTable = childTable; - parentCD = childCD; - parentCols = childCols; - existingTablePrimaryKeys = primaryKeys; - existingTableUniqueConstraints = uniqueConstraints; - } else { - nParentTable = getMTable(catName, pkTableDB, pkTableName, true); - parentTable = nParentTable.mtbl; - if (parentTable == null) { - throw new InvalidObjectException("Parent table not found: " + pkTableName); - } - parentCD = nParentTable.mcd; - parentCols = parentCD == null || parentCD.getCols() == null ? - new ArrayList<>() : new ArrayList<>(parentCD.getCols()); - if (parentTable.getPartitionKeys() != null) { - parentCols.addAll(parentTable.getPartitionKeys()); - } - PrimaryKeysRequest primaryKeysRequest = new PrimaryKeysRequest(pkTableDB, pkTableName); - primaryKeysRequest.setCatName(catName); - existingTablePrimaryKeys = getPrimaryKeys(primaryKeysRequest); - existingTableUniqueConstraints = - getUniqueConstraints(new UniqueConstraintsRequest(catName, pkTableDB, pkTableName)); - } - - // Here we build an aux structure that is used to verify that the foreign key that is declared - // is actually referencing a valid primary key or unique key. We also check that the types of - // the columns correspond. - if (existingTablePrimaryKeys.isEmpty() && existingTableUniqueConstraints.isEmpty()) { - throw new MetaException( - "Trying to define foreign key but there are no primary keys or unique keys for referenced table"); - } - final Set validPKsOrUnique = generateValidPKsOrUniqueSignatures(parentCols, - existingTablePrimaryKeys, existingTableUniqueConstraints); - - StringBuilder fkSignature = new StringBuilder(); - StringBuilder referencedKSignature = new StringBuilder(); - for (; i < foreignKeys.size(); i++) { - SQLForeignKey foreignKey = foreignKeys.get(i); - final String fkColumnName = normalizeIdentifier(foreignKey.getFkcolumn_name()); - int childIntegerIndex = getColumnIndexFromTableColumns(childCD.getCols(), fkColumnName); - if (childIntegerIndex == -1) { - if (childTable.getPartitionKeys() != null) { - childCD = null; - childIntegerIndex = getColumnIndexFromTableColumns(childTable.getPartitionKeys(), fkColumnName); - } - if (childIntegerIndex == -1) { - throw new InvalidObjectException("Child column not found: " + fkColumnName); - } - } - - final String pkColumnName = normalizeIdentifier(foreignKey.getPkcolumn_name()); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD.getCols(), pkColumnName); - if (parentIntegerIndex == -1) { - if (parentTable.getPartitionKeys() != null) { - parentCD = null; - parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), pkColumnName); - } - if (parentIntegerIndex == -1) { - throw new InvalidObjectException("Parent column not found: " + pkColumnName); - } - } - - if (foreignKey.getFk_name() == null) { - // When there is no explicit foreign key name associated with the constraint and the key is composite, - // we expect the foreign keys to be send in order in the input list. - // Otherwise, the below code will break. - // If this is the first column of the FK constraint, generate the foreign key name - // NB: The below code can result in race condition where duplicate names can be generated (in theory). - // However, this scenario can be ignored for practical purposes because of - // the uniqueness of the generated constraint name. - if (foreignKey.getKey_seq() == 1) { - currentConstraintName = generateConstraintName(parentTable, fkTableDB, fkTableName, pkTableDB, - pkTableName, pkColumnName, fkColumnName, "fk"); - } - } else { - currentConstraintName = normalizeIdentifier(foreignKey.getFk_name()); - if (constraintNameAlreadyExists(parentTable, currentConstraintName)) { - String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), - parentTable.getTableName(), currentConstraintName); - throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); - } - } - // Update Column, keys, table, database, catalog name - foreignKey.setFk_name(currentConstraintName); - foreignKey.setCatName(catName); - foreignKey.setFktable_db(fkTableDB); - foreignKey.setFktable_name(fkTableName); - foreignKey.setPktable_db(pkTableDB); - foreignKey.setPktable_name(pkTableName); - foreignKey.setFkcolumn_name(fkColumnName); - foreignKey.setPkcolumn_name(pkColumnName); - - Integer updateRule = foreignKey.getUpdate_rule(); - Integer deleteRule = foreignKey.getDelete_rule(); - int enableValidateRely = (foreignKey.isEnable_cstr() ? 4 : 0) + - (foreignKey.isValidate_cstr() ? 2 : 0) + (foreignKey.isRely_cstr() ? 1 : 0); - - MConstraint mpkfk = new MConstraint( - currentConstraintName, - foreignKey.getKey_seq(), - MConstraint.FOREIGN_KEY_CONSTRAINT, - deleteRule, - updateRule, - enableValidateRely, - parentTable, - childTable, - parentCD, - childCD, - childIntegerIndex, - parentIntegerIndex - ); - mpkfks.add(mpkfk); - - final String fkColType = getColumnFromTableColumns(childCols, fkColumnName).getType(); - fkSignature.append( - generateColNameTypeSignature(fkColumnName, fkColType)); - referencedKSignature.append( - generateColNameTypeSignature(pkColumnName, fkColType)); - - if (i + 1 < foreignKeys.size() && foreignKeys.get(i + 1).getKey_seq() == 1) { - // Next one is a new key, we bail out from the inner loop - break; - } - } - String referenced = referencedKSignature.toString(); - if (!validPKsOrUnique.contains(referenced)) { - throw new MetaException( - "Foreign key references " + referenced + " but no corresponding " - + "primary key or unique key exists. Possible keys: " + validPKsOrUnique); - } - if (sameTable && fkSignature.toString().equals(referenced)) { - throw new MetaException( - "Cannot be both foreign key and primary/unique key on same table: " + referenced); - } - fkSignature = new StringBuilder(); - referencedKSignature = new StringBuilder(); - } - pm.makePersistentAll(mpkfks); - - } - return foreignKeys; - } - - private static Set generateValidPKsOrUniqueSignatures(List tableCols, - List refTablePrimaryKeys, List refTableUniqueConstraints) { - final Set validPKsOrUnique = new HashSet<>(); - if (!refTablePrimaryKeys.isEmpty()) { - refTablePrimaryKeys.sort((o1, o2) -> { - int keyNameComp = o1.getPk_name().compareTo(o2.getPk_name()); - if (keyNameComp == 0) { - return Integer.compare(o1.getKey_seq(), o2.getKey_seq()); - } - return keyNameComp; - }); - StringBuilder pkSignature = new StringBuilder(); - for (SQLPrimaryKey pk : refTablePrimaryKeys) { - pkSignature.append( - generateColNameTypeSignature( - pk.getColumn_name(), getColumnFromTableColumns(tableCols, pk.getColumn_name()).getType())); - } - validPKsOrUnique.add(pkSignature.toString()); - } - if (!refTableUniqueConstraints.isEmpty()) { - refTableUniqueConstraints.sort((o1, o2) -> { - int keyNameComp = o1.getUk_name().compareTo(o2.getUk_name()); - if (keyNameComp == 0) { - return Integer.compare(o1.getKey_seq(), o2.getKey_seq()); - } - return keyNameComp; - }); - StringBuilder ukSignature = new StringBuilder(); - for (int j = 0; j < refTableUniqueConstraints.size(); j++) { - SQLUniqueConstraint uk = refTableUniqueConstraints.get(j); - ukSignature.append( - generateColNameTypeSignature( - uk.getColumn_name(), getColumnFromTableColumns(tableCols, uk.getColumn_name()).getType())); - if (j + 1 < refTableUniqueConstraints.size()) { - if (!refTableUniqueConstraints.get(j + 1).getUk_name().equals( - refTableUniqueConstraints.get(j).getUk_name())) { - validPKsOrUnique.add(ukSignature.toString()); - ukSignature = new StringBuilder(); - } - } else { - validPKsOrUnique.add(ukSignature.toString()); - } - } - } - return validPKsOrUnique; - } - - private static String generateColNameTypeSignature(String colName, String colType) { - return colName + ":" + colType + ";"; - } - - @Override - public List addPrimaryKeys(List pks) throws InvalidObjectException, - MetaException { - return addPrimaryKeys(pks, true); - } - - private List addPrimaryKeys(List pks, boolean retrieveCD) throws InvalidObjectException, - MetaException { - List mpks = new ArrayList<>(); - String constraintName = null; - - for (SQLPrimaryKey pk : pks) { - final String catName = normalizeIdentifier(pk.getCatName()); - final String tableDB = normalizeIdentifier(pk.getTable_db()); - final String tableName = normalizeIdentifier(pk.getTable_name()); - final String columnName = normalizeIdentifier(pk.getColumn_name()); - - // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. - // For instance, this is the case when we are creating the table. - AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); - MTable parentTable = nParentTable.mtbl; - if (parentTable == null) { - throw new InvalidObjectException("Parent table not found: " + tableName); - } - - MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); - if (parentIntegerIndex == -1) { - if (parentTable.getPartitionKeys() != null) { - parentCD = null; - parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); - } - if (parentIntegerIndex == -1) { - throw new InvalidObjectException("Parent column not found: " + columnName); - } - } - if (getPrimaryKeyConstraintName(parentTable.getDatabase().getCatalogName(), - parentTable.getDatabase().getName(), parentTable.getTableName()) != null) { - throw new MetaException(" Primary key already exists for: " + - TableName.getQualified(catName, tableDB, tableName)); - } - if (pk.getPk_name() == null) { - if (pk.getKey_seq() == 1) { - constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "pk"); - } - } else { - constraintName = normalizeIdentifier(pk.getPk_name()); - if (constraintNameAlreadyExists(parentTable, constraintName)) { - String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), - parentTable.getTableName(), constraintName); - throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); - } - } - - int enableValidateRely = (pk.isEnable_cstr() ? 4 : 0) + - (pk.isValidate_cstr() ? 2 : 0) + (pk.isRely_cstr() ? 1 : 0); - MConstraint mpk = new MConstraint( - constraintName, - pk.getKey_seq(), - MConstraint.PRIMARY_KEY_CONSTRAINT, - null, - null, - enableValidateRely, - parentTable, - null, - parentCD, - null, - null, - parentIntegerIndex); - mpks.add(mpk); - - // Add normalized identifier back to result - pk.setCatName(catName); - pk.setTable_db(tableDB); - pk.setTable_name(tableName); - pk.setColumn_name(columnName); - pk.setPk_name(constraintName); - } - pm.makePersistentAll(mpks); - return pks; - } - - @Override - public List addUniqueConstraints(List uks) - throws InvalidObjectException, MetaException { - return addUniqueConstraints(uks, true); - } - - private List addUniqueConstraints(List uks, boolean retrieveCD) - throws InvalidObjectException, MetaException { - - List cstrs = new ArrayList<>(); - String constraintName = null; - - for (SQLUniqueConstraint uk : uks) { - final String catName = normalizeIdentifier(uk.getCatName()); - final String tableDB = normalizeIdentifier(uk.getTable_db()); - final String tableName = normalizeIdentifier(uk.getTable_name()); - final String columnName = normalizeIdentifier(uk.getColumn_name()); - - // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. - // For instance, this is the case when we are creating the table. - AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); - MTable parentTable = nParentTable.mtbl; - if (parentTable == null) { - throw new InvalidObjectException("Parent table not found: " + tableName); - } - - MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); - if (parentIntegerIndex == -1) { - if (parentTable.getPartitionKeys() != null) { - parentCD = null; - parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); - } - if (parentIntegerIndex == -1) { - throw new InvalidObjectException("Parent column not found: " + columnName); - } - } - if (uk.getUk_name() == null) { - if (uk.getKey_seq() == 1) { - constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "uk"); - } - } else { - constraintName = normalizeIdentifier(uk.getUk_name()); - if (constraintNameAlreadyExists(parentTable, constraintName)) { - String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), - parentTable.getTableName(), constraintName); - throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); - } - } - - - int enableValidateRely = (uk.isEnable_cstr() ? 4 : 0) + - (uk.isValidate_cstr() ? 2 : 0) + (uk.isRely_cstr() ? 1 : 0); - MConstraint muk = new MConstraint( - constraintName, - uk.getKey_seq(), - MConstraint.UNIQUE_CONSTRAINT, - null, - null, - enableValidateRely, - parentTable, - null, - parentCD, - null, - null, - parentIntegerIndex); - cstrs.add(muk); - - // Add normalized identifier back to result - uk.setCatName(catName); - uk.setTable_db(tableDB); - uk.setTable_name(tableName); - uk.setColumn_name(columnName); - uk.setUk_name(constraintName); - - } - pm.makePersistentAll(cstrs); - return uks; - } - - @Override - public List addNotNullConstraints(List nns) - throws InvalidObjectException, MetaException { - return addNotNullConstraints(nns, true); - } - - @Override - public List addDefaultConstraints(List nns) - throws InvalidObjectException, MetaException { - return addDefaultConstraints(nns, true); - } - - @Override - public List addCheckConstraints(List nns) - throws InvalidObjectException, MetaException { - return addCheckConstraints(nns, true); - } - - private List addCheckConstraints(List ccs, boolean retrieveCD) - throws InvalidObjectException, MetaException { - List cstrs = new ArrayList<>(); - - for (SQLCheckConstraint cc: ccs) { - final String catName = normalizeIdentifier(cc.getCatName()); - final String tableDB = normalizeIdentifier(cc.getTable_db()); - final String tableName = normalizeIdentifier(cc.getTable_name()); - final String columnName = cc.getColumn_name() == null? null - : normalizeIdentifier(cc.getColumn_name()); - final String ccName = cc.getDc_name(); - boolean isEnable = cc.isEnable_cstr(); - boolean isValidate = cc.isValidate_cstr(); - boolean isRely = cc.isRely_cstr(); - String constraintValue = cc.getCheck_expression(); - MConstraint muk = addConstraint(catName, tableDB, tableName, columnName, ccName, isEnable, isRely, isValidate, - MConstraint.CHECK_CONSTRAINT, constraintValue, retrieveCD); - cstrs.add(muk); - - // Add normalized identifier back to result - cc.setCatName(catName); - cc.setTable_db(tableDB); - cc.setTable_name(tableName); - cc.setColumn_name(columnName); - cc.setDc_name(muk.getConstraintName()); - } - pm.makePersistentAll(cstrs); - return ccs; - } - - private MConstraint addConstraint(String catName, String tableDB, String tableName, String columnName, String ccName, - boolean isEnable, boolean isRely, boolean isValidate, int constraintType, - String constraintValue, boolean retrieveCD) - throws InvalidObjectException, MetaException { - String constraintName = null; - // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. - // For instance, this is the case when we are creating the table. - AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); - MTable parentTable = nParentTable.mtbl; - if (parentTable == null) { - throw new InvalidObjectException("Parent table not found: " + tableName); - } - - MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); - if (parentIntegerIndex == -1) { - if (parentTable.getPartitionKeys() != null) { - parentCD = null; - parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); - } - } - if (ccName == null) { - constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "dc"); - } else { - constraintName = normalizeIdentifier(ccName); - if (constraintNameAlreadyExists(parentTable, constraintName)) { - String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), - parentTable.getTableName(), constraintName); - throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); - } - } - - int enableValidateRely = (isEnable ? 4 : 0) + - (isValidate ? 2 : 0) + (isRely ? 1 : 0); - MConstraint muk = new MConstraint( - constraintName, - 1, - constraintType, // Not null constraint should reference a single column - null, - null, - enableValidateRely, - parentTable, - null, - parentCD, - null, - null, - parentIntegerIndex, - constraintValue); - - return muk; - } - - private List addDefaultConstraints(List dcs, boolean retrieveCD) - throws InvalidObjectException, MetaException { - - List cstrs = new ArrayList<>(); - for (SQLDefaultConstraint dc : dcs) { - final String catName = normalizeIdentifier(dc.getCatName()); - final String tableDB = normalizeIdentifier(dc.getTable_db()); - final String tableName = normalizeIdentifier(dc.getTable_name()); - final String columnName = normalizeIdentifier(dc.getColumn_name()); - final String dcName = dc.getDc_name(); - boolean isEnable = dc.isEnable_cstr(); - boolean isValidate = dc.isValidate_cstr(); - boolean isRely = dc.isRely_cstr(); - String constraintValue = dc.getDefault_value(); - MConstraint muk = addConstraint(catName, tableDB, tableName, columnName, dcName, isEnable, isRely, isValidate, - MConstraint.DEFAULT_CONSTRAINT, constraintValue, retrieveCD); - cstrs.add(muk); - - // Add normalized identifier back to result - dc.setCatName(catName); - dc.setTable_db(tableDB); - dc.setTable_name(tableName); - dc.setColumn_name(columnName); - dc.setDc_name(muk.getConstraintName()); - } - pm.makePersistentAll(cstrs); - return dcs; - } - - private List addNotNullConstraints(List nns, boolean retrieveCD) - throws InvalidObjectException, MetaException { - - List cstrs = new ArrayList<>(); - String constraintName; - - for (SQLNotNullConstraint nn : nns) { - final String catName = normalizeIdentifier(nn.getCatName()); - final String tableDB = normalizeIdentifier(nn.getTable_db()); - final String tableName = normalizeIdentifier(nn.getTable_name()); - final String columnName = normalizeIdentifier(nn.getColumn_name()); - - // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. - // For instance, this is the case when we are creating the table. - AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); - MTable parentTable = nParentTable.mtbl; - if (parentTable == null) { - throw new InvalidObjectException("Parent table not found: " + tableName); - } - - MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); - if (parentIntegerIndex == -1) { - if (parentTable.getPartitionKeys() != null) { - parentCD = null; - parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); - } - if (parentIntegerIndex == -1) { - throw new InvalidObjectException("Parent column not found: " + columnName); - } - } - if (nn.getNn_name() == null) { - constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "nn"); - } else { - constraintName = normalizeIdentifier(nn.getNn_name()); - if (constraintNameAlreadyExists(parentTable, constraintName)) { - String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), - parentTable.getTableName(), constraintName); - throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); - } - } - - int enableValidateRely = (nn.isEnable_cstr() ? 4 : 0) + - (nn.isValidate_cstr() ? 2 : 0) + (nn.isRely_cstr() ? 1 : 0); - MConstraint muk = new MConstraint( - constraintName, - 1, - MConstraint.NOT_NULL_CONSTRAINT, // Not null constraint should reference a single column - null, - null, - enableValidateRely, - parentTable, - null, - parentCD, - null, - null, - parentIntegerIndex); - cstrs.add(muk); - // Add normalized identifier back to result - nn.setCatName(catName); - nn.setTable_db(tableDB); - nn.setTable_name(tableName); - nn.setColumn_name(columnName); - nn.setNn_name(constraintName); - } - pm.makePersistentAll(cstrs); - return nns; - } - - private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, - MTableColumnStatistics oldStats) throws MetaException { - - Preconditions.checkState(this.currentTransaction.isActive()); - - String colName = mStatsObj.getColName(); - - LOG.info("Updating table level column statistics for table={} colName={}", - Warehouse.getCatalogQualifiedTableName(table), colName); - validateTableCols(table, Lists.newArrayList(colName)); - - if (oldStats != null) { - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); - } else { - pm.makePersistent(mStatsObj); - } - } - - private void writeMPartitionColumnStatistics(Table table, Partition partition, - MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) { - String catName = mStatsObj.getPartition().getTable().getDatabase().getCatalogName(); - String dbName = mStatsObj.getPartition().getTable().getDatabase().getName(); - String tableName = mStatsObj.getPartition().getTable().getTableName(); - String partName = mStatsObj.getPartition().getPartitionName(); - String colName = mStatsObj.getColName(); - - Preconditions.checkState(this.currentTransaction.isActive()); - - LOG.info("Updating partition level column statistics for table=" + - TableName.getQualified(catName, dbName, tableName) + - " partName=" + partName + " colName=" + colName); - - boolean foundCol = false; - List colList = partition.getSd().getCols(); - for (FieldSchema col : colList) { - if (col.getName().equals(mStatsObj.getColName())) { - foundCol = true; - break; - } - } - - if (!foundCol) { - LOG.warn("Column " + colName + " for which stats gathering is requested doesn't exist."); - } - - if (oldStats != null) { - StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); - } else { - pm.makePersistent(mStatsObj); - } - } - - @Override - public Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - boolean committed = false; - List statsObjs = colStats.getStatsObj(); - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - long start = System.currentTimeMillis(); - String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); - try { - openTransaction(); - // DataNucleus objects get detached all over the place for no (real) reason. - // So let's not use them anywhere unless absolutely necessary. - MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); - int maxRetries = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); - long sleepInterval = MetastoreConf.getTimeVar(conf, - ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); - Map result = new RetryingExecutor<>(maxRetries, () -> { - AtomicReference exceptionRef = new AtomicReference<>(); - String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); - executePlainSQL( - sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()), - true, - exception -> { - rollbackTransactionToSavePoint(savePoint); - exceptionRef.set(exception); - }); - if (exceptionRef.get() != null) { - throw new RetryingExecutor.RetryException(exceptionRef.get()); - } - pm.refresh(mTable); - Table table = convertToTable(mTable, conf); - List colNames = new ArrayList<>(); - for (ColumnStatisticsObj statsObj : statsObjs) { - colNames.add(statsObj.getColName()); - } - - Map oldStats = Maps.newHashMap(); - List stats = getMTableColumnStatistics(table, colNames, colStats.getEngine()); - for (MTableColumnStatistics cStat : stats) { - oldStats.put(cStat.getColName(), cStat); - } - - for (ColumnStatisticsObj statsObj : statsObjs) { - MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc, - statsObj, colStats.getEngine()); - writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); - // There is no need to add colname again, otherwise we will get duplicate colNames. - } - - // Set the table properties - // No need to check again if it exists. - String dbname = table.getDbName(); - String name = table.getTableName(); - MTable oldt = mTable; - Map newParams = new HashMap<>(table.getParameters()); - StatsSetupConst.setColumnStatsState(newParams, colNames); - boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters()); - if (isTxn) { - if (!areTxnStatsSupported) { - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - } else { - String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams, - writeId, validWriteIds, true); - if (errorMsg != null) { - throw new MetaException(errorMsg); - } - if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) { - // Make sure we set the flag to invalid regardless of the current value. - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name); - } - oldt.setWriteId(writeId); - } - } - oldt.setParameters(newParams); - return newParams; - }).onRetry(e -> e instanceof RetryingExecutor.RetryException) - .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval -> - ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); - committed = commitTransaction(); - return committed ? result : null; - } finally { - LOG.debug("{} updateTableColumnStatistics took {}ms, success: {}", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), - System.currentTimeMillis() - start, committed); - rollbackAndCleanup(committed, null); - } - } - - @Override - public Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats, - List partVals, String validWriteIds, long writeId) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - boolean committed = false; - long start = System.currentTimeMillis(); - List statsObjs = colStats.getStatsObj(); - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); - try { - openTransaction(); - MPartition mPartition = - ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals); - if (mPartition == null) { - throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); - } - - List colNames = new ArrayList<>(); - for(ColumnStatisticsObj statsObj : statsObjs) { - colNames.add(statsObj.getColName()); - } - int maxRetries = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); - long sleepInterval = MetastoreConf.getTimeVar(conf, - ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); - Map result = new RetryingExecutor<>(maxRetries, () -> { - AtomicReference exceptionRef = new AtomicReference<>(); - String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); - executePlainSQL(sqlGenerator.addForUpdateNoWait( - "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()), - true, - exception -> { - rollbackTransactionToSavePoint(savePoint); - exceptionRef.set(exception); - }); - if (exceptionRef.get() != null) { - throw new RetryingExecutor.RetryException(exceptionRef.get()); - } - pm.refresh(mPartition); - Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), - mPartition, TxnUtils.isAcidTable(table), conf); - Map oldStats = Maps.newHashMap(); - List stats = - getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine()); - for (MPartitionColumnStatistics cStat : stats) { - oldStats.put(cStat.getColName(), cStat); - } - - for (ColumnStatisticsObj statsObj : statsObjs) { - MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, - statsDesc, statsObj, colStats.getEngine()); - writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName())); - } - - Map newParams = new HashMap<>(mPartition.getParameters()); - StatsSetupConst.setColumnStatsState(newParams, colNames); - boolean isTxn = TxnUtils.isTransactionalTable(table); - if (isTxn) { - if (!areTxnStatsSupported) { - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - } else { - String errorMsg = verifyStatsChangeCtx( - TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(), - newParams, writeId, validWriteIds, true); - if (errorMsg != null) { - throw new MetaException(errorMsg); - } - if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) { - // Make sure we set the flag to invalid regardless of the current value. - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName()); - } - mPartition.setWriteId(writeId); - } - } - mPartition.setParameters(newParams); - return newParams; - }).onRetry(e -> e instanceof RetryingExecutor.RetryException) - .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval -> - ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); - committed = commitTransaction(); - return committed ? result : null; - } finally { - LOG.debug("{} updatePartitionColumnStatistics took {}ms, success: {}", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), - System.currentTimeMillis() - start, committed); - rollbackAndCleanup(committed, null); - } - } - - @Override - public Map updatePartitionColumnStatistics(ColumnStatistics colStats, - List partVals, String validWriteIds, long writeId) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - Table table = getTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); - MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); - return updatePartitionColumnStatistics(table, mTable, colStats, partVals, validWriteIds, writeId); - } - - @Override - public Map> updatePartitionColumnStatisticsInBatch( - Map partColStatsMap, - Table tbl, - List listeners, - String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - - return new GetHelper>>(tbl.getCatName(), - tbl.getDbName(), tbl.getTableName(), true, false) { - @Override - protected String describeResult() { - return "Map of partition key to column stats if successful"; - } - @Override - protected Map> getSqlResult(GetHelper>> ctx) - throws MetaException { - return directSql.updatePartitionColumnStatisticsBatch(partColStatsMap, tbl, - listeners, validWriteIds, writeId); - } - @Override - protected Map> getJdoResult(GetHelper>> ctx) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled"); - } - }.run(false); - } - - private List getMTableColumnStatistics(Table table, List colNames, String engine) - throws MetaException { - - Preconditions.checkState(this.currentTransaction.isActive()); - - if (colNames.isEmpty()) { - return Collections.emptyList(); - } - - boolean committed = false; - try { - openTransaction(); - - validateTableCols(table, colNames); - - List result = Collections.emptyList(); - try (Query query = pm.newQuery(MTableColumnStatistics.class)) { - result = - Batchable.runBatched(batchSize, colNames, new Batchable() { - @Override - public List run(List input) - throws MetaException { - StringBuilder filter = - new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && ("); - StringBuilder paramStr = new StringBuilder( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); - Object[] params = new Object[input.size() + 4]; - params[0] = table.getTableName(); - params[1] = table.getDbName(); - params[2] = table.getCatName(); - params[3] = engine; - for (int i = 0; i < input.size(); ++i) { - filter.append((i == 0) ? "" : " || ").append("colName == c").append(i); - paramStr.append(", java.lang.String c").append(i); - params[i + 4] = input.get(i); - } - filter.append(")"); - query.setFilter(filter.toString()); - query.declareParameters(paramStr.toString()); - List paritial = (List) query.executeWithArray(params); - pm.retrieveAll(paritial); - return paritial; - } - }); - - if (result.size() > colNames.size()) { - throw new MetaException("Unexpected " + result.size() + " statistics for " - + colNames.size() + " columns"); - } - result = new ArrayList<>(result); - } - committed = commitTransaction(); - return result; - } catch (Exception ex) { - LOG.error("Error retrieving statistics via jdo", ex); - if (ex instanceof MetaException) { - throw (MetaException) ex; - } - throw new MetaException(ex.getMessage()); - } finally { - rollbackAndCleanup(committed, null); - } - } - - @VisibleForTesting - public void validateTableCols(Table table, List colNames) throws MetaException { - List colList = table.getSd().getCols(); - for (String colName : colNames) { - boolean foundCol = false; - for (FieldSchema mCol : colList) { - if (mCol.getName().equals(colName)) { - foundCol = true; - break; - } - } - if (!foundCol) { - throw new MetaException("Column " + colName + " doesn't exist in table " - + table.getTableName() + " in database " + table.getDbName()); - } - } - } - - @Override - public List getTableColumnStatistics( - String catName, - String dbName, - String tableName, - List colNames) throws MetaException, NoSuchObjectException { - // Note: this will get stats without verifying ACID. - boolean committed = false; - Query query = null; - List result = new ArrayList<>(); - - try { - openTransaction(); - query = pm.newQuery(MTableColumnStatistics.class); - query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - query.setResult("DISTINCT engine"); - Collection names = (Collection) query.execute(tableName, dbName, catName); - List engines = new ArrayList<>(); - for (Iterator i = names.iterator(); i.hasNext();) { - engines.add((String) i.next()); - } - for (String e : engines) { - ColumnStatistics cs = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, e, true, true); - if (cs != null) { - result.add(cs); - } - } - committed = commitTransaction(); - return result; - } finally { - LOG.debug("Done executing getTableColumnStatistics with status : {}", - committed); - rollbackAndCleanup(committed, query); - } - } - - @Override - public ColumnStatistics getTableColumnStatistics( - String catName, - String dbName, - String tableName, - List colNames, - String engine) throws MetaException, NoSuchObjectException { - // Note: this will get stats without verifying ACID. - return getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, true, true); - } - - @Override - public ColumnStatistics getTableColumnStatistics( - String catName, - String dbName, - String tableName, - List colNames, - String engine, - String writeIdList) throws MetaException, NoSuchObjectException { - // If the current stats in the metastore doesn't comply with - // the isolation level of the query, set No to the compliance flag. - Boolean isCompliant = null; - if (writeIdList != null) { - MTable table = this.getMTable(catName, dbName, tableName); - if (table == null) { - throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tableName) + " table not found"); - } - isCompliant = !TxnUtils.isTransactionalTable(table.getParameters()) - || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table, writeIdList, false)); - } - ColumnStatistics stats = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, true, true); - if (stats != null && isCompliant != null) { - stats.setIsStatsCompliant(isCompliant); - } - return stats; - } - - protected ColumnStatistics getTableColumnStatisticsInternal( - String catName, String dbName, String tableName, final List colNames, String engine, - boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL); - return new GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName), - normalizeIdentifier(tableName), allowSql, allowJdo, null) { - @Override - protected ColumnStatistics getSqlResult(GetHelper ctx) throws MetaException { - return directSqlAggrStats.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll); - } - - @Override - protected ColumnStatistics getJdoResult(GetHelper ctx) throws MetaException { - - List mStats = getMTableColumnStatistics(getTable(), colNames, engine); - if (mStats.isEmpty()) { - return null; - } - // LastAnalyzed is stored per column, but thrift object has it per - // multiple columns. Luckily, nobody actually uses it, so we will set to - // lowest value of all columns for now. - ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0)); - List statObjs = new ArrayList<>(mStats.size()); - for (MTableColumnStatistics mStat : mStats) { - if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) { - desc.setLastAnalyzed(mStat.getLastAnalyzed()); - } - statObjs.add(StatObjectConverter.getColumnStatisticsObj(mStat, enableBitVector, enableKll)); - Deadline.checkTimeout(); - } - ColumnStatistics colStat = new ColumnStatistics(desc, statObjs); - colStat.setEngine(engine); - return colStat; - } - }.run(true); - } - - @Override - public List> getPartitionColumnStatistics(String catName, String dbName, String tableName, - List partNames, List colNames) throws MetaException, NoSuchObjectException { - // Note: this will get stats without verifying ACID. - boolean committed = false; - Query query = null; - List> result = new ArrayList<>(); - - try { - openTransaction(); - query = pm.newQuery(MPartitionColumnStatistics.class); - query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - query.setResult("DISTINCT engine"); - Collection names = (Collection) query.execute(tableName, dbName, catName); - List engines = new ArrayList<>(); - for (Iterator i = names.iterator(); i.hasNext();) { - engines.add((String) i.next()); - } - for (String e : engines) { - List cs = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, e, true, true); - if (cs != null) { - result.add(cs); - } - } - committed = commitTransaction(); - return result; - } finally { - LOG.debug("Done executing getTableColumnStatistics with status : {}", - committed); - rollbackAndCleanup(committed, query); - } - } - - @Override - public List getPartitionColumnStatistics(String catName, String dbName, String tableName, - List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException { - // Note: this will get stats without verifying ACID. - if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) { - LOG.debug("PartNames and/or ColNames are empty"); - return Collections.emptyList(); - } - return getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, true, true); - } - - @Override - public List getPartitionColumnStatistics( - String catName, String dbName, String tableName, - List partNames, List colNames, - String engine, String writeIdList) - throws MetaException, NoSuchObjectException { - if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) { - LOG.debug("PartNames and/or ColNames are empty"); - return Collections.emptyList(); - } - List allStats = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, true, true); - if (writeIdList != null) { - if (!areTxnStatsSupported) { - for (ColumnStatistics cs : allStats) { - cs.setIsStatsCompliant(false); - } - } else { - // TODO: this could be improved to get partitions in bulk - for (ColumnStatistics cs : allStats) { - MPartition mpart = ensureGetMPartition(new TableName(catName, dbName, tableName), - Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName())); - if (mpart == null - || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) { - if (mpart != null) { - LOG.debug("The current metastore transactional partition column statistics for {}.{}.{} " - + "(write ID {}) are not valid for current query ({} {})", dbName, tableName, - mpart.getPartitionName(), mpart.getWriteId(), writeIdList); - } - cs.setIsStatsCompliant(false); - } else { - cs.setIsStatsCompliant(true); - } - } - } - } - return allStats; - } - - protected List getPartitionColumnStatisticsInternal( - String catName, String dbName, String tableName, final List partNames, final List colNames, - String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL); - return new GetListHelper(catName, dbName, tableName, allowSql, allowJdo) { - @Override - protected List getSqlResult( - GetHelper> ctx) throws MetaException { - return directSqlAggrStats.getPartitionStats( - catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll); - } - @Override - protected List getJdoResult(GetHelper> ctx) - throws MetaException, NoSuchObjectException { - List mStats = - getMPartitionColumnStatistics(getTable(), partNames, colNames, engine); - List result = new ArrayList<>(Math.min(mStats.size(), partNames.size())); - String lastPartName = null; - List curList = null; - ColumnStatisticsDesc csd = null; - for (int i = 0; i <= mStats.size(); ++i) { - boolean isLast = i == mStats.size(); - MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i); - String partName = isLast ? null : mStatsObj.getPartition().getPartitionName(); - if (isLast || !partName.equals(lastPartName)) { - if (i != 0) { - ColumnStatistics colStat = new ColumnStatistics(csd, curList); - colStat.setEngine(engine); - result.add(colStat); - } - if (isLast) { - continue; - } - csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj); - curList = new ArrayList<>(colNames.size()); - } - curList.add(StatObjectConverter.getColumnStatisticsObj(mStatsObj, enableBitVector, enableKll)); - lastPartName = partName; - Deadline.checkTimeout(); - } - return result; - } - }.run(true); - } - - @Override - public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, - final List partNames, final List colNames, - String engine, String writeIdList) throws MetaException, NoSuchObjectException { - // If the current stats in the metastore doesn't comply with - // the isolation level of the query, return null. - if (writeIdList != null) { - if (partNames == null || partNames.isEmpty()) { - return null; - } - - Table table = getTable(catName, dbName, tblName); - boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); - if (isTxn && !areTxnStatsSupported) { - return null; - } - GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec(); - fs.setFilterMode(PartitionFilterMode.BY_NAMES); - fs.setFilters(partNames); - GetProjectionsSpec ps = new GetProjectionsSpec(); - ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%'); - ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values")); - List parts = getPartitionSpecsByFilterAndProjection(table, ps, fs); - - // Loop through the given "partNames" list - // checking isolation-level-compliance of each partition column stats. - for (Partition part : parts) { - - if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) { - String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); - LOG.debug("The current metastore transactional partition column " - + "statistics for {}.{}.{} is not valid for the current query", - dbName, tblName, partName); - return null; - } - } - } - return get_aggr_stats_for(catName, dbName, tblName, partNames, colNames, engine); - } - - @Override - public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, - final List partNames, final List colNames, String engine) - throws MetaException, NoSuchObjectException { - final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(getConf(), - ConfVars.STATS_NDV_DENSITY_FUNCTION); - final double ndvTuner = MetastoreConf.getDoubleVar(getConf(), ConfVars.STATS_NDV_TUNER); - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL); - return new GetHelper(catName, dbName, tblName, true, false) { - @Override - protected AggrStats getSqlResult(GetHelper ctx) - throws MetaException { - return directSql.aggrColStatsForPartitions(catName, dbName, tblName, partNames, - colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll); - } - @Override - protected AggrStats getJdoResult(GetHelper ctx) - throws MetaException, NoSuchObjectException { - // This is fast path for query optimizations, if we can find this info - // quickly using - // directSql, do it. No point in failing back to slow path here. - throw new MetaException("Jdo path is not implemented for stats aggr."); - } - @Override - protected String describeResult() { - return null; - } - }.run(true); - } - - @Override - public List getPartitionColStatsForDatabase(String catName, String dbName) - throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_KLL); - return new GetHelper>( - catName, dbName, null, true, false) { - @Override - protected List getSqlResult( - GetHelper> ctx) throws MetaException { - return directSqlAggrStats.getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) - throws MetaException, NoSuchObjectException { - // This is fast path for query optimizations, if we can find this info - // quickly using directSql, do it. No point in failing back to slow path - // here. - throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase."); - } - - @Override - protected String describeResult() { - return null; - } - }.run(true); - } - - private List getMPartitionColumnStatistics(Table table, List partNames, - List colNames, String engine) throws MetaException { - boolean committed = false; - - try { - openTransaction(); - // We are not going to verify SD for each partition. Just verify for the - // table. TODO: we need verify the partition column instead - try { - validateTableCols(table, colNames); - } catch (MetaException me) { - LOG.warn("The table does not have the same column definition as its partition."); - } - List result = Collections.emptyList(); - try (Query query = pm.newQuery(MPartitionColumnStatistics.class)) { - String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"; - String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3 && engine == t4 && ("; - Object[] params = new Object[colNames.size() + partNames.size() + 4]; - int i = 0; - params[i++] = table.getTableName(); - params[i++] = table.getDbName(); - params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf); - params[i++] = engine; - int firstI = i; - for (String s : partNames) { - filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i; - paramStr += ", java.lang.String p" + i; - params[i++] = s; - } - filter += ") && ("; - firstI = i; - for (String s : colNames) { - filter += ((i == firstI) ? "" : " || ") + "colName == c" + i; - paramStr += ", java.lang.String c" + i; - params[i++] = s; - } - filter += ")"; - query.setFilter(filter); - query.declareParameters(paramStr); - query.setOrdering("partition.partitionName ascending"); - result = (List) query.executeWithArray(params); - pm.retrieveAll(result); - result = new ArrayList<>(result); - } catch (Exception ex) { - LOG.error("Error retrieving statistics via jdo", ex); - throw new MetaException(ex.getMessage()); - } - committed = commitTransaction(); - return result; - } finally { - if (!committed) { - rollbackTransaction(); - return Collections.emptyList(); - } - } - } - - @Override - public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) { - - String catName = tn.getCat(); - String dbName = tn.getDb(); - String tableName = tn.getTable(); - - Query query = null; - dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); - catName = normalizeIdentifier(catName); - if (tableName == null) { - throw new RuntimeException("Table name is null."); - } - boolean ret = false; - try { - openTransaction(); - MTable mTable = getMTable(catName, dbName, tableName); - - query = pm.newQuery(MPartitionColumnStatistics.class); - - String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4"; - String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4"; - - query.setFilter(filter); - query.declareParameters(parameters); - - Long number = query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), - normalizeIdentifier(catName)); - - new GetHelper(catName, dbName, tableName, true, true) { - private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); - - @Override - protected String describeResult() { - return "Partition count"; - } - - @Override - protected boolean canUseDirectSql(GetHelper ctx) throws MetaException { - return true; - } - - @Override - protected Integer getSqlResult(GetHelper ctx) throws MetaException { - directSql.deleteColumnStatsState(getTable().getId()); - return 0; - } - - @Override - protected Integer getJdoResult(GetHelper ctx) throws MetaException, NoSuchObjectException { - try { - List parts = getPartitions(catName, dbName, tableName, - GetPartitionsArgs.getAllPartitions()); - for (Partition part : parts) { - Partition newPart = new Partition(part); - StatsSetupConst.clearColumnStatsState(newPart.getParameters()); - alterPartition(catName, dbName, tableName, part.getValues(), newPart, writeIdList); - } - return parts.size(); - } catch (InvalidObjectException e) { - LOG.error("error updating parts", e); - return -1; - } - } - }.run(true); - - ret = commitTransaction(); - } catch (Exception e) { - LOG.error("Couldn't clear stats for table", e); - } finally { - rollbackAndCleanup(ret, query); - } - } - - @Override - public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, - List partNames, List colNames, String engine) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - if (partNames == null || partNames.isEmpty()) { - throw new InvalidInputException("No partition specified for dropping the statistics"); - } - dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); - catName = normalizeIdentifier(catName); - List cols = normalizeIdentifiers(colNames); - return new GetHelper(catName, dbName, tableName, true, true) { - @Override - protected String describeResult() { - return "delete partition column stats"; - } - @Override - protected Boolean getSqlResult(GetHelper ctx) throws MetaException { - DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); - return deleteStats.deletePartitionColumnStats(catName, dbName, tableName, partNames, cols, engine); - } - @Override - protected Boolean getJdoResult(GetHelper ctx) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - return deletePartitionColumnStatisticsViaJdo(catName, dbName, tableName, partNames, cols, engine); - } - }.run(false); - } - - private boolean deletePartitionColumnStatisticsViaJdo(String catName, String dbName, String tableName, - List partNames, List colNames, String engine) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - boolean ret = false; - String database = org.apache.commons.lang3.StringUtils.defaultString(dbName, - Warehouse.DEFAULT_DATABASE_NAME); - String catalog = normalizeIdentifier(catName); - try { - openTransaction(); - Batchable b = new Batchable() { - @Override - public List run(List input) throws Exception { - Query query = pm.newQuery(MPartitionColumnStatistics.class); - addQueryAfterUse(query); - String filter; - String parameters; - if (colNames != null && !colNames.isEmpty()) { - filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " - + "t4.contains(colName) && partition.table.database.catalogName == t5" + (engine != null ? " && engine == t6" : ""); - parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, " - + "java.util.Collection t4, java.lang.String t5" + (engine != null ? ", java.lang.String t6" : ""); - } else { - filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + - "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : ""); - parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + (engine != null ? ", java.lang.String t5" : ""); - } - query.setFilter(filter); - query.declareParameters(parameters); - List params = new ArrayList<>(); - params.add(input); - params.add(normalizeIdentifier(database)); - params.add(normalizeIdentifier(tableName)); - if (colNames != null && !colNames.isEmpty()) { - params.add(colNames); - } - params.add(catalog); - if (engine != null) { - params.add(engine); - } - List mStatsObjColl = - (List) query.executeWithArray(params.toArray()); - pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { - pm.deletePersistentAll(mStatsObjColl); - } - return null; - } - }; - try { - Batchable.runBatched(batchSize, partNames, b); - } finally { - b.closeAllQueries(); - } - - Batchable.runBatched(batchSize, partNames, new Batchable() { - @Override - public List run(List input) throws MetaException { - Pair> queryWithParams = getPartQueryWithParams(pm, catalog, database, tableName, - input); - try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) { - qw.setResultClass(MPartition.class); - qw.setClass(MPartition.class); - List mparts = (List) qw.executeWithMap(queryWithParams.getRight()); - for (MPartition mPart : mparts) { - Map params = mPart.getParameters(); - if (params != null && params.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { - if (colNames == null || colNames.isEmpty()) { - StatsSetupConst.clearColumnStatsState(params); - } else { - StatsSetupConst.removeColumnStatsState(params, colNames); - } - mPart.setParameters(params); - } - } - } - return Collections.emptyList(); - } - }); - ret = commitTransaction(); - } finally { - rollbackAndCleanup(ret, null); - } - return ret; - } - - @Override - public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, - List colNames, String engine) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); - if (tableName == null) { - throw new InvalidInputException("Table name is null."); - } - List cols = normalizeIdentifiers(colNames); - return new GetHelper(catName, dbName, tableName, true, true) { - @Override - protected String describeResult() { - return "delete table column stats"; - } - @Override - protected Boolean getSqlResult(GetHelper ctx) throws MetaException { - DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); - return deleteStats.deleteTableColumnStatistics(getTable(), cols, engine); - } - @Override - protected Boolean getJdoResult(GetHelper ctx) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - return deleteTableColumnStatisticsViaJdo(catName, dbName, tableName, cols, engine); - } - }.run(true); + } + return properties; } - private boolean deleteTableColumnStatisticsViaJdo(String catName, String dbName, String tableName, - List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - boolean ret = false; - Query query = null; + public Map selectProperties(String key, java.util.function.Function transform) { + boolean success = false; + Query query = null; + Map results = null; try { - openTransaction(); - List mStatsObjColl; - // Note: this does not verify ACID state; called internally when removing cols/etc. - // Also called via an unused metastore API that checks for ACID tables. - query = pm.newQuery(MTableColumnStatistics.class); - String filter; - String parameters; - if (colNames != null && !colNames.isEmpty()) { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + (engine != null ? " && engine == t5" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + (engine != null ? ", java.lang.String t5" : ""); - } else { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + (engine != null ? " && engine == t4" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + (engine != null ? ", java.lang.String t4" : ""); - } - - query.setFilter(filter); - query.declareParameters(parameters); - List params = new ArrayList<>(); - params.add(normalizeIdentifier(tableName)); - params.add(normalizeIdentifier(dbName)); - params.add(catName == null ? null : normalizeIdentifier(catName)); - if (colNames != null && !colNames.isEmpty()) { - params.add(colNames); - } - if (engine != null) { - params.add(engine); - } - mStatsObjColl = (List) query.executeWithArray(params.toArray()); - pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { - pm.deletePersistentAll(mStatsObjColl); - } - - MTable mTable = getMTable(catName, dbName, tableName); - if (mTable != null) { - Map tableParams = mTable.getParameters(); - if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { - if (colNames == null || colNames.isEmpty()) { - StatsSetupConst.clearColumnStatsState(tableParams); - } else { - StatsSetupConst.removeColumnStatsState(tableParams, colNames); + if (openTransaction()) { + Collection properties; + if (key == null || key.isEmpty()) { + query = pm.newQuery(MMetastoreDBProperties.class); + properties = (Collection) query.execute(); + } else { + query = pm.newQuery(MMetastoreDBProperties.class, "this.propertyKey.startsWith(key)"); + query.declareParameters(PTYPARAM_STR_KEY); + properties = (Collection) query.execute(key); + } + pm.retrieveAll(properties); + if (!properties.isEmpty()) { + results = new TreeMap(); + for(MMetastoreDBProperties ptys : properties) { + T t = (T) (transform != null? transform.apply(ptys) : ptys); + if (t != null) { + results.put(ptys.getPropertykey(), t); + } } - mTable.setParameters(tableParams); } + success = commitTransaction(); } - ret = commitTransaction(); } finally { - rollbackAndCleanup(ret, query); + rollbackAndCleanup(success, query); } - return ret; + return results; } @Override @@ -5253,583 +3201,6 @@ private MPackage findMPackage(String catName, String db, String packageName) { return pkg; } - private void executePlainSQL(String sql, - boolean atLeastOneRecord, - Consumer exceptionConsumer) - throws SQLException, MetaException { - String s = dbType.getPrepareTxnStmt(); - assert pm.currentTransaction().isActive(); - JDOConnection jdoConn = pm.getDataStoreConnection(); - Connection conn = (Connection) jdoConn.getNativeConnection(); - try (Statement statement = conn.createStatement()) { - if (s != null) { - statement.execute(s); - } - try { - statement.execute(sql); - try (ResultSet rs = statement.getResultSet()) { - // sqlserver needs rs.next for validating the s4u nowait - if (atLeastOneRecord && !rs.next()) { - throw new MetaException("At least one record but none is returned from the query: " + sql); - } - } - } catch (SQLException e) { - if (exceptionConsumer != null) { - exceptionConsumer.accept(e); - } else { - throw e; - } - } - } finally { - jdoConn.close(); - } - } - - @Override - public List getPrimaryKeys(PrimaryKeysRequest request) throws MetaException { - try { - return getPrimaryKeysInternal(request.getCatName(), - request.getDb_name(),request.getTbl_name()); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - private List getPrimaryKeysInternal(final String catName, - final String dbNameInput, - final String tblNameInput) - throws MetaException, NoSuchObjectException { - final String dbName = dbNameInput != null ? normalizeIdentifier(dbNameInput) : null; - final String tblName = normalizeIdentifier(tblNameInput); - return new GetListHelper(catName, dbName, tblName, true, true) { - - @Override - protected List getSqlResult(GetHelper> ctx) throws MetaException { - return directSql.getPrimaryKeys(catName, dbName, tblName); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) throws MetaException, NoSuchObjectException { - return getPrimaryKeysViaJdo(catName, dbName, tblName); - } - }.run(false); - } - - private List getPrimaryKeysViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; - List primaryKeys = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == cat_name &&" - + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + - "java.lang.String cat_name"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - primaryKeys = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPK = (MConstraint) i.next(); - List cols = currPK.getParentColumn() != null ? - currPK.getParentColumn().getCols() : currPK.getParentTable().getPartitionKeys(); - int enableValidateRely = currPK.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, - tblName, - cols.get(currPK.getParentIntegerIndex()).getName(), - currPK.getPosition(), - currPK.getConstraintName(), enable, validate, rely); - keyCol.setCatName(catName); - primaryKeys.add(keyCol); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return primaryKeys; - } - - private String getPrimaryKeyConstraintName(String catName, String dbName, String tblName) { - boolean commited = false; - String ret = null; - Query query = null; - - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + - "java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPK = (MConstraint) i.next(); - ret = currPK.getConstraintName(); - break; - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return ret; - } - - @Override - public List getForeignKeys(ForeignKeysRequest request) throws MetaException { - try { - return getForeignKeysInternal(request.getCatName(), - request.getParent_db_name(), request.getParent_tbl_name() , - request.getForeign_db_name(),request.getForeign_tbl_name(), true, - true); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - private List getForeignKeysInternal( - final String catName, final String parent_db_name_input, final String parent_tbl_name_input, - final String foreign_db_name_input, final String foreign_tbl_name_input, boolean allowSql, - boolean allowJdo) throws MetaException, NoSuchObjectException { - final String parent_db_name = (parent_db_name_input != null) ? normalizeIdentifier(parent_db_name_input) : null; - final String parent_tbl_name = (parent_tbl_name_input != null) ? normalizeIdentifier(parent_tbl_name_input) : null; - final String foreign_db_name = (foreign_db_name_input != null) ? normalizeIdentifier(foreign_db_name_input) : null; - final String foreign_tbl_name = (foreign_tbl_name_input != null) - ? normalizeIdentifier(foreign_tbl_name_input) : null; - final String db_name; - final String tbl_name; - if (foreign_tbl_name == null) { - // The FK table name might be null if we are retrieving the constraint from the PK side - db_name = parent_db_name; - tbl_name = parent_tbl_name; - } else { - db_name = foreign_db_name; - tbl_name = foreign_tbl_name; - } - return new GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { - - @Override - protected List getSqlResult(GetHelper> ctx) throws MetaException { - return directSql.getForeignKeys(catName, parent_db_name, - parent_tbl_name, foreign_db_name, foreign_tbl_name); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) throws MetaException, NoSuchObjectException { - return getForeignKeysViaJdo(catName, parent_db_name, - parent_tbl_name, foreign_db_name, foreign_tbl_name); - } - }.run(false); - } - - private List getForeignKeysViaJdo(String catName, String parentDbName, - String parentTblName, String foreignDbName, String foreignTblName) { - boolean commited = false; - List foreignKeys = null; - Collection constraints = null; - Query query = null; - Map tblToConstraint = new HashMap<>(); - try { - openTransaction(); - String queryText = - " parentTable.database.catalogName == catName1 &&" + "childTable.database.catalogName == catName2 && " + ( - parentTblName != null ? "parentTable.tableName == parent_tbl_name && " : "") + ( - parentDbName != null ? " parentTable.database.name == parent_db_name && " : "") + ( - foreignTblName != null ? " childTable.tableName == foreign_tbl_name && " : "") + ( - foreignDbName != null ? " childTable.database.name == foreign_db_name && " : "") - + " constraintType == MConstraint.FOREIGN_KEY_CONSTRAINT"; - queryText = queryText.trim(); - query = pm.newQuery(MConstraint.class, queryText); - String paramText = "java.lang.String catName1, java.lang.String catName2" + ( - parentTblName == null ? "" : ", java.lang.String parent_tbl_name") + ( - parentDbName == null ? "" : " , java.lang.String parent_db_name") + ( - foreignTblName == null ? "" : ", java.lang.String foreign_tbl_name") + ( - foreignDbName == null ? "" : " , java.lang.String foreign_db_name"); - query.declareParameters(paramText); - List params = new ArrayList<>(); - params.add(catName); - params.add(catName); // This is not a mistake, catName is in the where clause twice - if (parentTblName != null) { - params.add(parentTblName); - } - if (parentDbName != null) { - params.add(parentDbName); - } - if (foreignTblName != null) { - params.add(foreignTblName); - } - if (foreignDbName != null) { - params.add(foreignDbName); - } - constraints = (Collection) query.executeWithArray(params.toArray(new String[0])); - - pm.retrieveAll(constraints); - foreignKeys = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPKFK = (MConstraint) i.next(); - List parentCols = currPKFK.getParentColumn() != null ? - currPKFK.getParentColumn().getCols() : currPKFK.getParentTable().getPartitionKeys(); - List childCols = currPKFK.getChildColumn() != null ? - currPKFK.getChildColumn().getCols() : currPKFK.getChildTable().getPartitionKeys(); - int enableValidateRely = currPKFK.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - String consolidatedtblName = - currPKFK.getParentTable().getDatabase().getName() + "." + - currPKFK.getParentTable().getTableName(); - String pkName; - if (tblToConstraint.containsKey(consolidatedtblName)) { - pkName = tblToConstraint.get(consolidatedtblName); - } else { - pkName = getPrimaryKeyConstraintName(currPKFK.getParentTable().getDatabase().getCatalogName(), - currPKFK.getParentTable().getDatabase().getName(), - currPKFK.getParentTable().getTableName()); - tblToConstraint.put(consolidatedtblName, pkName); - } - SQLForeignKey fk = new SQLForeignKey( - currPKFK.getParentTable().getDatabase().getName(), - currPKFK.getParentTable().getTableName(), - parentCols.get(currPKFK.getParentIntegerIndex()).getName(), - currPKFK.getChildTable().getDatabase().getName(), - currPKFK.getChildTable().getTableName(), - childCols.get(currPKFK.getChildIntegerIndex()).getName(), - currPKFK.getPosition(), - currPKFK.getUpdateRule(), - currPKFK.getDeleteRule(), - currPKFK.getConstraintName(), pkName, enable, validate, rely); - fk.setCatName(catName); - foreignKeys.add(fk); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return foreignKeys; - } - - @Override - public List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException { - try { - return getUniqueConstraintsInternal(request.getCatName(), - request.getDb_name(),request.getTbl_name(), true, true); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - private List getUniqueConstraintsInternal( - String catNameInput, final String db_name_input, final String tbl_name_input, - boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - final String catName = normalizeIdentifier(catNameInput); - final String db_name = normalizeIdentifier(db_name_input); - final String tbl_name = normalizeIdentifier(tbl_name_input); - return new GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { - - @Override - protected List getSqlResult(GetHelper> ctx) - throws MetaException { - return directSql.getUniqueConstraints(catName, db_name, tbl_name); - } - - @Override - protected List getJdoResult(GetHelper> ctx) - throws MetaException, NoSuchObjectException { - return getUniqueConstraintsViaJdo(catName, db_name, tbl_name); - } - }.run(false); - } - - private List getUniqueConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; - List uniqueConstraints = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name && parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.UNIQUE_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - uniqueConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - uniqueConstraints.add(new SQLUniqueConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getPosition(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return uniqueConstraints; - } - - @Override - public List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException { - try { - return getNotNullConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - @Override - public List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException { - try { - return getDefaultConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - @Override - public List getCheckConstraints(CheckConstraintsRequest request) throws MetaException { - try { - return getCheckConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); - } catch (NoSuchObjectException e) { - throw new MetaException(ExceptionUtils.getStackTrace(e)); - } - } - - private List getDefaultConstraintsInternal( - String catName, final String db_name_input, final String tbl_name_input, boolean allowSql, - boolean allowJdo) throws MetaException, NoSuchObjectException { - catName = normalizeIdentifier(catName); - final String db_name = normalizeIdentifier(db_name_input); - final String tbl_name = normalizeIdentifier(tbl_name_input); - return new GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { - - @Override - protected List getSqlResult(GetHelper> ctx) - throws MetaException { - return directSql.getDefaultConstraints(catName, db_name, tbl_name); - } - - @Override - protected List getJdoResult(GetHelper> ctx) - throws MetaException, NoSuchObjectException { - return getDefaultConstraintsViaJdo(catName, db_name, tbl_name); - } - }.run(false); - } - - protected List getCheckConstraintsInternal(String catName, final String db_name_input, - final String tbl_name_input, boolean allowSql, - boolean allowJdo) - throws MetaException, NoSuchObjectException { - final String db_name = normalizeIdentifier(db_name_input); - final String tbl_name = normalizeIdentifier(tbl_name_input); - return new GetListHelper(normalizeIdentifier(catName), db_name, tbl_name, - allowSql, allowJdo) { - - @Override - protected List getSqlResult(GetHelper> ctx) - throws MetaException { - return directSql.getCheckConstraints(catName, db_name, tbl_name); - } - - @Override - protected List getJdoResult(GetHelper> ctx) - throws MetaException, NoSuchObjectException { - return getCheckConstraintsViaJdo(catName, db_name, tbl_name); - } - }.run(false); - } - - private List getCheckConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; - List checkConstraints= null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName && constraintType == MConstraint.CHECK_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - checkConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - checkConstraints.add(new SQLCheckConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), - currConstraint.getDefaultValue(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return checkConstraints; - } - - private List getDefaultConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; - List defaultConstraints= null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.DEFAULT_CONSTRAINT"); - query.declareParameters( - "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - defaultConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - defaultConstraints.add(new SQLDefaultConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getDefaultValue(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return defaultConstraints; - } - - protected List getNotNullConstraintsInternal(String catName, final String db_name_input, - final String tbl_name_input, boolean allowSql, boolean allowJdo) - throws MetaException, NoSuchObjectException { - catName = normalizeIdentifier(catName); - final String db_name = normalizeIdentifier(db_name_input); - final String tbl_name = normalizeIdentifier(tbl_name_input); - return new GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { - - @Override - protected List getSqlResult(GetHelper> ctx) - throws MetaException { - return directSql.getNotNullConstraints(catName, db_name, tbl_name); - } - - @Override - protected List getJdoResult(GetHelper> ctx) - throws MetaException, NoSuchObjectException { - return getNotNullConstraintsViaJdo(catName, db_name, tbl_name); - } - }.run(false); - } - - private List getNotNullConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; - List notNullConstraints = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName && constraintType == MConstraint.NOT_NULL_CONSTRAINT"); - query.declareParameters( - "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - notNullConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - notNullConstraints.add(new SQLNotNullConstraint(catName, dbName, - tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return notNullConstraints; - } - - /** - * Api to fetch all constraints at once - * @param request request object - * @return all table constraints - * @throws MetaException - * @throws NoSuchObjectException - */ - @Override - public SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request) - throws MetaException, NoSuchObjectException { - String catName = request.getCatName(); - String dbName = request.getDbName(); - String tblName = request.getTblName(); - debugLog("Get all table constraints for the table - " + catName + "." + dbName + "." + tblName - + " in class ObjectStore.java"); - SQLAllTableConstraints sqlAllTableConstraints = new SQLAllTableConstraints(); - PrimaryKeysRequest primaryKeysRequest = new PrimaryKeysRequest(dbName, tblName); - primaryKeysRequest.setCatName(catName); - sqlAllTableConstraints.setPrimaryKeys(getPrimaryKeys(primaryKeysRequest)); - ForeignKeysRequest foreignKeysRequest = - new ForeignKeysRequest(null, null, dbName, tblName); - foreignKeysRequest.setCatName(catName); - sqlAllTableConstraints.setForeignKeys(getForeignKeys(foreignKeysRequest)); - sqlAllTableConstraints. - setUniqueConstraints(getUniqueConstraints(new UniqueConstraintsRequest(catName, dbName, tblName))); - sqlAllTableConstraints. - setDefaultConstraints(getDefaultConstraints(new DefaultConstraintsRequest(catName, dbName, tblName))); - sqlAllTableConstraints. - setCheckConstraints(getCheckConstraints(new CheckConstraintsRequest(catName, dbName, tblName))); - sqlAllTableConstraints. - setNotNullConstraints(getNotNullConstraints(new NotNullConstraintsRequest(catName, dbName, tblName))); - return sqlAllTableConstraints; - } - - @Override - public void dropConstraint(String catName, String dbName, String tableName, - String constraintName, boolean missingOk) - throws NoSuchObjectException { - boolean success = false; - try { - openTransaction(); - - List tabConstraints = - listAllTableConstraintsWithOptionalConstraintName(catName, dbName, tableName, constraintName); - if (CollectionUtils.isNotEmpty(tabConstraints)) { - pm.deletePersistentAll(tabConstraints); - } else if (!missingOk) { - throw new NoSuchObjectException("The constraint: " + constraintName + - " does not exist for the associated table: " + dbName + "." + tableName); - } - success = commitTransaction(); - } finally { - rollbackAndCleanup(success, null); - } - } - @Override public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException, NoSuchObjectException { @@ -7465,72 +4836,6 @@ private List getMRuntimeStats(int maxEntries, int maxCreateTime) { } } - /** - * Return true if the current statistics in the Metastore is valid - * for the query of the given "txnId" and "queryValidWriteIdList". - * - * Note that a statistics entity is valid iff - * the stats is written by the current query or - * the conjunction of the following two are true: - * ~ COLUMN_STATE_ACCURATE(CSA) state is true - * ~ Isolation-level (snapshot) compliant with the query - * @param tbl MTable of the stats entity - * @param queryValidWriteIdList valid writeId list of the query - * @Precondition "tbl" should be retrieved from the TBLS table. - */ - private boolean isCurrentStatsValidForTheQuery(MTable tbl, String queryValidWriteIdList, - boolean isCompleteStatsWriter) throws MetaException { - return isCurrentStatsValidForTheQuery(tbl.getParameters(), tbl.getWriteId(), - queryValidWriteIdList, isCompleteStatsWriter); - } - - /** - * Return true if the current statistics in the Metastore is valid - * for the query of the given "txnId" and "queryValidWriteIdList". - * - * Note that a statistics entity is valid iff - * the stats is written by the current query or - * the conjunction of the following two are true: - * ~ COLUMN_STATE_ACCURATE(CSA) state is true - * ~ Isolation-level (snapshot) compliant with the query - * @param queryValidWriteIdList valid writeId list of the query - */ - // TODO: move to somewhere else - public static boolean isCurrentStatsValidForTheQuery( - Map statsParams, long statsWriteId, String queryValidWriteIdList, - boolean isCompleteStatsWriter) throws MetaException { - - // Note: can be changed to debug/info to verify the calls. - LOG.debug("isCurrentStatsValidForTheQuery with stats write ID {}; query {}; writer: {} params {}", - statsWriteId, queryValidWriteIdList, isCompleteStatsWriter, statsParams); - // return true since the stats does not seem to be transactional. - if (statsWriteId < 1) { - return true; - } - // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is - // written by an aborted transaction but TXNS has no entry for the transaction - // after compaction. Don't check for a complete stats writer - it may replace invalid stats. - if (!isCompleteStatsWriter && !StatsSetupConst.areBasicStatsUptoDate(statsParams)) { - return false; - } - - if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid. - ValidWriteIdList list4TheQuery = ValidReaderWriteIdList.fromValue(queryValidWriteIdList); - // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), - // that means it cannot possibly be a concurrent write. If it's not valid (we are not - // allowed to see it), that means it's either concurrent or aborted, same thing for us. - if (list4TheQuery.isWriteIdValid(statsWriteId)) { - return true; - } - // Updater is also allowed to overwrite stats from aborted txns, as long as they are not concurrent. - if (isCompleteStatsWriter && list4TheQuery.isWriteIdAborted(statsWriteId)) { - return true; - } - } - - return false; - } - @Override public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) throws MetaException { ensureScheduledQueriesEnabled(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 4fae846e5455..0633d01ef903 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -118,6 +118,7 @@ import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; +import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MTable; @@ -1280,8 +1281,10 @@ default List listPartitionsPsWithAuth(String catName, String db_name, * @throws InvalidObjectException the stats object is invalid * @throws InvalidInputException unable to record the stats for the table */ - Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + default Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return unwrap(ColStatsStore.class).updateTableColumnStatistics(colStats, validWriteIds, writeId); + } /** Persists the given column statistics object to the metastore * @deprecated Use {@link #updatePartitionColumnStatistics(Table, MTable, ColumnStatistics, List, String, long)} instead @@ -1298,10 +1301,12 @@ Map updatePartitionColumnStatistics(ColumnStatistics statsObj, List partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; - Map updatePartitionColumnStatistics(Table table, MTable mTable, + default Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics statsObj, List partVals, String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return unwrap(ColStatsStore.class).updatePartitionColumnStatistics(table, mTable, statsObj, partVals, validWriteIds, writeId); + } /** * Returns the relevant column statistics for a given column in a given table in a given database diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java new file mode 100644 index 000000000000..ed2d582487cb --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.metastore.iface; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; +import org.apache.hadoop.hive.metastore.metastore.impl.ColStatsStoreImpl; +import org.apache.hadoop.hive.metastore.model.MTable; + +@MetaDescriptor(alias = "stats", defaultImpl = ColStatsStoreImpl.class) +public interface ColStatsStore { + /** Persists the given column statistics object to the metastore + * @param colStats object to persist + * @return Boolean indicating the outcome of the operation + * @throws NoSuchObjectException No such table. + * @throws MetaException error accessing the RDBMS. + * @throws InvalidObjectException the stats object is invalid + * @throws InvalidInputException unable to record the stats for the table + */ + Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + + Map updatePartitionColumnStatistics(Table table, MTable mTable, + ColumnStatistics statsObj, List partVals, + String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java new file mode 100644 index 000000000000..8032ecde335e --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java @@ -0,0 +1,8 @@ +package org.apache.hadoop.hive.metastore.metastore.iface; + +import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; +import org.apache.hadoop.hive.metastore.metastore.impl.ConstraintStoreImpl; + +@MetaDescriptor(alias = "constraint", defaultImpl = ConstraintStoreImpl.class) +public interface ConstraintStore { +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java new file mode 100644 index 000000000000..277f1e545b4b --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -0,0 +1,1321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.metastore.impl; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import javax.jdo.Query; +import javax.jdo.datastore.JDOConnection; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.metastore.Batchable; +import org.apache.hadoop.hive.metastore.Deadline; +import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.StatObjectConverter; +import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec; +import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionFilterMode; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats; +import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; +import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; +import org.apache.hadoop.hive.metastore.model.MPartition; +import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; +import org.apache.hadoop.hive.metastore.model.MTable; +import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; +import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; +import org.apache.hadoop.hive.metastore.utils.RetryingExecutor; + +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers; + +public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { + + @Override + public List getTableNamesWithStats() throws MetaException, NoSuchObjectException { + return new ObjectStore.GetListHelper(null, null, null, true, false) { + @Override + protected List getSqlResult( + ObjectStore.GetHelper> ctx) throws MetaException { + return directSql.getTableNamesWithStats(); + } + + @Override + protected List getJdoResult( + ObjectStore.GetHelper> ctx) throws MetaException { + throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? + } + }.run(false); + } + + @Override + public Map> getPartitionColsWithStats(String catName, String dbName, String tableName) + throws MetaException, NoSuchObjectException { + return new ObjectStore.GetHelper>>(catName, dbName, null, true, false) { + @Override + protected Map> getSqlResult( + ObjectStore.GetHelper>> ctx) throws MetaException { + try { + return directSql.getColAndPartNamesWithStats(catName, dbName, tableName); + } catch (Throwable ex) { + LOG.error("DirectSQL failed", ex); + throw new MetaException(ex.getMessage()); + } + } + + @Override + protected Map> getJdoResult( + ObjectStore.GetHelper>> ctx) throws MetaException { + throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? + } + + @Override + protected String describeResult() { + return results.size() + " partitions"; + } + }.run(false); + } + + @Override + public List getAllTableNamesForStats() throws MetaException, NoSuchObjectException { + return new ObjectStore.GetListHelper(null, null, null, true, false) { + @Override + protected List getSqlResult( + ObjectStore.GetHelper> ctx) throws MetaException { + return directSql.getAllTableNamesForStats(); + } + + @Override + protected List getJdoResult( + ObjectStore.GetHelper> ctx) throws MetaException { + boolean commited = false; + Query query = null; + List result = new ArrayList<>(); + openTransaction(); + try { + String paramStr = "", whereStr = ""; + for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) { + if (i != 0) { + paramStr += ", "; + whereStr += "||"; + } + paramStr += "java.lang.String tt" + i; + whereStr += " tableType == tt" + i; + } + query = pm.newQuery(MTable.class, whereStr); + query.declareParameters(paramStr); + Collection tbls = (Collection) query.executeWithArray( + query, MetaStoreDirectSql.STATS_TABLE_TYPES); + pm.retrieveAll(tbls); + for (MTable tbl : tbls) { + result.add(new TableName( + tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName())); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return result; + } + }.run(false); + } + + private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, + MTableColumnStatistics oldStats) throws MetaException { + + Preconditions.checkState(this.currentTransaction.isActive()); + + String colName = mStatsObj.getColName(); + + LOG.info("Updating table level column statistics for table={} colName={}", + Warehouse.getCatalogQualifiedTableName(table), colName); + validateTableCols(table, Lists.newArrayList(colName)); + + if (oldStats != null) { + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); + } else { + pm.makePersistent(mStatsObj); + } + } + + private void writeMPartitionColumnStatistics(Table table, Partition partition, + MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStats) { + String catName = mStatsObj.getPartition().getTable().getDatabase().getCatalogName(); + String dbName = mStatsObj.getPartition().getTable().getDatabase().getName(); + String tableName = mStatsObj.getPartition().getTable().getTableName(); + String partName = mStatsObj.getPartition().getPartitionName(); + String colName = mStatsObj.getColName(); + + Preconditions.checkState(this.currentTransaction.isActive()); + + LOG.info("Updating partition level column statistics for table=" + + TableName.getQualified(catName, dbName, tableName) + + " partName=" + partName + " colName=" + colName); + + boolean foundCol = false; + List colList = partition.getSd().getCols(); + for (FieldSchema col : colList) { + if (col.getName().equals(mStatsObj.getColName())) { + foundCol = true; + break; + } + } + + if (!foundCol) { + LOG.warn("Column " + colName + " for which stats gathering is requested doesn't exist."); + } + + if (oldStats != null) { + StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats); + } else { + pm.makePersistent(mStatsObj); + } + } + + @Override + public Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + boolean committed = false; + List statsObjs = colStats.getStatsObj(); + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + long start = System.currentTimeMillis(); + String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); + try { + openTransaction(); + // DataNucleus objects get detached all over the place for no (real) reason. + // So let's not use them anywhere unless absolutely necessary. + MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); + int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); + long sleepInterval = MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + Map result = new RetryingExecutor<>(maxRetries, () -> { + AtomicReference exceptionRef = new AtomicReference<>(); + String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); + setTransactionSavePoint(savePoint); + executePlainSQL( + sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()), + true, + exception -> { + rollbackTransactionToSavePoint(savePoint); + exceptionRef.set(exception); + }); + if (exceptionRef.get() != null) { + throw new RetryingExecutor.RetryException(exceptionRef.get()); + } + pm.refresh(mTable); + Table table = convertToTable(mTable, conf); + List colNames = new ArrayList<>(); + for (ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } + + Map oldStats = Maps.newHashMap(); + List stats = getMTableColumnStatistics(table, colNames, colStats.getEngine()); + for (MTableColumnStatistics cStat : stats) { + oldStats.put(cStat.getColName(), cStat); + } + + for (ColumnStatisticsObj statsObj : statsObjs) { + MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc, + statsObj, colStats.getEngine()); + writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); + // There is no need to add colname again, otherwise we will get duplicate colNames. + } + + // Set the table properties + // No need to check again if it exists. + String dbname = table.getDbName(); + String name = table.getTableName(); + MTable oldt = mTable; + Map newParams = new HashMap<>(table.getParameters()); + StatsSetupConst.setColumnStatsState(newParams, colNames); + boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters()); + if (isTxn) { + if (!areTxnStatsSupported) { + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + } else { + String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams, + writeId, validWriteIds, true); + if (errorMsg != null) { + throw new MetaException(errorMsg); + } + if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) { + // Make sure we set the flag to invalid regardless of the current value. + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name); + } + oldt.setWriteId(writeId); + } + } + oldt.setParameters(newParams); + return newParams; + }).onRetry(e -> e instanceof RetryingExecutor.RetryException) + .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval -> + ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); + committed = commitTransaction(); + return committed ? result : null; + } finally { + LOG.debug("{} updateTableColumnStatistics took {}ms, success: {}", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), + System.currentTimeMillis() - start, committed); + rollbackAndCleanup(committed, null); + } + } + + @Override + public Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats, + List partVals, String validWriteIds, long writeId) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + boolean committed = false; + long start = System.currentTimeMillis(); + List statsObjs = colStats.getStatsObj(); + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); + try { + openTransaction(); + MPartition mPartition = + ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals); + if (mPartition == null) { + throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); + } + + List colNames = new ArrayList<>(); + for(ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } + int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); + long sleepInterval = MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + Map result = new RetryingExecutor<>(maxRetries, () -> { + AtomicReference exceptionRef = new AtomicReference<>(); + String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); + setTransactionSavePoint(savePoint); + executePlainSQL(sqlGenerator.addForUpdateNoWait( + "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()), + true, + exception -> { + rollbackTransactionToSavePoint(savePoint); + exceptionRef.set(exception); + }); + if (exceptionRef.get() != null) { + throw new RetryingExecutor.RetryException(exceptionRef.get()); + } + pm.refresh(mPartition); + Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), + mPartition, TxnUtils.isAcidTable(table), conf); + Map oldStats = Maps.newHashMap(); + List stats = + getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine()); + for (MPartitionColumnStatistics cStat : stats) { + oldStats.put(cStat.getColName(), cStat); + } + + for (ColumnStatisticsObj statsObj : statsObjs) { + MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, + statsDesc, statsObj, colStats.getEngine()); + writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName())); + } + + Map newParams = new HashMap<>(mPartition.getParameters()); + StatsSetupConst.setColumnStatsState(newParams, colNames); + boolean isTxn = TxnUtils.isTransactionalTable(table); + if (isTxn) { + if (!areTxnStatsSupported) { + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + } else { + String errorMsg = verifyStatsChangeCtx( + TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(), + newParams, writeId, validWriteIds, true); + if (errorMsg != null) { + throw new MetaException(errorMsg); + } + if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) { + // Make sure we set the flag to invalid regardless of the current value. + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName()); + } + mPartition.setWriteId(writeId); + } + } + mPartition.setParameters(newParams); + return newParams; + }).onRetry(e -> e instanceof RetryingExecutor.RetryException) + .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval -> + ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); + committed = commitTransaction(); + return committed ? result : null; + } finally { + LOG.debug("{} updatePartitionColumnStatistics took {}ms, success: {}", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), + System.currentTimeMillis() - start, committed); + rollbackAndCleanup(committed, null); + } + } + + @Override + public Map updatePartitionColumnStatistics(ColumnStatistics colStats, + List partVals, String validWriteIds, long writeId) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + Table table = getTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); + MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); + return updatePartitionColumnStatistics(table, mTable, colStats, partVals, validWriteIds, writeId); + } + + @Override + public Map> updatePartitionColumnStatisticsInBatch( + Map partColStatsMap, + Table tbl, + List listeners, + String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + + return new ObjectStore.GetHelper>>(tbl.getCatName(), + tbl.getDbName(), tbl.getTableName(), true, false) { + @Override + protected String describeResult() { + return "Map of partition key to column stats if successful"; + } + @Override + protected Map> getSqlResult( + ObjectStore.GetHelper>> ctx) + throws MetaException { + return directSql.updatePartitionColumnStatisticsBatch(partColStatsMap, tbl, + listeners, validWriteIds, writeId); + } + @Override + protected Map> getJdoResult( + ObjectStore.GetHelper>> ctx) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled"); + } + }.run(false); + } + + private List getMTableColumnStatistics(Table table, List colNames, String engine) + throws MetaException { + + Preconditions.checkState(this.currentTransaction.isActive()); + + if (colNames.isEmpty()) { + return Collections.emptyList(); + } + + boolean committed = false; + try { + openTransaction(); + + validateTableCols(table, colNames); + + List result = Collections.emptyList(); + try (Query query = pm.newQuery(MTableColumnStatistics.class)) { + result = + Batchable.runBatched(batchSize, colNames, new Batchable() { + @Override + public List run(List input) + throws MetaException { + StringBuilder filter = + new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && ("); + StringBuilder paramStr = new StringBuilder( + "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); + Object[] params = new Object[input.size() + 4]; + params[0] = table.getTableName(); + params[1] = table.getDbName(); + params[2] = table.getCatName(); + params[3] = engine; + for (int i = 0; i < input.size(); ++i) { + filter.append((i == 0) ? "" : " || ").append("colName == c").append(i); + paramStr.append(", java.lang.String c").append(i); + params[i + 4] = input.get(i); + } + filter.append(")"); + query.setFilter(filter.toString()); + query.declareParameters(paramStr.toString()); + List paritial = (List) query.executeWithArray(params); + pm.retrieveAll(paritial); + return paritial; + } + }); + + if (result.size() > colNames.size()) { + throw new MetaException("Unexpected " + result.size() + " statistics for " + + colNames.size() + " columns"); + } + result = new ArrayList<>(result); + } + committed = commitTransaction(); + return result; + } catch (Exception ex) { + LOG.error("Error retrieving statistics via jdo", ex); + if (ex instanceof MetaException) { + throw (MetaException) ex; + } + throw new MetaException(ex.getMessage()); + } finally { + rollbackAndCleanup(committed, null); + } + } + + @Override + public List getTableColumnStatistics( + String catName, + String dbName, + String tableName, + List colNames) throws MetaException, NoSuchObjectException { + // Note: this will get stats without verifying ACID. + boolean committed = false; + Query query = null; + List result = new ArrayList<>(); + + try { + openTransaction(); + query = pm.newQuery(MTableColumnStatistics.class); + query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + query.setResult("DISTINCT engine"); + Collection names = (Collection) query.execute(tableName, dbName, catName); + List engines = new ArrayList<>(); + for (Iterator i = names.iterator(); i.hasNext();) { + engines.add((String) i.next()); + } + for (String e : engines) { + ColumnStatistics cs = getTableColumnStatisticsInternal( + catName, dbName, tableName, colNames, e, true, true); + if (cs != null) { + result.add(cs); + } + } + committed = commitTransaction(); + return result; + } finally { + LOG.debug("Done executing getTableColumnStatistics with status : {}", + committed); + rollbackAndCleanup(committed, query); + } + } + + @Override + public ColumnStatistics getTableColumnStatistics( + String catName, + String dbName, + String tableName, + List colNames, + String engine) throws MetaException, NoSuchObjectException { + // Note: this will get stats without verifying ACID. + return getTableColumnStatisticsInternal( + catName, dbName, tableName, colNames, engine, true, true); + } + + @Override + public ColumnStatistics getTableColumnStatistics( + String catName, + String dbName, + String tableName, + List colNames, + String engine, + String writeIdList) throws MetaException, NoSuchObjectException { + // If the current stats in the metastore doesn't comply with + // the isolation level of the query, set No to the compliance flag. + Boolean isCompliant = null; + if (writeIdList != null) { + MTable table = this.getMTable(catName, dbName, tableName); + if (table == null) { + throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tableName) + " table not found"); + } + isCompliant = !TxnUtils.isTransactionalTable(table.getParameters()) + || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table, writeIdList, false)); + } + ColumnStatistics stats = getTableColumnStatisticsInternal( + catName, dbName, tableName, colNames, engine, true, true); + if (stats != null && isCompliant != null) { + stats.setIsStatsCompliant(isCompliant); + } + return stats; + } + + protected ColumnStatistics getTableColumnStatisticsInternal( + String catName, String dbName, String tableName, final List colNames, String engine, + boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + return new ObjectStore.GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName), + normalizeIdentifier(tableName), allowSql, allowJdo, null) { + @Override + protected ColumnStatistics getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + return directSqlAggrStats.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll); + } + + @Override + protected ColumnStatistics getJdoResult(ObjectStore.GetHelper ctx) throws MetaException { + + List mStats = getMTableColumnStatistics(getTable(), colNames, engine); + if (mStats.isEmpty()) { + return null; + } + // LastAnalyzed is stored per column, but thrift object has it per + // multiple columns. Luckily, nobody actually uses it, so we will set to + // lowest value of all columns for now. + ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0)); + List statObjs = new ArrayList<>(mStats.size()); + for (MTableColumnStatistics mStat : mStats) { + if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) { + desc.setLastAnalyzed(mStat.getLastAnalyzed()); + } + statObjs.add(StatObjectConverter.getColumnStatisticsObj(mStat, enableBitVector, enableKll)); + Deadline.checkTimeout(); + } + ColumnStatistics colStat = new ColumnStatistics(desc, statObjs); + colStat.setEngine(engine); + return colStat; + } + }.run(true); + } + + @Override + public List> getPartitionColumnStatistics(String catName, String dbName, String tableName, + List partNames, List colNames) throws MetaException, NoSuchObjectException { + // Note: this will get stats without verifying ACID. + boolean committed = false; + Query query = null; + List> result = new ArrayList<>(); + + try { + openTransaction(); + query = pm.newQuery(MPartitionColumnStatistics.class); + query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + query.setResult("DISTINCT engine"); + Collection names = (Collection) query.execute(tableName, dbName, catName); + List engines = new ArrayList<>(); + for (Iterator i = names.iterator(); i.hasNext();) { + engines.add((String) i.next()); + } + for (String e : engines) { + List cs = getPartitionColumnStatisticsInternal( + catName, dbName, tableName, partNames, colNames, e, true, true); + if (cs != null) { + result.add(cs); + } + } + committed = commitTransaction(); + return result; + } finally { + LOG.debug("Done executing getTableColumnStatistics with status : {}", + committed); + rollbackAndCleanup(committed, query); + } + } + + @Override + public List getPartitionColumnStatistics(String catName, String dbName, String tableName, + List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException { + // Note: this will get stats without verifying ACID. + if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) { + LOG.debug("PartNames and/or ColNames are empty"); + return Collections.emptyList(); + } + return getPartitionColumnStatisticsInternal( + catName, dbName, tableName, partNames, colNames, engine, true, true); + } + + @Override + public List getPartitionColumnStatistics( + String catName, String dbName, String tableName, + List partNames, List colNames, + String engine, String writeIdList) + throws MetaException, NoSuchObjectException { + if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) { + LOG.debug("PartNames and/or ColNames are empty"); + return Collections.emptyList(); + } + List allStats = getPartitionColumnStatisticsInternal( + catName, dbName, tableName, partNames, colNames, engine, true, true); + if (writeIdList != null) { + if (!areTxnStatsSupported) { + for (ColumnStatistics cs : allStats) { + cs.setIsStatsCompliant(false); + } + } else { + // TODO: this could be improved to get partitions in bulk + for (ColumnStatistics cs : allStats) { + MPartition mpart = ensureGetMPartition(new TableName(catName, dbName, tableName), + Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName())); + if (mpart == null + || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) { + if (mpart != null) { + LOG.debug("The current metastore transactional partition column statistics for {}.{}.{} " + + "(write ID {}) are not valid for current query ({} {})", dbName, tableName, + mpart.getPartitionName(), mpart.getWriteId(), writeIdList); + } + cs.setIsStatsCompliant(false); + } else { + cs.setIsStatsCompliant(true); + } + } + } + } + return allStats; + } + + protected List getPartitionColumnStatisticsInternal( + String catName, String dbName, String tableName, final List partNames, final List colNames, + String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + return new ObjectStore.GetListHelper(catName, dbName, tableName, allowSql, allowJdo) { + @Override + protected List getSqlResult( + ObjectStore.GetHelper> ctx) throws MetaException { + return directSqlAggrStats.getPartitionStats( + catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll); + } + @Override + protected List getJdoResult(ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + List mStats = + getMPartitionColumnStatistics(getTable(), partNames, colNames, engine); + List result = new ArrayList<>(Math.min(mStats.size(), partNames.size())); + String lastPartName = null; + List curList = null; + ColumnStatisticsDesc csd = null; + for (int i = 0; i <= mStats.size(); ++i) { + boolean isLast = i == mStats.size(); + MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i); + String partName = isLast ? null : mStatsObj.getPartition().getPartitionName(); + if (isLast || !partName.equals(lastPartName)) { + if (i != 0) { + ColumnStatistics colStat = new ColumnStatistics(csd, curList); + colStat.setEngine(engine); + result.add(colStat); + } + if (isLast) { + continue; + } + csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj); + curList = new ArrayList<>(colNames.size()); + } + curList.add(StatObjectConverter.getColumnStatisticsObj(mStatsObj, enableBitVector, enableKll)); + lastPartName = partName; + Deadline.checkTimeout(); + } + return result; + } + }.run(true); + } + + @Override + public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, + final List partNames, final List colNames, + String engine, String writeIdList) throws MetaException, NoSuchObjectException { + // If the current stats in the metastore doesn't comply with + // the isolation level of the query, return null. + if (writeIdList != null) { + if (partNames == null || partNames.isEmpty()) { + return null; + } + + Table table = getTable(catName, dbName, tblName); + boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); + if (isTxn && !areTxnStatsSupported) { + return null; + } + GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec(); + fs.setFilterMode(PartitionFilterMode.BY_NAMES); + fs.setFilters(partNames); + GetProjectionsSpec ps = new GetProjectionsSpec(); + ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%'); + ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values")); + List parts = getPartitionSpecsByFilterAndProjection(table, ps, fs); + + // Loop through the given "partNames" list + // checking isolation-level-compliance of each partition column stats. + for (Partition part : parts) { + + if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + LOG.debug("The current metastore transactional partition column " + + "statistics for {}.{}.{} is not valid for the current query", + dbName, tblName, partName); + return null; + } + } + } + return get_aggr_stats_for(catName, dbName, tblName, partNames, colNames, engine); + } + + @Override + public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, + final List partNames, final List colNames, String engine) + throws MetaException, NoSuchObjectException { + final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(getConf(), + MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION); + final double ndvTuner = MetastoreConf.getDoubleVar(getConf(), MetastoreConf.ConfVars.STATS_NDV_TUNER); + final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + return new ObjectStore.GetHelper(catName, dbName, tblName, true, false) { + @Override + protected AggrStats getSqlResult(ObjectStore.GetHelper ctx) + throws MetaException { + return directSql.aggrColStatsForPartitions(catName, dbName, tblName, partNames, + colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll); + } + @Override + protected AggrStats getJdoResult(ObjectStore.GetHelper ctx) + throws MetaException, NoSuchObjectException { + // This is fast path for query optimizations, if we can find this info + // quickly using + // directSql, do it. No point in failing back to slow path here. + throw new MetaException("Jdo path is not implemented for stats aggr."); + } + @Override + protected String describeResult() { + return null; + } + }.run(true); + } + + @Override + public List getPartitionColStatsForDatabase(String catName, String dbName) + throws MetaException, NoSuchObjectException { + final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + return new ObjectStore.GetHelper>( + catName, dbName, null, true, false) { + @Override + protected List getSqlResult( + ObjectStore.GetHelper> ctx) throws MetaException { + return directSqlAggrStats.getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll); + } + + @Override + protected List getJdoResult( + ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + // This is fast path for query optimizations, if we can find this info + // quickly using directSql, do it. No point in failing back to slow path + // here. + throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase."); + } + + @Override + protected String describeResult() { + return null; + } + }.run(true); + } + + private List getMPartitionColumnStatistics(Table table, List partNames, + List colNames, String engine) throws MetaException { + boolean committed = false; + + try { + openTransaction(); + // We are not going to verify SD for each partition. Just verify for the + // table. TODO: we need verify the partition column instead + try { + validateTableCols(table, colNames); + } catch (MetaException me) { + LOG.warn("The table does not have the same column definition as its partition."); + } + List result = Collections.emptyList(); + try (Query query = pm.newQuery(MPartitionColumnStatistics.class)) { + String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"; + String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3 && engine == t4 && ("; + Object[] params = new Object[colNames.size() + partNames.size() + 4]; + int i = 0; + params[i++] = table.getTableName(); + params[i++] = table.getDbName(); + params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf); + params[i++] = engine; + int firstI = i; + for (String s : partNames) { + filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i; + paramStr += ", java.lang.String p" + i; + params[i++] = s; + } + filter += ") && ("; + firstI = i; + for (String s : colNames) { + filter += ((i == firstI) ? "" : " || ") + "colName == c" + i; + paramStr += ", java.lang.String c" + i; + params[i++] = s; + } + filter += ")"; + query.setFilter(filter); + query.declareParameters(paramStr); + query.setOrdering("partition.partitionName ascending"); + result = (List) query.executeWithArray(params); + pm.retrieveAll(result); + result = new ArrayList<>(result); + } catch (Exception ex) { + LOG.error("Error retrieving statistics via jdo", ex); + throw new MetaException(ex.getMessage()); + } + committed = commitTransaction(); + return result; + } finally { + if (!committed) { + rollbackTransaction(); + return Collections.emptyList(); + } + } + } + + @Override + public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) { + + String catName = tn.getCat(); + String dbName = tn.getDb(); + String tableName = tn.getTable(); + + Query query = null; + dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); + catName = normalizeIdentifier(catName); + if (tableName == null) { + throw new RuntimeException("Table name is null."); + } + boolean ret = false; + try { + openTransaction(); + MTable mTable = getMTable(catName, dbName, tableName); + + query = pm.newQuery(MPartitionColumnStatistics.class); + + String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4"; + String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4"; + + query.setFilter(filter); + query.declareParameters(parameters); + + Long number = query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), + normalizeIdentifier(catName)); + + new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { + private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown(); + + @Override + protected String describeResult() { + return "Partition count"; + } + + @Override + protected boolean canUseDirectSql(ObjectStore.GetHelper ctx) throws MetaException { + return true; + } + + @Override + protected Integer getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + directSql.deleteColumnStatsState(getTable().getId()); + return 0; + } + + @Override + protected Integer getJdoResult(ObjectStore.GetHelper ctx) throws MetaException, NoSuchObjectException { + try { + List parts = getPartitions(catName, dbName, tableName, + GetPartitionsArgs.getAllPartitions()); + for (Partition part : parts) { + Partition newPart = new Partition(part); + StatsSetupConst.clearColumnStatsState(newPart.getParameters()); + alterPartition(catName, dbName, tableName, part.getValues(), newPart, writeIdList); + } + return parts.size(); + } catch (InvalidObjectException e) { + LOG.error("error updating parts", e); + return -1; + } + } + }.run(true); + + ret = commitTransaction(); + } catch (Exception e) { + LOG.error("Couldn't clear stats for table", e); + } finally { + rollbackAndCleanup(ret, query); + } + } + + @Override + public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, + List partNames, List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + if (partNames == null || partNames.isEmpty()) { + throw new InvalidInputException("No partition specified for dropping the statistics"); + } + dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); + catName = normalizeIdentifier(catName); + List cols = normalizeIdentifiers(colNames); + return new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { + @Override + protected String describeResult() { + return "delete partition column stats"; + } + @Override + protected Boolean getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); + return deleteStats.deletePartitionColumnStats(catName, dbName, tableName, partNames, cols, engine); + } + @Override + protected Boolean getJdoResult(ObjectStore.GetHelper ctx) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + return deletePartitionColumnStatisticsViaJdo(catName, dbName, tableName, partNames, cols, engine); + } + }.run(false); + } + + private boolean deletePartitionColumnStatisticsViaJdo(String catName, String dbName, String tableName, + List partNames, List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + boolean ret = false; + String database = org.apache.commons.lang3.StringUtils.defaultString(dbName, + Warehouse.DEFAULT_DATABASE_NAME); + String catalog = normalizeIdentifier(catName); + try { + openTransaction(); + Batchable b = new Batchable() { + @Override + public List run(List input) throws Exception { + Query query = pm.newQuery(MPartitionColumnStatistics.class); + addQueryAfterUse(query); + String filter; + String parameters; + if (colNames != null && !colNames.isEmpty()) { + filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + + "t4.contains(colName) && partition.table.database.catalogName == t5" + (engine != null ? " && engine == t6" : ""); + parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, " + + "java.util.Collection t4, java.lang.String t5" + (engine != null ? ", java.lang.String t6" : ""); + } else { + filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + + "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : ""); + parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + (engine != null ? ", java.lang.String t5" : ""); + } + query.setFilter(filter); + query.declareParameters(parameters); + List params = new ArrayList<>(); + params.add(input); + params.add(normalizeIdentifier(database)); + params.add(normalizeIdentifier(tableName)); + if (colNames != null && !colNames.isEmpty()) { + params.add(colNames); + } + params.add(catalog); + if (engine != null) { + params.add(engine); + } + List mStatsObjColl = + (List) query.executeWithArray(params.toArray()); + pm.retrieveAll(mStatsObjColl); + if (mStatsObjColl != null) { + pm.deletePersistentAll(mStatsObjColl); + } + return null; + } + }; + try { + Batchable.runBatched(batchSize, partNames, b); + } finally { + b.closeAllQueries(); + } + + Batchable.runBatched(batchSize, partNames, new Batchable() { + @Override + public List run(List input) throws MetaException { + Pair> queryWithParams = getPartQueryWithParams(pm, catalog, database, tableName, + input); + try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) { + qw.setResultClass(MPartition.class); + qw.setClass(MPartition.class); + List mparts = (List) qw.executeWithMap(queryWithParams.getRight()); + for (MPartition mPart : mparts) { + Map params = mPart.getParameters(); + if (params != null && params.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { + if (colNames == null || colNames.isEmpty()) { + StatsSetupConst.clearColumnStatsState(params); + } else { + StatsSetupConst.removeColumnStatsState(params, colNames); + } + mPart.setParameters(params); + } + } + } + return Collections.emptyList(); + } + }); + ret = commitTransaction(); + } finally { + rollbackAndCleanup(ret, null); + } + return ret; + } + + @Override + public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, + List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); + if (tableName == null) { + throw new InvalidInputException("Table name is null."); + } + List cols = normalizeIdentifiers(colNames); + return new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { + @Override + protected String describeResult() { + return "delete table column stats"; + } + @Override + protected Boolean getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); + return deleteStats.deleteTableColumnStatistics(getTable(), cols, engine); + } + @Override + protected Boolean getJdoResult(ObjectStore.GetHelper ctx) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + return deleteTableColumnStatisticsViaJdo(catName, dbName, tableName, cols, engine); + } + }.run(true); + } + + private boolean deleteTableColumnStatisticsViaJdo(String catName, String dbName, String tableName, + List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + boolean ret = false; + Query query = null; + try { + openTransaction(); + List mStatsObjColl; + // Note: this does not verify ACID state; called internally when removing cols/etc. + // Also called via an unused metastore API that checks for ACID tables. + query = pm.newQuery(MTableColumnStatistics.class); + String filter; + String parameters; + if (colNames != null && !colNames.isEmpty()) { + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + (engine != null ? " && engine == t5" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + (engine != null ? ", java.lang.String t5" : ""); + } else { + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + (engine != null ? " && engine == t4" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + (engine != null ? ", java.lang.String t4" : ""); + } + + query.setFilter(filter); + query.declareParameters(parameters); + List params = new ArrayList<>(); + params.add(normalizeIdentifier(tableName)); + params.add(normalizeIdentifier(dbName)); + params.add(catName == null ? null : normalizeIdentifier(catName)); + if (colNames != null && !colNames.isEmpty()) { + params.add(colNames); + } + if (engine != null) { + params.add(engine); + } + mStatsObjColl = (List) query.executeWithArray(params.toArray()); + pm.retrieveAll(mStatsObjColl); + if (mStatsObjColl != null) { + pm.deletePersistentAll(mStatsObjColl); + } + + MTable mTable = getMTable(catName, dbName, tableName); + if (mTable != null) { + Map tableParams = mTable.getParameters(); + if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { + if (colNames == null || colNames.isEmpty()) { + StatsSetupConst.clearColumnStatsState(tableParams); + } else { + StatsSetupConst.removeColumnStatsState(tableParams, colNames); + } + mTable.setParameters(tableParams); + } + } + ret = commitTransaction(); + } finally { + rollbackAndCleanup(ret, query); + } + return ret; + } + + private void executePlainSQL(String sql, + boolean atLeastOneRecord, + Consumer exceptionConsumer) + throws SQLException, MetaException { + String s = dbType.getPrepareTxnStmt(); + assert pm.currentTransaction().isActive(); + JDOConnection jdoConn = pm.getDataStoreConnection(); + Connection conn = (Connection) jdoConn.getNativeConnection(); + try (Statement statement = conn.createStatement()) { + if (s != null) { + statement.execute(s); + } + try { + statement.execute(sql); + try (ResultSet rs = statement.getResultSet()) { + // sqlserver needs rs.next for validating the s4u nowait + if (atLeastOneRecord && !rs.next()) { + throw new MetaException("At least one record but none is returned from the query: " + sql); + } + } + } catch (SQLException e) { + if (exceptionConsumer != null) { + exceptionConsumer.accept(e); + } else { + throw e; + } + } + } finally { + jdoConn.close(); + } + } + + /** + * Return true if the current statistics in the Metastore is valid + * for the query of the given "txnId" and "queryValidWriteIdList". + * + * Note that a statistics entity is valid iff + * the stats is written by the current query or + * the conjunction of the following two are true: + * ~ COLUMN_STATE_ACCURATE(CSA) state is true + * ~ Isolation-level (snapshot) compliant with the query + * @param tbl MTable of the stats entity + * @param queryValidWriteIdList valid writeId list of the query + * @Precondition "tbl" should be retrieved from the TBLS table. + */ + private boolean isCurrentStatsValidForTheQuery(MTable tbl, String queryValidWriteIdList, + boolean isCompleteStatsWriter) throws MetaException { + return isCurrentStatsValidForTheQuery(tbl.getParameters(), tbl.getWriteId(), + queryValidWriteIdList, isCompleteStatsWriter); + } + + /** + * Return true if the current statistics in the Metastore is valid + * for the query of the given "txnId" and "queryValidWriteIdList". + * + * Note that a statistics entity is valid iff + * the stats is written by the current query or + * the conjunction of the following two are true: + * ~ COLUMN_STATE_ACCURATE(CSA) state is true + * ~ Isolation-level (snapshot) compliant with the query + * @param queryValidWriteIdList valid writeId list of the query + */ + // TODO: move to somewhere else + public static boolean isCurrentStatsValidForTheQuery( + Map statsParams, long statsWriteId, String queryValidWriteIdList, + boolean isCompleteStatsWriter) throws MetaException { + + // Note: can be changed to debug/info to verify the calls. + LOG.debug("isCurrentStatsValidForTheQuery with stats write ID {}; query {}; writer: {} params {}", + statsWriteId, queryValidWriteIdList, isCompleteStatsWriter, statsParams); + // return true since the stats does not seem to be transactional. + if (statsWriteId < 1) { + return true; + } + // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is + // written by an aborted transaction but TXNS has no entry for the transaction + // after compaction. Don't check for a complete stats writer - it may replace invalid stats. + if (!isCompleteStatsWriter && !StatsSetupConst.areBasicStatsUptoDate(statsParams)) { + return false; + } + + if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid. + ValidWriteIdList list4TheQuery = ValidReaderWriteIdList.fromValue(queryValidWriteIdList); + // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), + // that means it cannot possibly be a concurrent write. If it's not valid (we are not + // allowed to see it), that means it's either concurrent or aborted, same thing for us. + if (list4TheQuery.isWriteIdValid(statsWriteId)) { + return true; + } + // Updater is also allowed to overwrite stats from aborted txns, as long as they are not concurrent. + if (isCompleteStatsWriter && list4TheQuery.isWriteIdAborted(statsWriteId)) { + return true; + } + } + + return false; + } + + private abstract class GetStatHelper extends ObjectStore.GetHelper { + public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql, + boolean allowJdo, String writeIdList) throws MetaException { + super(catalogName, dbName, tblName, allowSql, allowJdo); + } + + @Override + protected String describeResult() { + return "statistics for " + (results == null ? 0 : results.getStatsObjSize()) + " columns"; + } + } + + @VisibleForTesting + public void validateTableCols(Table table, List colNames) throws MetaException { + List colList = table.getSd().getCols(); + for (String colName : colNames) { + boolean foundCol = false; + for (FieldSchema mCol : colList) { + if (mCol.getName().equals(colName)) { + foundCol = true; + break; + } + } + if (!foundCol) { + throw new MetaException("Column " + colName + " doesn't exist in table " + + table.getTableName() + " in database " + table.getDbName()); + } + } + } + +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java new file mode 100644 index 000000000000..67b129f40b77 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -0,0 +1,1399 @@ +package org.apache.hadoop.hive.metastore.metastore.impl; + +import javax.jdo.Query; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.api.AllTableConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest; +import org.apache.hadoop.hive.metastore.api.SQLAllTableConstraints; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; +import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; +import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; +import org.apache.hadoop.hive.metastore.model.MConstraint; +import org.apache.hadoop.hive.metastore.model.MFieldSchema; +import org.apache.hadoop.hive.metastore.model.MTable; + +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + +public class ConstraintStoreImpl extends RawStoreAware implements ConstraintStore { + + @Override + public SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) + throws InvalidObjectException, MetaException { + boolean success = false; + try { + openTransaction(); + createTable(tbl); + // Add constraints. + // We need not do a deep retrieval of the Table Column Descriptor while persisting the + // constraints since this transaction involving create table is not yet committed. + if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) { + constraints.setForeignKeys(addForeignKeys(constraints.getForeignKeys(), false, constraints.getPrimaryKeys(), + constraints.getUniqueConstraints())); + } + if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) { + constraints.setPrimaryKeys(addPrimaryKeys(constraints.getPrimaryKeys(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) { + constraints.setUniqueConstraints(addUniqueConstraints(constraints.getUniqueConstraints(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) { + constraints.setNotNullConstraints(addNotNullConstraints(constraints.getNotNullConstraints(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) { + constraints.setDefaultConstraints(addDefaultConstraints(constraints.getDefaultConstraints(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) { + constraints.setCheckConstraints(addCheckConstraints(constraints.getCheckConstraints(), false)); + } + success = commitTransaction(); + return constraints; + } finally { + rollbackAndCleanup(success, null); + } + } + + private List listAllTableConstraintsWithOptionalConstraintName( + String catName, String dbName, String tableName, String constraintname) { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tableName = normalizeIdentifier(tableName); + constraintname = constraintname!=null?normalizeIdentifier(constraintname):null; + List mConstraints = null; + List constraintNames = new ArrayList<>(); + + try (QueryWrapper queryForConstraintName = new QueryWrapper(pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where " + + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname && " + + "parentTable.database.catalogName == pcatname) || " + + "(childTable != null && childTable.tableName == ctblname &&" + + "childTable.database.name == cdbname && childTable.database.catalogName == ccatname)) " + + (constraintname != null ? " && constraintName == constraintname" : ""))); + QueryWrapper queryForMConstraint = new QueryWrapper(pm.newQuery(MConstraint.class))) { + + queryForConstraintName.declareParameters("java.lang.String ptblname, java.lang.String pdbname," + + "java.lang.String pcatname, java.lang.String ctblname, java.lang.String cdbname," + + "java.lang.String ccatname" + + (constraintname != null ? ", java.lang.String constraintname" : "")); + Collection constraintNamesColl = + constraintname != null ? + ((Collection) queryForConstraintName. + executeWithArray(tableName, dbName, catName, tableName, dbName, catName, constraintname)): + ((Collection) queryForConstraintName. + executeWithArray(tableName, dbName, catName, tableName, dbName, catName)); + for (Iterator i = constraintNamesColl.iterator(); i.hasNext();) { + String currName = (String) i.next(); + constraintNames.add(currName); + } + + queryForMConstraint.setFilter("param.contains(constraintName)"); + queryForMConstraint.declareParameters("java.util.Collection param"); + Collection constraints = (Collection)queryForMConstraint.execute(constraintNames); + mConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + mConstraints.add(currConstraint); + } + } + return mConstraints; + } + + private boolean constraintNameAlreadyExists(MTable table, String constraintName) { + boolean commited = false; + Query constraintExistsQuery = null; + String constraintNameIfExists = null; + try { + openTransaction(); + constraintName = normalizeIdentifier(constraintName); + constraintExistsQuery = pm.newQuery(MConstraint.class, + "parentTable == parentTableP && constraintName == constraintNameP"); + constraintExistsQuery.declareParameters("MTable parentTableP, java.lang.String constraintNameP"); + constraintExistsQuery.setUnique(true); + constraintExistsQuery.setResult("constraintName"); + constraintNameIfExists = (String) constraintExistsQuery.executeWithArray(table, constraintName); + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, constraintExistsQuery); + } + return constraintNameIfExists != null && !constraintNameIfExists.isEmpty(); + } + + private String generateConstraintName(MTable table, String... parameters) throws MetaException { + int hashcode = ArrayUtils.toString(parameters).hashCode() & 0xfffffff; + int counter = 0; + final int MAX_RETRIES = 10; + while (counter < MAX_RETRIES) { + String currName = (parameters.length == 0 ? "constraint_" : parameters[parameters.length-1]) + + "_" + hashcode + "_" + System.currentTimeMillis() + "_" + (counter++); + if (!constraintNameAlreadyExists(table, currName)) { + return currName; + } + } + throw new MetaException("Error while trying to generate the constraint name for " + ArrayUtils.toString(parameters)); + } + + @Override + public List addForeignKeys( + List fks) throws InvalidObjectException, MetaException { + return addForeignKeys(fks, true, null, null); + } + + //TODO: clean up this method + private List addForeignKeys(List foreignKeys, boolean retrieveCD, + List primaryKeys, List uniqueConstraints) + throws InvalidObjectException, MetaException { + if (CollectionUtils.isNotEmpty(foreignKeys)) { + List mpkfks = new ArrayList<>(); + String currentConstraintName = null; + String catName = null; + // We start iterating through the foreign keys. This list might contain more than a single + // foreign key, and each foreign key might contain multiple columns. The outer loop retrieves + // the information that is common for a single key (table information) while the inner loop + // checks / adds information about each column. + for (int i = 0; i < foreignKeys.size(); i++) { + if (catName == null) { + catName = normalizeIdentifier(foreignKeys.get(i).isSetCatName() ? foreignKeys.get(i).getCatName() : + getDefaultCatalog(conf)); + } else { + String tmpCatName = normalizeIdentifier(foreignKeys.get(i).isSetCatName() ? + foreignKeys.get(i).getCatName() : getDefaultCatalog(conf)); + if (!catName.equals(tmpCatName)) { + throw new InvalidObjectException("Foreign keys cannot span catalogs"); + } + } + final String fkTableDB = normalizeIdentifier(foreignKeys.get(i).getFktable_db()); + final String fkTableName = normalizeIdentifier(foreignKeys.get(i).getFktable_name()); + // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. + // For instance, this is the case when we are creating the table. + final ObjectStore.AttachedMTableInfo nChildTable = getMTable(catName, fkTableDB, fkTableName, retrieveCD); + final MTable childTable = nChildTable.mtbl; + if (childTable == null) { + throw new InvalidObjectException("Child table not found: " + fkTableName); + } + MColumnDescriptor childCD = retrieveCD ? nChildTable.mcd : childTable.getSd().getCD(); + final List childCols = childCD == null || childCD.getCols() == null ? + new ArrayList<>() : new ArrayList<>(childCD.getCols()); + if (childTable.getPartitionKeys() != null) { + childCols.addAll(childTable.getPartitionKeys()); + } + + final String pkTableDB = normalizeIdentifier(foreignKeys.get(i).getPktable_db()); + final String pkTableName = normalizeIdentifier(foreignKeys.get(i).getPktable_name()); + // For primary keys, we retrieve the column descriptors if retrieveCD is true (which means + // it is an alter table statement) or if it is a create table statement but we are + // referencing another table instead of self for the primary key. + final ObjectStore.AttachedMTableInfo nParentTable; + final MTable parentTable; + MColumnDescriptor parentCD; + final List parentCols; + final List existingTablePrimaryKeys; + final List existingTableUniqueConstraints; + final boolean sameTable = fkTableDB.equals(pkTableDB) && fkTableName.equals(pkTableName); + if (sameTable) { + nParentTable = nChildTable; + parentTable = childTable; + parentCD = childCD; + parentCols = childCols; + existingTablePrimaryKeys = primaryKeys; + existingTableUniqueConstraints = uniqueConstraints; + } else { + nParentTable = getMTable(catName, pkTableDB, pkTableName, true); + parentTable = nParentTable.mtbl; + if (parentTable == null) { + throw new InvalidObjectException("Parent table not found: " + pkTableName); + } + parentCD = nParentTable.mcd; + parentCols = parentCD == null || parentCD.getCols() == null ? + new ArrayList<>() : new ArrayList<>(parentCD.getCols()); + if (parentTable.getPartitionKeys() != null) { + parentCols.addAll(parentTable.getPartitionKeys()); + } + PrimaryKeysRequest primaryKeysRequest = new PrimaryKeysRequest(pkTableDB, pkTableName); + primaryKeysRequest.setCatName(catName); + existingTablePrimaryKeys = getPrimaryKeys(primaryKeysRequest); + existingTableUniqueConstraints = + getUniqueConstraints(new UniqueConstraintsRequest(catName, pkTableDB, pkTableName)); + } + + // Here we build an aux structure that is used to verify that the foreign key that is declared + // is actually referencing a valid primary key or unique key. We also check that the types of + // the columns correspond. + if (existingTablePrimaryKeys.isEmpty() && existingTableUniqueConstraints.isEmpty()) { + throw new MetaException( + "Trying to define foreign key but there are no primary keys or unique keys for referenced table"); + } + final Set validPKsOrUnique = generateValidPKsOrUniqueSignatures(parentCols, + existingTablePrimaryKeys, existingTableUniqueConstraints); + + StringBuilder fkSignature = new StringBuilder(); + StringBuilder referencedKSignature = new StringBuilder(); + for (; i < foreignKeys.size(); i++) { + SQLForeignKey foreignKey = foreignKeys.get(i); + final String fkColumnName = normalizeIdentifier(foreignKey.getFkcolumn_name()); + int childIntegerIndex = getColumnIndexFromTableColumns(childCD.getCols(), fkColumnName); + if (childIntegerIndex == -1) { + if (childTable.getPartitionKeys() != null) { + childCD = null; + childIntegerIndex = getColumnIndexFromTableColumns(childTable.getPartitionKeys(), fkColumnName); + } + if (childIntegerIndex == -1) { + throw new InvalidObjectException("Child column not found: " + fkColumnName); + } + } + + final String pkColumnName = normalizeIdentifier(foreignKey.getPkcolumn_name()); + int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD.getCols(), pkColumnName); + if (parentIntegerIndex == -1) { + if (parentTable.getPartitionKeys() != null) { + parentCD = null; + parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), pkColumnName); + } + if (parentIntegerIndex == -1) { + throw new InvalidObjectException("Parent column not found: " + pkColumnName); + } + } + + if (foreignKey.getFk_name() == null) { + // When there is no explicit foreign key name associated with the constraint and the key is composite, + // we expect the foreign keys to be send in order in the input list. + // Otherwise, the below code will break. + // If this is the first column of the FK constraint, generate the foreign key name + // NB: The below code can result in race condition where duplicate names can be generated (in theory). + // However, this scenario can be ignored for practical purposes because of + // the uniqueness of the generated constraint name. + if (foreignKey.getKey_seq() == 1) { + currentConstraintName = generateConstraintName(parentTable, fkTableDB, fkTableName, pkTableDB, + pkTableName, pkColumnName, fkColumnName, "fk"); + } + } else { + currentConstraintName = normalizeIdentifier(foreignKey.getFk_name()); + if (constraintNameAlreadyExists(parentTable, currentConstraintName)) { + String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), + parentTable.getTableName(), currentConstraintName); + throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); + } + } + // Update Column, keys, table, database, catalog name + foreignKey.setFk_name(currentConstraintName); + foreignKey.setCatName(catName); + foreignKey.setFktable_db(fkTableDB); + foreignKey.setFktable_name(fkTableName); + foreignKey.setPktable_db(pkTableDB); + foreignKey.setPktable_name(pkTableName); + foreignKey.setFkcolumn_name(fkColumnName); + foreignKey.setPkcolumn_name(pkColumnName); + + Integer updateRule = foreignKey.getUpdate_rule(); + Integer deleteRule = foreignKey.getDelete_rule(); + int enableValidateRely = (foreignKey.isEnable_cstr() ? 4 : 0) + + (foreignKey.isValidate_cstr() ? 2 : 0) + (foreignKey.isRely_cstr() ? 1 : 0); + + MConstraint mpkfk = new MConstraint( + currentConstraintName, + foreignKey.getKey_seq(), + MConstraint.FOREIGN_KEY_CONSTRAINT, + deleteRule, + updateRule, + enableValidateRely, + parentTable, + childTable, + parentCD, + childCD, + childIntegerIndex, + parentIntegerIndex + ); + mpkfks.add(mpkfk); + + final String fkColType = getColumnFromTableColumns(childCols, fkColumnName).getType(); + fkSignature.append( + generateColNameTypeSignature(fkColumnName, fkColType)); + referencedKSignature.append( + generateColNameTypeSignature(pkColumnName, fkColType)); + + if (i + 1 < foreignKeys.size() && foreignKeys.get(i + 1).getKey_seq() == 1) { + // Next one is a new key, we bail out from the inner loop + break; + } + } + String referenced = referencedKSignature.toString(); + if (!validPKsOrUnique.contains(referenced)) { + throw new MetaException( + "Foreign key references " + referenced + " but no corresponding " + + "primary key or unique key exists. Possible keys: " + validPKsOrUnique); + } + if (sameTable && fkSignature.toString().equals(referenced)) { + throw new MetaException( + "Cannot be both foreign key and primary/unique key on same table: " + referenced); + } + fkSignature = new StringBuilder(); + referencedKSignature = new StringBuilder(); + } + pm.makePersistentAll(mpkfks); + + } + return foreignKeys; + } + + private static Set generateValidPKsOrUniqueSignatures(List tableCols, + List refTablePrimaryKeys, List refTableUniqueConstraints) { + final Set validPKsOrUnique = new HashSet<>(); + if (!refTablePrimaryKeys.isEmpty()) { + refTablePrimaryKeys.sort((o1, o2) -> { + int keyNameComp = o1.getPk_name().compareTo(o2.getPk_name()); + if (keyNameComp == 0) { + return Integer.compare(o1.getKey_seq(), o2.getKey_seq()); + } + return keyNameComp; + }); + StringBuilder pkSignature = new StringBuilder(); + for (SQLPrimaryKey pk : refTablePrimaryKeys) { + pkSignature.append( + generateColNameTypeSignature( + pk.getColumn_name(), getColumnFromTableColumns(tableCols, pk.getColumn_name()).getType())); + } + validPKsOrUnique.add(pkSignature.toString()); + } + if (!refTableUniqueConstraints.isEmpty()) { + refTableUniqueConstraints.sort((o1, o2) -> { + int keyNameComp = o1.getUk_name().compareTo(o2.getUk_name()); + if (keyNameComp == 0) { + return Integer.compare(o1.getKey_seq(), o2.getKey_seq()); + } + return keyNameComp; + }); + StringBuilder ukSignature = new StringBuilder(); + for (int j = 0; j < refTableUniqueConstraints.size(); j++) { + SQLUniqueConstraint uk = refTableUniqueConstraints.get(j); + ukSignature.append( + generateColNameTypeSignature( + uk.getColumn_name(), getColumnFromTableColumns(tableCols, uk.getColumn_name()).getType())); + if (j + 1 < refTableUniqueConstraints.size()) { + if (!refTableUniqueConstraints.get(j + 1).getUk_name().equals( + refTableUniqueConstraints.get(j).getUk_name())) { + validPKsOrUnique.add(ukSignature.toString()); + ukSignature = new StringBuilder(); + } + } else { + validPKsOrUnique.add(ukSignature.toString()); + } + } + } + return validPKsOrUnique; + } + + private static String generateColNameTypeSignature(String colName, String colType) { + return colName + ":" + colType + ";"; + } + + @Override + public List addPrimaryKeys(List pks) throws InvalidObjectException, + MetaException { + return addPrimaryKeys(pks, true); + } + + private List addPrimaryKeys(List pks, boolean retrieveCD) throws InvalidObjectException, + MetaException { + List mpks = new ArrayList<>(); + String constraintName = null; + + for (SQLPrimaryKey pk : pks) { + final String catName = normalizeIdentifier(pk.getCatName()); + final String tableDB = normalizeIdentifier(pk.getTable_db()); + final String tableName = normalizeIdentifier(pk.getTable_name()); + final String columnName = normalizeIdentifier(pk.getColumn_name()); + + // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. + // For instance, this is the case when we are creating the table. + ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + MTable parentTable = nParentTable.mtbl; + if (parentTable == null) { + throw new InvalidObjectException("Parent table not found: " + tableName); + } + + MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); + int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + if (parentIntegerIndex == -1) { + if (parentTable.getPartitionKeys() != null) { + parentCD = null; + parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); + } + if (parentIntegerIndex == -1) { + throw new InvalidObjectException("Parent column not found: " + columnName); + } + } + if (getPrimaryKeyConstraintName(parentTable.getDatabase().getCatalogName(), + parentTable.getDatabase().getName(), parentTable.getTableName()) != null) { + throw new MetaException(" Primary key already exists for: " + + TableName.getQualified(catName, tableDB, tableName)); + } + if (pk.getPk_name() == null) { + if (pk.getKey_seq() == 1) { + constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "pk"); + } + } else { + constraintName = normalizeIdentifier(pk.getPk_name()); + if (constraintNameAlreadyExists(parentTable, constraintName)) { + String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), + parentTable.getTableName(), constraintName); + throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); + } + } + + int enableValidateRely = (pk.isEnable_cstr() ? 4 : 0) + + (pk.isValidate_cstr() ? 2 : 0) + (pk.isRely_cstr() ? 1 : 0); + MConstraint mpk = new MConstraint( + constraintName, + pk.getKey_seq(), + MConstraint.PRIMARY_KEY_CONSTRAINT, + null, + null, + enableValidateRely, + parentTable, + null, + parentCD, + null, + null, + parentIntegerIndex); + mpks.add(mpk); + + // Add normalized identifier back to result + pk.setCatName(catName); + pk.setTable_db(tableDB); + pk.setTable_name(tableName); + pk.setColumn_name(columnName); + pk.setPk_name(constraintName); + } + pm.makePersistentAll(mpks); + return pks; + } + + @Override + public List addUniqueConstraints(List uks) + throws InvalidObjectException, MetaException { + return addUniqueConstraints(uks, true); + } + + private List addUniqueConstraints(List uks, boolean retrieveCD) + throws InvalidObjectException, MetaException { + + List cstrs = new ArrayList<>(); + String constraintName = null; + + for (SQLUniqueConstraint uk : uks) { + final String catName = normalizeIdentifier(uk.getCatName()); + final String tableDB = normalizeIdentifier(uk.getTable_db()); + final String tableName = normalizeIdentifier(uk.getTable_name()); + final String columnName = normalizeIdentifier(uk.getColumn_name()); + + // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. + // For instance, this is the case when we are creating the table. + ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + MTable parentTable = nParentTable.mtbl; + if (parentTable == null) { + throw new InvalidObjectException("Parent table not found: " + tableName); + } + + MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); + int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + if (parentIntegerIndex == -1) { + if (parentTable.getPartitionKeys() != null) { + parentCD = null; + parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); + } + if (parentIntegerIndex == -1) { + throw new InvalidObjectException("Parent column not found: " + columnName); + } + } + if (uk.getUk_name() == null) { + if (uk.getKey_seq() == 1) { + constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "uk"); + } + } else { + constraintName = normalizeIdentifier(uk.getUk_name()); + if (constraintNameAlreadyExists(parentTable, constraintName)) { + String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), + parentTable.getTableName(), constraintName); + throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); + } + } + + + int enableValidateRely = (uk.isEnable_cstr() ? 4 : 0) + + (uk.isValidate_cstr() ? 2 : 0) + (uk.isRely_cstr() ? 1 : 0); + MConstraint muk = new MConstraint( + constraintName, + uk.getKey_seq(), + MConstraint.UNIQUE_CONSTRAINT, + null, + null, + enableValidateRely, + parentTable, + null, + parentCD, + null, + null, + parentIntegerIndex); + cstrs.add(muk); + + // Add normalized identifier back to result + uk.setCatName(catName); + uk.setTable_db(tableDB); + uk.setTable_name(tableName); + uk.setColumn_name(columnName); + uk.setUk_name(constraintName); + + } + pm.makePersistentAll(cstrs); + return uks; + } + + @Override + public List addNotNullConstraints(List nns) + throws InvalidObjectException, MetaException { + return addNotNullConstraints(nns, true); + } + + @Override + public List addDefaultConstraints(List nns) + throws InvalidObjectException, MetaException { + return addDefaultConstraints(nns, true); + } + + @Override + public List addCheckConstraints(List nns) + throws InvalidObjectException, MetaException { + return addCheckConstraints(nns, true); + } + + private List addCheckConstraints(List ccs, boolean retrieveCD) + throws InvalidObjectException, MetaException { + List cstrs = new ArrayList<>(); + + for (SQLCheckConstraint cc: ccs) { + final String catName = normalizeIdentifier(cc.getCatName()); + final String tableDB = normalizeIdentifier(cc.getTable_db()); + final String tableName = normalizeIdentifier(cc.getTable_name()); + final String columnName = cc.getColumn_name() == null? null + : normalizeIdentifier(cc.getColumn_name()); + final String ccName = cc.getDc_name(); + boolean isEnable = cc.isEnable_cstr(); + boolean isValidate = cc.isValidate_cstr(); + boolean isRely = cc.isRely_cstr(); + String constraintValue = cc.getCheck_expression(); + MConstraint muk = addConstraint(catName, tableDB, tableName, columnName, ccName, isEnable, isRely, isValidate, + MConstraint.CHECK_CONSTRAINT, constraintValue, retrieveCD); + cstrs.add(muk); + + // Add normalized identifier back to result + cc.setCatName(catName); + cc.setTable_db(tableDB); + cc.setTable_name(tableName); + cc.setColumn_name(columnName); + cc.setDc_name(muk.getConstraintName()); + } + pm.makePersistentAll(cstrs); + return ccs; + } + + private MConstraint addConstraint(String catName, String tableDB, String tableName, String columnName, String ccName, + boolean isEnable, boolean isRely, boolean isValidate, int constraintType, + String constraintValue, boolean retrieveCD) + throws InvalidObjectException, MetaException { + String constraintName = null; + // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. + // For instance, this is the case when we are creating the table. + ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + MTable parentTable = nParentTable.mtbl; + if (parentTable == null) { + throw new InvalidObjectException("Parent table not found: " + tableName); + } + + MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); + int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + if (parentIntegerIndex == -1) { + if (parentTable.getPartitionKeys() != null) { + parentCD = null; + parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); + } + } + if (ccName == null) { + constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "dc"); + } else { + constraintName = normalizeIdentifier(ccName); + if (constraintNameAlreadyExists(parentTable, constraintName)) { + String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), + parentTable.getTableName(), constraintName); + throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); + } + } + + int enableValidateRely = (isEnable ? 4 : 0) + + (isValidate ? 2 : 0) + (isRely ? 1 : 0); + MConstraint muk = new MConstraint( + constraintName, + 1, + constraintType, // Not null constraint should reference a single column + null, + null, + enableValidateRely, + parentTable, + null, + parentCD, + null, + null, + parentIntegerIndex, + constraintValue); + + return muk; + } + + private List addDefaultConstraints(List dcs, boolean retrieveCD) + throws InvalidObjectException, MetaException { + + List cstrs = new ArrayList<>(); + for (SQLDefaultConstraint dc : dcs) { + final String catName = normalizeIdentifier(dc.getCatName()); + final String tableDB = normalizeIdentifier(dc.getTable_db()); + final String tableName = normalizeIdentifier(dc.getTable_name()); + final String columnName = normalizeIdentifier(dc.getColumn_name()); + final String dcName = dc.getDc_name(); + boolean isEnable = dc.isEnable_cstr(); + boolean isValidate = dc.isValidate_cstr(); + boolean isRely = dc.isRely_cstr(); + String constraintValue = dc.getDefault_value(); + MConstraint muk = addConstraint(catName, tableDB, tableName, columnName, dcName, isEnable, isRely, isValidate, + MConstraint.DEFAULT_CONSTRAINT, constraintValue, retrieveCD); + cstrs.add(muk); + + // Add normalized identifier back to result + dc.setCatName(catName); + dc.setTable_db(tableDB); + dc.setTable_name(tableName); + dc.setColumn_name(columnName); + dc.setDc_name(muk.getConstraintName()); + } + pm.makePersistentAll(cstrs); + return dcs; + } + + private List addNotNullConstraints(List nns, boolean retrieveCD) + throws InvalidObjectException, MetaException { + + List cstrs = new ArrayList<>(); + String constraintName; + + for (SQLNotNullConstraint nn : nns) { + final String catName = normalizeIdentifier(nn.getCatName()); + final String tableDB = normalizeIdentifier(nn.getTable_db()); + final String tableName = normalizeIdentifier(nn.getTable_name()); + final String columnName = normalizeIdentifier(nn.getColumn_name()); + + // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. + // For instance, this is the case when we are creating the table. + ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + MTable parentTable = nParentTable.mtbl; + if (parentTable == null) { + throw new InvalidObjectException("Parent table not found: " + tableName); + } + + MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); + int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + if (parentIntegerIndex == -1) { + if (parentTable.getPartitionKeys() != null) { + parentCD = null; + parentIntegerIndex = getColumnIndexFromTableColumns(parentTable.getPartitionKeys(), columnName); + } + if (parentIntegerIndex == -1) { + throw new InvalidObjectException("Parent column not found: " + columnName); + } + } + if (nn.getNn_name() == null) { + constraintName = generateConstraintName(parentTable, tableDB, tableName, columnName, "nn"); + } else { + constraintName = normalizeIdentifier(nn.getNn_name()); + if (constraintNameAlreadyExists(parentTable, constraintName)) { + String fqConstraintName = String.format("%s.%s.%s", parentTable.getDatabase().getName(), + parentTable.getTableName(), constraintName); + throw new InvalidObjectException("Constraint name already exists: " + fqConstraintName); + } + } + + int enableValidateRely = (nn.isEnable_cstr() ? 4 : 0) + + (nn.isValidate_cstr() ? 2 : 0) + (nn.isRely_cstr() ? 1 : 0); + MConstraint muk = new MConstraint( + constraintName, + 1, + MConstraint.NOT_NULL_CONSTRAINT, // Not null constraint should reference a single column + null, + null, + enableValidateRely, + parentTable, + null, + parentCD, + null, + null, + parentIntegerIndex); + cstrs.add(muk); + // Add normalized identifier back to result + nn.setCatName(catName); + nn.setTable_db(tableDB); + nn.setTable_name(tableName); + nn.setColumn_name(columnName); + nn.setNn_name(constraintName); + } + pm.makePersistentAll(cstrs); + return nns; + } + + @Override + public List getPrimaryKeys(PrimaryKeysRequest request) throws MetaException { + try { + return getPrimaryKeysInternal(request.getCatName(), + request.getDb_name(),request.getTbl_name()); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + private List getPrimaryKeysInternal(final String catName, + final String dbNameInput, + final String tblNameInput) + throws MetaException, NoSuchObjectException { + final String dbName = dbNameInput != null ? normalizeIdentifier(dbNameInput) : null; + final String tblName = normalizeIdentifier(tblNameInput); + return new ObjectStore.GetListHelper(catName, dbName, tblName, true, true) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) throws MetaException { + return directSql.getPrimaryKeys(catName, dbName, tblName); + } + + @Override + protected List getJdoResult( + ObjectStore.GetHelper> ctx) throws MetaException, NoSuchObjectException { + return getPrimaryKeysViaJdo(catName, dbName, tblName); + } + }.run(false); + } + + private List getPrimaryKeysViaJdo(String catName, String dbName, String tblName) { + boolean commited = false; + List primaryKeys = null; + Query query = null; + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == cat_name &&" + + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + + "java.lang.String cat_name"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + primaryKeys = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPK = (MConstraint) i.next(); + List cols = currPK.getParentColumn() != null ? + currPK.getParentColumn().getCols() : currPK.getParentTable().getPartitionKeys(); + int enableValidateRely = currPK.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, + tblName, + cols.get(currPK.getParentIntegerIndex()).getName(), + currPK.getPosition(), + currPK.getConstraintName(), enable, validate, rely); + keyCol.setCatName(catName); + primaryKeys.add(keyCol); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return primaryKeys; + } + + private String getPrimaryKeyConstraintName(String catName, String dbName, String tblName) { + boolean commited = false; + String ret = null; + Query query = null; + + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + + "java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPK = (MConstraint) i.next(); + ret = currPK.getConstraintName(); + break; + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return ret; + } + + @Override + public List getForeignKeys(ForeignKeysRequest request) throws MetaException { + try { + return getForeignKeysInternal(request.getCatName(), + request.getParent_db_name(), request.getParent_tbl_name() , + request.getForeign_db_name(),request.getForeign_tbl_name(), true, + true); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + private List getForeignKeysInternal( + final String catName, final String parent_db_name_input, final String parent_tbl_name_input, + final String foreign_db_name_input, final String foreign_tbl_name_input, boolean allowSql, + boolean allowJdo) throws MetaException, NoSuchObjectException { + final String parent_db_name = (parent_db_name_input != null) ? normalizeIdentifier(parent_db_name_input) : null; + final String parent_tbl_name = (parent_tbl_name_input != null) ? normalizeIdentifier(parent_tbl_name_input) : null; + final String foreign_db_name = (foreign_db_name_input != null) ? normalizeIdentifier(foreign_db_name_input) : null; + final String foreign_tbl_name = (foreign_tbl_name_input != null) + ? normalizeIdentifier(foreign_tbl_name_input) : null; + final String db_name; + final String tbl_name; + if (foreign_tbl_name == null) { + // The FK table name might be null if we are retrieving the constraint from the PK side + db_name = parent_db_name; + tbl_name = parent_tbl_name; + } else { + db_name = foreign_db_name; + tbl_name = foreign_tbl_name; + } + return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) throws MetaException { + return directSql.getForeignKeys(catName, parent_db_name, + parent_tbl_name, foreign_db_name, foreign_tbl_name); + } + + @Override + protected List getJdoResult( + ObjectStore.GetHelper> ctx) throws MetaException, NoSuchObjectException { + return getForeignKeysViaJdo(catName, parent_db_name, + parent_tbl_name, foreign_db_name, foreign_tbl_name); + } + }.run(false); + } + + private List getForeignKeysViaJdo(String catName, String parentDbName, + String parentTblName, String foreignDbName, String foreignTblName) { + boolean commited = false; + List foreignKeys = null; + Collection constraints = null; + Query query = null; + Map tblToConstraint = new HashMap<>(); + try { + openTransaction(); + String queryText = + " parentTable.database.catalogName == catName1 &&" + "childTable.database.catalogName == catName2 && " + ( + parentTblName != null ? "parentTable.tableName == parent_tbl_name && " : "") + ( + parentDbName != null ? " parentTable.database.name == parent_db_name && " : "") + ( + foreignTblName != null ? " childTable.tableName == foreign_tbl_name && " : "") + ( + foreignDbName != null ? " childTable.database.name == foreign_db_name && " : "") + + " constraintType == MConstraint.FOREIGN_KEY_CONSTRAINT"; + queryText = queryText.trim(); + query = pm.newQuery(MConstraint.class, queryText); + String paramText = "java.lang.String catName1, java.lang.String catName2" + ( + parentTblName == null ? "" : ", java.lang.String parent_tbl_name") + ( + parentDbName == null ? "" : " , java.lang.String parent_db_name") + ( + foreignTblName == null ? "" : ", java.lang.String foreign_tbl_name") + ( + foreignDbName == null ? "" : " , java.lang.String foreign_db_name"); + query.declareParameters(paramText); + List params = new ArrayList<>(); + params.add(catName); + params.add(catName); // This is not a mistake, catName is in the where clause twice + if (parentTblName != null) { + params.add(parentTblName); + } + if (parentDbName != null) { + params.add(parentDbName); + } + if (foreignTblName != null) { + params.add(foreignTblName); + } + if (foreignDbName != null) { + params.add(foreignDbName); + } + constraints = (Collection) query.executeWithArray(params.toArray(new String[0])); + + pm.retrieveAll(constraints); + foreignKeys = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPKFK = (MConstraint) i.next(); + List parentCols = currPKFK.getParentColumn() != null ? + currPKFK.getParentColumn().getCols() : currPKFK.getParentTable().getPartitionKeys(); + List childCols = currPKFK.getChildColumn() != null ? + currPKFK.getChildColumn().getCols() : currPKFK.getChildTable().getPartitionKeys(); + int enableValidateRely = currPKFK.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + String consolidatedtblName = + currPKFK.getParentTable().getDatabase().getName() + "." + + currPKFK.getParentTable().getTableName(); + String pkName; + if (tblToConstraint.containsKey(consolidatedtblName)) { + pkName = tblToConstraint.get(consolidatedtblName); + } else { + pkName = getPrimaryKeyConstraintName(currPKFK.getParentTable().getDatabase().getCatalogName(), + currPKFK.getParentTable().getDatabase().getName(), + currPKFK.getParentTable().getTableName()); + tblToConstraint.put(consolidatedtblName, pkName); + } + SQLForeignKey fk = new SQLForeignKey( + currPKFK.getParentTable().getDatabase().getName(), + currPKFK.getParentTable().getTableName(), + parentCols.get(currPKFK.getParentIntegerIndex()).getName(), + currPKFK.getChildTable().getDatabase().getName(), + currPKFK.getChildTable().getTableName(), + childCols.get(currPKFK.getChildIntegerIndex()).getName(), + currPKFK.getPosition(), + currPKFK.getUpdateRule(), + currPKFK.getDeleteRule(), + currPKFK.getConstraintName(), pkName, enable, validate, rely); + fk.setCatName(catName); + foreignKeys.add(fk); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return foreignKeys; + } + + @Override + public List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException { + try { + return getUniqueConstraintsInternal(request.getCatName(), + request.getDb_name(),request.getTbl_name(), true, true); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + private List getUniqueConstraintsInternal( + String catNameInput, final String db_name_input, final String tbl_name_input, + boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + final String catName = normalizeIdentifier(catNameInput); + final String db_name = normalizeIdentifier(db_name_input); + final String tbl_name = normalizeIdentifier(tbl_name_input); + return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) + throws MetaException { + return directSql.getUniqueConstraints(catName, db_name, tbl_name); + } + + @Override + protected List getJdoResult(ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + return getUniqueConstraintsViaJdo(catName, db_name, tbl_name); + } + }.run(false); + } + + private List getUniqueConstraintsViaJdo(String catName, String dbName, String tblName) { + boolean commited = false; + List uniqueConstraints = null; + Query query = null; + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name && parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.UNIQUE_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + uniqueConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + uniqueConstraints.add(new SQLUniqueConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getPosition(), + currConstraint.getConstraintName(), enable, validate, rely)); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return uniqueConstraints; + } + + @Override + public List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException { + try { + return getNotNullConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + @Override + public List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException { + try { + return getDefaultConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + @Override + public List getCheckConstraints(CheckConstraintsRequest request) throws MetaException { + try { + return getCheckConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + } catch (NoSuchObjectException e) { + throw new MetaException(ExceptionUtils.getStackTrace(e)); + } + } + + private List getDefaultConstraintsInternal( + String catName, final String db_name_input, final String tbl_name_input, boolean allowSql, + boolean allowJdo) throws MetaException, NoSuchObjectException { + catName = normalizeIdentifier(catName); + final String db_name = normalizeIdentifier(db_name_input); + final String tbl_name = normalizeIdentifier(tbl_name_input); + return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) + throws MetaException { + return directSql.getDefaultConstraints(catName, db_name, tbl_name); + } + + @Override + protected List getJdoResult(ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + return getDefaultConstraintsViaJdo(catName, db_name, tbl_name); + } + }.run(false); + } + + protected List getCheckConstraintsInternal(String catName, final String db_name_input, + final String tbl_name_input, boolean allowSql, + boolean allowJdo) + throws MetaException, NoSuchObjectException { + final String db_name = normalizeIdentifier(db_name_input); + final String tbl_name = normalizeIdentifier(tbl_name_input); + return new ObjectStore.GetListHelper(normalizeIdentifier(catName), db_name, tbl_name, + allowSql, allowJdo) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) + throws MetaException { + return directSql.getCheckConstraints(catName, db_name, tbl_name); + } + + @Override + protected List getJdoResult(ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + return getCheckConstraintsViaJdo(catName, db_name, tbl_name); + } + }.run(false); + } + + private List getCheckConstraintsViaJdo(String catName, String dbName, String tblName) { + boolean commited = false; + List checkConstraints= null; + Query query = null; + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName && constraintType == MConstraint.CHECK_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + checkConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + checkConstraints.add(new SQLCheckConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), + currConstraint.getDefaultValue(), + currConstraint.getConstraintName(), enable, validate, rely)); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return checkConstraints; + } + + private List getDefaultConstraintsViaJdo(String catName, String dbName, String tblName) { + boolean commited = false; + List defaultConstraints= null; + Query query = null; + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.DEFAULT_CONSTRAINT"); + query.declareParameters( + "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + defaultConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + defaultConstraints.add(new SQLDefaultConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getDefaultValue(), + currConstraint.getConstraintName(), enable, validate, rely)); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return defaultConstraints; + } + + protected List getNotNullConstraintsInternal(String catName, final String db_name_input, + final String tbl_name_input, boolean allowSql, boolean allowJdo) + throws MetaException, NoSuchObjectException { + catName = normalizeIdentifier(catName); + final String db_name = normalizeIdentifier(db_name_input); + final String tbl_name = normalizeIdentifier(tbl_name_input); + return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + + @Override + protected List getSqlResult(ObjectStore.GetHelper> ctx) + throws MetaException { + return directSql.getNotNullConstraints(catName, db_name, tbl_name); + } + + @Override + protected List getJdoResult(ObjectStore.GetHelper> ctx) + throws MetaException, NoSuchObjectException { + return getNotNullConstraintsViaJdo(catName, db_name, tbl_name); + } + }.run(false); + } + + private List getNotNullConstraintsViaJdo(String catName, String dbName, String tblName) { + boolean commited = false; + List notNullConstraints = null; + Query query = null; + try { + openTransaction(); + query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName && constraintType == MConstraint.NOT_NULL_CONSTRAINT"); + query.declareParameters( + "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + notNullConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + notNullConstraints.add(new SQLNotNullConstraint(catName, dbName, + tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), + currConstraint.getConstraintName(), enable, validate, rely)); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + return notNullConstraints; + } + + /** + * Api to fetch all constraints at once + * @param request request object + * @return all table constraints + * @throws MetaException + * @throws NoSuchObjectException + */ + @Override + public SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request) + throws MetaException, NoSuchObjectException { + String catName = request.getCatName(); + String dbName = request.getDbName(); + String tblName = request.getTblName(); + debugLog("Get all table constraints for the table - " + catName + "." + dbName + "." + tblName + + " in class ObjectStore.java"); + SQLAllTableConstraints sqlAllTableConstraints = new SQLAllTableConstraints(); + PrimaryKeysRequest primaryKeysRequest = new PrimaryKeysRequest(dbName, tblName); + primaryKeysRequest.setCatName(catName); + sqlAllTableConstraints.setPrimaryKeys(getPrimaryKeys(primaryKeysRequest)); + ForeignKeysRequest foreignKeysRequest = + new ForeignKeysRequest(null, null, dbName, tblName); + foreignKeysRequest.setCatName(catName); + sqlAllTableConstraints.setForeignKeys(getForeignKeys(foreignKeysRequest)); + sqlAllTableConstraints. + setUniqueConstraints(getUniqueConstraints(new UniqueConstraintsRequest(catName, dbName, tblName))); + sqlAllTableConstraints. + setDefaultConstraints(getDefaultConstraints(new DefaultConstraintsRequest(catName, dbName, tblName))); + sqlAllTableConstraints. + setCheckConstraints(getCheckConstraints(new CheckConstraintsRequest(catName, dbName, tblName))); + sqlAllTableConstraints. + setNotNullConstraints(getNotNullConstraints(new NotNullConstraintsRequest(catName, dbName, tblName))); + return sqlAllTableConstraints; + } + + @Override + public void dropConstraint(String catName, String dbName, String tableName, + String constraintName, boolean missingOk) + throws NoSuchObjectException { + boolean success = false; + try { + openTransaction(); + + List tabConstraints = + listAllTableConstraintsWithOptionalConstraintName(catName, dbName, tableName, constraintName); + if (CollectionUtils.isNotEmpty(tabConstraints)) { + pm.deletePersistentAll(tabConstraints); + } else if (!missingOk) { + throw new NoSuchObjectException("The constraint: " + constraintName + + " does not exist for the associated table: " + dbName + "." + tableName); + } + success = commitTransaction(); + } finally { + rollbackAndCleanup(success, null); + } + } + + private static MFieldSchema getColumnFromTableColumns(List cols, String col) { + if (cols == null) { + return null; + } + for (MFieldSchema mfs : cols) { + if (mfs.getName().equalsIgnoreCase(col)) { + return mfs; + } + } + return null; + } + + private static int getColumnIndexFromTableColumns(List cols, String col) { + if (cols == null) { + return -1; + } + for (int i = 0; i < cols.size(); i++) { + MFieldSchema mfs = cols.get(i); + if (mfs.getName().equalsIgnoreCase(col)) { + return i; + } + } + return -1; + } + + class AttachedMTableInfo { + MTable mtbl; + MColumnDescriptor mcd; + + public AttachedMTableInfo() {} + + public AttachedMTableInfo(MTable mtbl, MColumnDescriptor mcd) { + this.mtbl = mtbl; + this.mcd = mcd; + } + } + + private AttachedMTableInfo getMTable(String catName, String db, String table, + boolean retrieveCD) { + AttachedMTableInfo nmtbl = new AttachedMTableInfo(); + MTable mtbl = null; + boolean commited = false; + Query query = null; + try { + openTransaction(); + catName = normalizeIdentifier(Optional.ofNullable(catName).orElse(getDefaultCatalog(conf))); + db = normalizeIdentifier(db); + table = normalizeIdentifier(table); + query = pm.newQuery(MTable.class, + "tableName == table && database.name == db && database.catalogName == catname"); + query.declareParameters( + "java.lang.String table, java.lang.String db, java.lang.String catname"); + query.setUnique(true); + if (LOG.isDebugEnabled()) { + LOG.debug("Executing getMTable for {}", + TableName.getQualified(catName, db, table)); + } + mtbl = (MTable) query.execute(table, db, catName); + pm.retrieve(mtbl); + // Retrieving CD can be expensive and unnecessary, so do it only when required. + if (mtbl != null && retrieveCD) { + pm.retrieve(mtbl.getSd()); + pm.retrieveAll(mtbl.getSd().getCD()); + nmtbl.mcd = mtbl.getSd().getCD(); + } + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + nmtbl.mtbl = mtbl; + return nmtbl; + } + + private MTable getMTable(String catName, String db, String table) { + AttachedMTableInfo nmtbl = getMTable(catName, db, table, false); + return nmtbl.mtbl; + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index c03e1b8e0016..b7cf5dfa636a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; +import javax.jdo.PersistenceManager; import javax.jdo.Query; import java.util.ArrayList; import java.util.Arrays; @@ -121,10 +122,6 @@ import static org.apache.hadoop.hive.metastore.ObjectStore.convertToPart; import static org.apache.hadoop.hive.metastore.ObjectStore.convertToParts; import static org.apache.hadoop.hive.metastore.ObjectStore.convertToTable; -import static org.apache.hadoop.hive.metastore.ObjectStore.getJDOFilterStrForPartitionNames; -import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams; -import static org.apache.hadoop.hive.metastore.ObjectStore.makeParameterDeclarationString; -import static org.apache.hadoop.hive.metastore.ObjectStore.putPersistentPrivObjects; import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx; import static org.apache.hadoop.hive.metastore.metastore.impl.PrivilegeStoreImpl.getPrincipalTypeFromStr; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; @@ -2963,4 +2960,75 @@ private MPartition getMPartition(String catName, String dbName, String tableName } return ret; } + + /** + * Convert PrivilegeGrantInfo from privMap to MTablePrivilege, and add all of + * them to the toPersistPrivObjs. These privilege objects will be persisted as + * part of createTable. + */ + private void putPersistentPrivObjects(MTable mtbl, List toPersistPrivObjs, + int now, Map> privMap, PrincipalType type, String authorizer) { + if (privMap != null) { + for (Map.Entry> entry : privMap + .entrySet()) { + String principalName = entry.getKey(); + List privs = entry.getValue(); + for (int i = 0; i < privs.size(); i++) { + PrivilegeGrantInfo priv = privs.get(i); + if (priv == null) { + continue; + } + MTablePrivilege mTblSec = new MTablePrivilege( + principalName, type.toString(), mtbl, priv.getPrivilege(), + now, priv.getGrantor(), priv.getGrantorType().toString(), priv + .isGrantOption(), authorizer); + toPersistPrivObjs.add(mTblSec); + } + } + } + } + + private Pair> getPartQueryWithParams( + PersistenceManager pm, + String catName, String dbName, String tblName, + List partNames) { + Query query = pm.newQuery(); + Map params = new HashMap<>(); + String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); + query.setFilter(filterStr); + LOG.debug(" JDOQL filter is {}", filterStr); + query.declareParameters(makeParameterDeclarationString(params)); + return Pair.of(query, params); + } + + private String getJDOFilterStrForPartitionNames(String catName, String dbName, String tblName, + List partNames, Map params) { + StringBuilder sb = new StringBuilder( + "table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); + params.put("t1", normalizeIdentifier(tblName)); + params.put("t2", normalizeIdentifier(dbName)); + params.put("t3", normalizeIdentifier(catName)); + int n = 0; + for (Iterator itr = partNames.iterator(); itr.hasNext(); ) { + String pn = "p" + n; + n++; + String part = itr.next(); + params.put(pn, part); + sb.append("partitionName == ").append(pn); + sb.append(" || "); + } + sb.setLength(sb.length() - 4); // remove the last " || " + sb.append(')'); + return sb.toString(); + } + + private String makeParameterDeclarationString(Map params) { + //Create the parameter declaration string + StringBuilder paramDecl = new StringBuilder(); + for (String key : params.keySet()) { + paramDecl.append(", java.lang.String ").append(key); + } + return paramDecl.toString(); + } + } From 50e2334491a9774b345600230e48cfe9e178fde0 Mon Sep 17 00:00:00 2001 From: zdeng Date: Thu, 18 Jun 2026 22:58:01 +0800 Subject: [PATCH 02/22] fix --- .../hadoop/hive/metastore/ObjectStore.java | 471 +---------------- .../metastore/metastore/iface/WLMStore.java | 4 + .../metastore/impl/TableStoreImpl.java | 472 +++++++++++++++++- 3 files changed, 477 insertions(+), 470 deletions(-) create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 4a130d81bdf3..6f8ad482be6a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING; import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPACTOR_USE_CUSTOM_POOL; +import static org.apache.hadoop.hive.metastore.metastore.impl.TableStoreImpl.convertToFieldSchemas; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; @@ -64,12 +65,11 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.AddPackageRequest; import org.apache.hadoop.hive.metastore.api.DropPackageRequest; @@ -88,9 +88,7 @@ import org.apache.hadoop.hive.metastore.api.ListStoredProcedureRequest; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Package; -import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.QueryState; import org.apache.hadoop.hive.metastore.api.ResourceType; @@ -113,9 +111,6 @@ import org.apache.hadoop.hive.metastore.api.SchemaVersionState; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SerdeType; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.api.SourceTable; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.StoredProcedure; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableParamsUpdate; @@ -135,11 +130,11 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats; import org.apache.hadoop.hive.metastore.metastore.iface.PrivilegeStore; +import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MCatalog; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; -import org.apache.hadoop.hive.metastore.model.MCreationMetadata; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDataConnector; import org.apache.hadoop.hive.metastore.model.MDCPrivilege; @@ -148,12 +143,9 @@ import org.apache.hadoop.hive.metastore.model.MFieldSchema; import org.apache.hadoop.hive.metastore.model.MFunction; import org.apache.hadoop.hive.metastore.model.MISchema; -import org.apache.hadoop.hive.metastore.model.MMVSource; import org.apache.hadoop.hive.metastore.model.MMasterKey; import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties; -import org.apache.hadoop.hive.metastore.model.MOrder; import org.apache.hadoop.hive.metastore.model.MPackage; -import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MPartitionEvent; import org.apache.hadoop.hive.metastore.model.MResourceUri; import org.apache.hadoop.hive.metastore.model.MRuntimeStat; @@ -161,10 +153,7 @@ import org.apache.hadoop.hive.metastore.model.MScheduledQuery; import org.apache.hadoop.hive.metastore.model.MSchemaVersion; import org.apache.hadoop.hive.metastore.model.MSerDeInfo; -import org.apache.hadoop.hive.metastore.model.MStorageDescriptor; import org.apache.hadoop.hive.metastore.model.MStoredProc; -import org.apache.hadoop.hive.metastore.model.MStringList; -import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.model.MType; import org.apache.hadoop.hive.metastore.model.MVersionTable; import org.apache.hadoop.hive.metastore.model.MWMMapping; @@ -181,7 +170,6 @@ import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; import org.apache.hadoop.hive.metastore.metastore.TransactionHandler; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -200,8 +188,6 @@ import com.cronutils.model.time.ExecutionTime; import com.cronutils.parser.CronParser; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -1311,13 +1297,8 @@ private static StringBuilder appendCondition(StringBuilder builder, return builder; } - /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */ - private static List convertList(List dnList) { - return (dnList == null) ? null : Lists.newArrayList(dnList); - } - /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */ - private static Map convertMap(Map dnMap, Configuration conf, GetPartitionsArgs... args) { + public static Map convertMap(Map dnMap, Configuration conf, GetPartitionsArgs... args) { Map parameters = MetaStoreServerUtils.trimMapNulls(dnMap, MetastoreConf.getBoolVar(conf, ConfVars.ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS)); if (parameters != null && args != null && args.length == 1) { @@ -1351,96 +1332,7 @@ private static Map convertMap(Map dnMap, Configu return parameters; } - public static Table convertToTable(MTable mtbl, Configuration conf) throws MetaException { - if (mtbl == null) { - return null; - } - String tableType = mtbl.getTableType(); - String viewOriginalText = null; - String viewExpandedText = null; - if (tableType == null) { - // for backwards compatibility with old metastore persistence - if (mtbl.getViewOriginalText() != null) { - tableType = TableType.VIRTUAL_VIEW.toString(); - } else if (mtbl.getParameters() != null && Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) { - tableType = TableType.EXTERNAL_TABLE.toString(); - } else { - tableType = TableType.MANAGED_TABLE.toString(); - } - } else { - if (tableType.equals(TableType.VIRTUAL_VIEW.toString()) || tableType.equals(TableType.MATERIALIZED_VIEW.toString())) { - viewOriginalText = mtbl.getViewOriginalText(); - viewExpandedText = mtbl.getViewExpandedText(); - } - } - Map parameters = convertMap(mtbl.getParameters(), conf); - boolean isAcidTable = TxnUtils.isAcidTable(parameters); - final Table t = new Table(mtbl.getTableName(), mtbl.getDatabase() != null ? mtbl.getDatabase().getName() : null, - mtbl.getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl.getRetention(), - convertToStorageDescriptor(mtbl.getSd(), false, isAcidTable, conf), - convertToFieldSchemas(mtbl.getPartitionKeys()), parameters, viewOriginalText, - viewExpandedText, tableType); - - if (Strings.isNullOrEmpty(mtbl.getOwnerType())) { - // Before the ownerType exists in an old Hive schema, USER was the default type for owner. - // Let's set the default to USER to keep backward compatibility. - t.setOwnerType(PrincipalType.USER); - } else { - t.setOwnerType(PrincipalType.valueOf(mtbl.getOwnerType())); - } - - t.setId(mtbl.getId()); - t.setRewriteEnabled(mtbl.isRewriteEnabled()); - t.setCatName(mtbl.getDatabase() != null ? mtbl.getDatabase().getCatalogName() : null); - t.setWriteId(mtbl.getWriteId()); - return t; - } - - public static MTable convertToMTable(Table tbl, RawStore base) throws InvalidObjectException, - MetaException { - // NOTE: we don't set writeId in this method. Write ID is only set after validating the - // existing write ID against the caller's valid list. - if (tbl == null) { - return null; - } - MDatabase mdb = null; - String catName = tbl.isSetCatName() ? tbl.getCatName() : getDefaultCatalog(base.getConf()); - try { - mdb = base.ensureGetMDatabase(catName, tbl.getDbName()); - } catch (NoSuchObjectException e) { - LOG.error("Could not convert to MTable", e); - throw new InvalidObjectException("Database " + - DatabaseName.getQualified(catName, tbl.getDbName()) + " doesn't exist."); - } - - // If the table has property EXTERNAL set, update table type - // accordingly - String tableType = tbl.getTableType(); - boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); - if (TableType.MANAGED_TABLE.toString().equals(tableType)) { - if (isExternal) { - tableType = TableType.EXTERNAL_TABLE.toString(); - } - } - if (TableType.EXTERNAL_TABLE.toString().equals(tableType)) { - if (!isExternal) { - tableType = TableType.MANAGED_TABLE.toString(); - } - } - - PrincipalType ownerPrincipalType = tbl.getOwnerType(); - String ownerType = (ownerPrincipalType == null) ? PrincipalType.USER.name() : ownerPrincipalType.name(); - - // A new table is always created with a new column descriptor - return new MTable(normalizeIdentifier(tbl.getTableName()), mdb, - convertToMStorageDescriptor(tbl.getSd()), tbl.getOwner(), ownerType, tbl - .getCreateTime(), tbl.getLastAccessTime(), tbl.getRetention(), - convertToMFieldSchemas(tbl.getPartitionKeys()), tbl.getParameters(), - tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(), - tableType); - } - - private static List convertToMFieldSchemas(List keys) { + public static List convertToMFieldSchemas(List keys) { List mkeys = null; if (keys != null) { mkeys = new ArrayList<>(keys.size()); @@ -1452,41 +1344,7 @@ private static List convertToMFieldSchemas(List keys) return mkeys; } - public static List convertToFieldSchemas(List mkeys) { - List keys = null; - if (mkeys != null) { - keys = new ArrayList<>(); - for (MFieldSchema part : mkeys) { - keys.add(new FieldSchema(part.getName(), part.getType(), part - .getComment())); - } - } - return keys; - } - - private static List convertToMOrders(List keys) { - List mkeys = null; - if (keys != null) { - mkeys = new ArrayList<>(); - for (Order part : keys) { - mkeys.add(new MOrder(normalizeIdentifier(part.getCol()), part.getOrder())); - } - } - return mkeys; - } - - private static List convertToOrders(List mkeys) { - List keys = null; - if (mkeys != null) { - keys = new ArrayList<>(); - for (MOrder part : mkeys) { - keys.add(new Order(part.getCol(), part.getOrder())); - } - } - return keys; - } - - private static SerDeInfo convertToSerDeInfo(MSerDeInfo ms, Configuration conf, boolean allowNull) + public static SerDeInfo convertToSerDeInfo(MSerDeInfo ms, Configuration conf, boolean allowNull) throws MetaException { if (ms == null) { if (allowNull) { @@ -1511,7 +1369,7 @@ private static SerDeInfo convertToSerDeInfo(MSerDeInfo ms, Configuration conf, b return serde; } - private static MSerDeInfo convertToMSerDeInfo(SerDeInfo ms) throws MetaException { + public static MSerDeInfo convertToMSerDeInfo(SerDeInfo ms) throws MetaException { if (ms == null) { throw new MetaException("Invalid SerDeInfo object"); } @@ -1525,317 +1383,13 @@ private static MSerDeInfo convertToMSerDeInfo(SerDeInfo ms) throws MetaException * @param cols the columns the column descriptor contains * @return a new column descriptor db-backed object */ - private static MColumnDescriptor createNewMColumnDescriptor(List cols) { + public static MColumnDescriptor createNewMColumnDescriptor(List cols) { if (cols == null) { return null; } return new MColumnDescriptor(cols); } - private static StorageDescriptor convertToStorageDescriptor( - MStorageDescriptor msd, boolean noFS, boolean isAcidTable, Configuration conf) throws MetaException { - if (msd == null) { - return null; - } - List mFieldSchemas; - if (noFS) { - mFieldSchemas = Collections.emptyList(); - } else { - mFieldSchemas = msd.getCD() == null ? null : msd.getCD().getCols(); - } - List orderList = (isAcidTable) ? Collections.emptyList() : convertToOrders(msd.getSortCols()); - List bucList = convertList(msd.getBucketCols()); - SkewedInfo skewedInfo = null; - - Map sdParams = isAcidTable ? Collections.emptyMap() : convertMap(msd.getParameters(), conf); - StorageDescriptor sd = new StorageDescriptor(convertToFieldSchemas(mFieldSchemas), - msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd - .isCompressed(), msd.getNumBuckets(), - (!isAcidTable) ? convertToSerDeInfo(msd.getSerDeInfo(), conf, true) - : new SerDeInfo(msd.getSerDeInfo().getName(), msd.getSerDeInfo().getSerializationLib(), Collections.emptyMap()), - bucList , orderList, sdParams); - if (!isAcidTable) { - skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()), - convertToSkewedValues(msd.getSkewedColValues()), - covertToSkewedMap(msd.getSkewedColValueLocationMaps())); - } else { - skewedInfo = new SkewedInfo(Collections.emptyList(), Collections.emptyList(), - Collections.emptyMap()); - } - sd.setSkewedInfo(skewedInfo); - sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); - return sd; - } - - /** - * Convert a list of MStringList to a list of list string - */ - private static List> convertToSkewedValues(List mLists) { - List> lists = null; - if (mLists != null) { - lists = new ArrayList<>(); - for (MStringList element : mLists) { - lists.add(new ArrayList<>(element.getInternalList())); - } - } - return lists; - } - - private static List convertToMStringLists(List> mLists) { - List lists = null ; - if (null != mLists) { - lists = new ArrayList<>(); - for (List mList : mLists) { - lists.add(new MStringList(mList)); - } - } - return lists; - } - - /** - * Convert a MStringList Map to a Map - */ - private static Map, String> covertToSkewedMap(Map mMap) { - Map, String> map = null; - if (mMap != null) { - map = new HashMap<>(); - Set keys = mMap.keySet(); - for (MStringList key : keys) { - map.put(new ArrayList<>(key.getInternalList()), mMap.get(key)); - } - } - return map; - } - - /** - * Covert a Map to a MStringList Map - */ - private static Map covertToMapMStringList(Map, String> mMap) { - Map map = null; - if (mMap != null) { - map = new HashMap<>(); - Set> keys = mMap.keySet(); - for (List key : keys) { - map.put(new MStringList(key), mMap.get(key)); - } - } - return map; - } - - /** - * Converts a storage descriptor to a db-backed storage descriptor. Creates a - * new db-backed column descriptor object for this SD. - * @param sd the storage descriptor to wrap in a db-backed object - * @return the storage descriptor db-backed object - */ - private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd) - throws MetaException { - if (sd == null) { - return null; - } - MColumnDescriptor mcd = createNewMColumnDescriptor(convertToMFieldSchemas(sd.getCols())); - return convertToMStorageDescriptor(sd, mcd); - } - - /** - * Converts a storage descriptor to a db-backed storage descriptor. It points the - * storage descriptor's column descriptor to the one passed as an argument, - * so it does not create a new mcolumn descriptor object. - * @param sd the storage descriptor to wrap in a db-backed object - * @param mcd the db-backed column descriptor - * @return the db-backed storage descriptor object - */ - private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, - MColumnDescriptor mcd) throws MetaException { - if (sd == null) { - return null; - } - return new MStorageDescriptor(mcd, sd - .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd - .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd - .getSerdeInfo()), sd.getBucketCols(), - convertToMOrders(sd.getSortCols()), sd.getParameters(), - (null == sd.getSkewedInfo()) ? null - : sd.getSkewedInfo().getSkewedColNames(), - convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() - .getSkewedColValues()), - covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() - .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); - } - - public static MCreationMetadata convertToMCreationMetadata(CreationMetadata m, RawStore base) - throws MetaException { - if (m == null) { - return null; - } - assert !m.isSetMaterializationTime(); - try { - Set tablesUsed = new HashSet<>(); - if (m.isSetSourceTables()) { - for (SourceTable sourceTable : m.getSourceTables()) { - tablesUsed.add(convertToSourceTable(m.getCatName(), sourceTable, base)); - } - } else { - for (String fullyQualifiedName : m.getTablesUsed()) { - tablesUsed.add(convertToSourceTable(m.getCatName(), fullyQualifiedName, base)); - } - } - return new MCreationMetadata(normalizeIdentifier(m.getCatName()), normalizeIdentifier(m.getDbName()), - normalizeIdentifier(m.getTblName()), tablesUsed, m.getValidTxnList(), System.currentTimeMillis()); - } catch (NoSuchObjectException nse) { - throw new MetaException(nse.getMessage()); - } - } - - public static MMVSource convertToSourceTable(String catalog, SourceTable sourceTable, RawStore base) - throws NoSuchObjectException { - Table table = sourceTable.getTable(); - MTable mtbl = base.ensureGetMTable(catalog, table.getDbName(), table.getTableName()); - MMVSource source = new MMVSource(); - source.setTable(mtbl); - source.setInsertedCount(sourceTable.getInsertedCount()); - source.setUpdatedCount(sourceTable.getUpdatedCount()); - source.setDeletedCount(sourceTable.getDeletedCount()); - return source; - } - - /** - * This method resets the stats to 0 and supports only backward compatibility with clients does not - * send {@link SourceTable} instances. - * - * Use {@link ObjectStore#convertToSourceTable(String, SourceTable, RawStore)} instead. - * - * @param catalog Catalog name where source table is located - * @param fullyQualifiedTableName fully qualified name of source table - * @return {@link MMVSource} instance represents this source table. - */ - @Deprecated - private static MMVSource convertToSourceTable(String catalog, String fullyQualifiedTableName, RawStore base) - throws NoSuchObjectException { - String[] names = fullyQualifiedTableName.split("\\."); - MTable mtbl = base.ensureGetMTable(catalog, names[0], names[1]); - MMVSource source = new MMVSource(); - source.setTable(mtbl); - source.setInsertedCount(0L); - source.setUpdatedCount(0L); - source.setDeletedCount(0L); - return source; - } - - public static CreationMetadata convertToCreationMetadata(MCreationMetadata s, RawStore base) - throws MetaException { - if (s == null) { - return null; - } - try { - Set tablesUsed = new HashSet<>(); - List sourceTables = new ArrayList<>(s.getTables().size()); - for (MMVSource mtbl : s.getTables()) { - tablesUsed.add( - Warehouse.getQualifiedName(mtbl.getTable().getDatabase().getName(), mtbl.getTable().getTableName())); - sourceTables.add(convertToSourceTable(mtbl, s.getCatalogName(), base)); - } - CreationMetadata r = new CreationMetadata(s.getCatalogName(), s.getDbName(), s.getTblName(), tablesUsed); - r.setMaterializationTime(s.getMaterializationTime()); - if (s.getTxnList() != null) { - r.setValidTxnList(s.getTxnList()); - } - r.setSourceTables(sourceTables); - return r; - } catch (NoSuchObjectException nse) { - throw new MetaException(nse.getMessage()); - } - } - - private static SourceTable convertToSourceTable(MMVSource mmvSource, String catalogName, RawStore base) - throws MetaException, NoSuchObjectException { - SourceTable sourceTable = new SourceTable(); - MTable mTable = mmvSource.getTable(); - Table table = - convertToTable(base.ensureGetMTable(catalogName, mTable.getDatabase().getName(), mTable.getTableName()), - base.getConf()); - sourceTable.setTable(table); - sourceTable.setInsertedCount(mmvSource.getInsertedCount()); - sourceTable.setUpdatedCount(mmvSource.getUpdatedCount()); - sourceTable.setDeletedCount(mmvSource.getDeletedCount()); - return sourceTable; - } - - /** - * Convert a Partition object into an MPartition, which is an object backed by the db - * If the Partition's set of columns is the same as the parent table's AND useTableCD - * is true, then this partition's storage descriptor's column descriptor will point - * to the same one as the table's storage descriptor. - * @param part the partition to convert - * @param mt the parent table object - * @return the model partition object, and null if the input partition is null. - */ - public static MPartition convertToMPart(Partition part, MTable mt) - throws InvalidObjectException, MetaException { - // NOTE: we don't set writeId in this method. Write ID is only set after validating the - // existing write ID against the caller's valid list. - if (part == null) { - return null; - } - if (mt == null) { - throw new InvalidObjectException( - "Partition doesn't have a valid table or database name"); - } - - // If this partition's set of columns is the same as the parent table's, - // use the parent table's, so we do not create a duplicate column descriptor, - // thereby saving space - MStorageDescriptor msd; - if (mt.getSd() != null && mt.getSd().getCD() != null && - mt.getSd().getCD().getCols() != null && - part.getSd() != null && - convertToFieldSchemas(mt.getSd().getCD().getCols()). - equals(part.getSd().getCols())) { - msd = convertToMStorageDescriptor(part.getSd(), mt.getSd().getCD()); - } else { - msd = convertToMStorageDescriptor(part.getSd()); - } - - return new MPartition(Warehouse.makePartName(convertToFieldSchemas(mt - .getPartitionKeys()), part.getValues()), mt, part.getValues(), part - .getCreateTime(), part.getLastAccessTime(), - msd, part.getParameters()); - } - - public static Partition convertToPart(String catName, String dbName, String tblName, - MPartition mpart, boolean isAcidTable, Configuration conf, GetPartitionsArgs... args) - throws MetaException { - if (mpart == null) { - return null; - } - catName = normalizeIdentifier(catName); - dbName = normalizeIdentifier(dbName); - tblName = normalizeIdentifier(tblName); - Map params = convertMap(mpart.getParameters(), conf, args); - boolean noFS = args != null && args.length == 1 && args[0].isSkipColumnSchemaForPartition(); - Partition p = new Partition(convertList(mpart.getValues()), dbName, tblName, - mpart.getCreateTime(), mpart.getLastAccessTime(), - convertToStorageDescriptor(mpart.getSd(), noFS, isAcidTable, conf), params); - p.setCatName(catName); - if(mpart.getWriteId()>0) { - p.setWriteId(mpart.getWriteId()); - }else { - p.setWriteId(-1L); - } - return p; - } - - public static List convertToParts(String catName, String dbName, String tblName, - List mparts, boolean isAcidTable, Configuration conf, GetPartitionsArgs args) - throws MetaException { - List parts = new ArrayList<>(mparts.size()); - for (MPartition mp : mparts) { - parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable, conf, args)); - Deadline.checkTimeout(); - } - return parts; - } - /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */ @VisibleForTesting public abstract class GetHelper { @@ -2070,7 +1624,14 @@ protected String describeResult() { private Table ensureGetTable(String catName, String dbName, String tblName) throws NoSuchObjectException, MetaException { - return convertToTable(ensureGetMTable(catName, dbName, tblName), conf); + TableName tableName = new TableName(catName, dbName, tblName); + Table table = + unwrap(TableStore.class).getTable(tableName, null, -1); + if (table == null) { + throw new NoSuchObjectException( + "Specified catalog.database.table does not exist : " + tableName); + } + return table; } /** diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java new file mode 100644 index 000000000000..11f984339637 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java @@ -0,0 +1,4 @@ +package org.apache.hadoop.hive.metastore.metastore.iface; +// work load management +public interface WLMStore { +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index b7cf5dfa636a..f8a2e1e70036 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.metastore.metastore.impl; import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -43,12 +45,20 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.Batchable; import org.apache.hadoop.hive.metastore.DatabaseProduct; +import org.apache.hadoop.hive.metastore.Deadline; +import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.SourceTable; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.PartFilterExprUtil; import org.apache.hadoop.hive.metastore.PartitionExpressionProxy; @@ -88,13 +98,17 @@ import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MCreationMetadata; +import org.apache.hadoop.hive.metastore.model.MDatabase; +import org.apache.hadoop.hive.metastore.model.MFieldSchema; import org.apache.hadoop.hive.metastore.model.MMVSource; +import org.apache.hadoop.hive.metastore.model.MOrder; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; import org.apache.hadoop.hive.metastore.model.MPartitionEvent; import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; import org.apache.hadoop.hive.metastore.model.MStorageDescriptor; +import org.apache.hadoop.hive.metastore.model.MStringList; import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; @@ -114,14 +128,11 @@ import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING; import static org.apache.hadoop.hive.metastore.ObjectStore.appendPatternCondition; import static org.apache.hadoop.hive.metastore.ObjectStore.appendSimpleCondition; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToCreationMetadata; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToFieldSchemas; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToMCreationMetadata; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToMPart; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToMTable; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToPart; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToParts; -import static org.apache.hadoop.hive.metastore.ObjectStore.convertToTable; +import static org.apache.hadoop.hive.metastore.ObjectStore.convertMap; +import static org.apache.hadoop.hive.metastore.ObjectStore.convertToMFieldSchemas; +import static org.apache.hadoop.hive.metastore.ObjectStore.convertToMSerDeInfo; +import static org.apache.hadoop.hive.metastore.ObjectStore.convertToSerDeInfo; +import static org.apache.hadoop.hive.metastore.ObjectStore.createNewMColumnDescriptor; import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx; import static org.apache.hadoop.hive.metastore.metastore.impl.PrivilegeStoreImpl.getPrincipalTypeFromStr; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; @@ -621,7 +632,7 @@ public Table getTable(TableName table, String writeIdList, long tableId) // Retrieve creation metadata if needed if (tbl != null && TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) { tbl.setCreationMetadata( - convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName), baseStore)); + convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName))); } // If transactional non partitioned table, @@ -951,7 +962,7 @@ public Table alterTable(TableName tableName, Table newTable, String queryValidWr String name = normalizeIdentifier(tableName.getTable()); String dbname = normalizeIdentifier(tableName.getDb()); String catName = normalizeIdentifier(tableName.getCat()); - MTable newt = convertToMTable(newTable, baseStore); + MTable newt = convertToMTable(newTable); if (newt == null) { throw new InvalidObjectException("new table is invalid"); } @@ -1092,7 +1103,7 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { @Override public void createTable(Table tbl) throws InvalidObjectException, MetaException { - MTable mtbl = convertToMTable(tbl, baseStore);; + MTable mtbl = convertToMTable(tbl);; if (TxnUtils.isTransactionalTable(tbl)) { mtbl.setWriteId(tbl.getWriteId()); @@ -1100,7 +1111,7 @@ public void createTable(Table tbl) throws InvalidObjectException, MetaException pm.makePersistent(mtbl); if (tbl.getCreationMetadata() != null) { - MCreationMetadata mcm = convertToMCreationMetadata(tbl.getCreationMetadata(), baseStore); + MCreationMetadata mcm = convertToMCreationMetadata(tbl.getCreationMetadata()); pm.makePersistent(mcm); } tbl.setId(mtbl.getId()); @@ -1285,7 +1296,7 @@ public List getTableObjectsByName(String catName, String db, List if (TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) { tbl.setCreationMetadata( convertToCreationMetadata( - getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()), baseStore)); + getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()))); } tables.add(tbl); } @@ -2758,7 +2769,7 @@ public void updateCreationMetadata(TableName tableName, CreationMetadata cm) thr String catName = normalizeIdentifier(tableName.getCat()); String dbName = normalizeIdentifier(tableName.getDb()); String name = normalizeIdentifier(tableName.getTable()); - MCreationMetadata newMcm = convertToMCreationMetadata(cm, baseStore); + MCreationMetadata newMcm = convertToMCreationMetadata(cm); MCreationMetadata mcm = getCreationMetadata(catName, dbName, name); mcm.setTables(newMcm.getTables()); mcm.setMaterializationTime(newMcm.getMaterializationTime()); @@ -2781,7 +2792,7 @@ public List
getAllMaterializedViewObjectsForRewriting(String catName) thr Table tbl = convertToTable(mTbl, conf); tbl.setCreationMetadata( convertToCreationMetadata( - getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()), baseStore)); + getCreationMetadata(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()))); allMaterializedViews.add(tbl); } return allMaterializedViews; @@ -3031,4 +3042,435 @@ private String makeParameterDeclarationString(Map params) { return paramDecl.toString(); } + private Table convertToTable(MTable mtbl, Configuration conf) throws MetaException { + if (mtbl == null) { + return null; + } + String tableType = mtbl.getTableType(); + String viewOriginalText = null; + String viewExpandedText = null; + if (tableType == null) { + // for backwards compatibility with old metastore persistence + if (mtbl.getViewOriginalText() != null) { + tableType = TableType.VIRTUAL_VIEW.toString(); + } else if (mtbl.getParameters() != null && Boolean.parseBoolean(mtbl.getParameters().get("EXTERNAL"))) { + tableType = TableType.EXTERNAL_TABLE.toString(); + } else { + tableType = TableType.MANAGED_TABLE.toString(); + } + } else { + if (tableType.equals(TableType.VIRTUAL_VIEW.toString()) || tableType.equals(TableType.MATERIALIZED_VIEW.toString())) { + viewOriginalText = mtbl.getViewOriginalText(); + viewExpandedText = mtbl.getViewExpandedText(); + } + } + Map parameters = convertMap(mtbl.getParameters(), conf); + boolean isAcidTable = TxnUtils.isAcidTable(parameters); + final Table t = new Table(mtbl.getTableName(), mtbl.getDatabase() != null ? mtbl.getDatabase().getName() : null, + mtbl.getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl.getRetention(), + convertToStorageDescriptor(mtbl.getSd(), false, isAcidTable, conf), + convertToFieldSchemas(mtbl.getPartitionKeys()), parameters, viewOriginalText, + viewExpandedText, tableType); + + if (Strings.isNullOrEmpty(mtbl.getOwnerType())) { + // Before the ownerType exists in an old Hive schema, USER was the default type for owner. + // Let's set the default to USER to keep backward compatibility. + t.setOwnerType(PrincipalType.USER); + } else { + t.setOwnerType(PrincipalType.valueOf(mtbl.getOwnerType())); + } + + t.setId(mtbl.getId()); + t.setRewriteEnabled(mtbl.isRewriteEnabled()); + t.setCatName(mtbl.getDatabase() != null ? mtbl.getDatabase().getCatalogName() : null); + t.setWriteId(mtbl.getWriteId()); + return t; + } + + private MTable convertToMTable(Table tbl) throws InvalidObjectException, + MetaException { + // NOTE: we don't set writeId in this method. Write ID is only set after validating the + // existing write ID against the caller's valid list. + if (tbl == null) { + return null; + } + MDatabase mdb = null; + String catName = tbl.isSetCatName() ? tbl.getCatName() : getDefaultCatalog(baseStore.getConf()); + try { + mdb = baseStore.ensureGetMDatabase(catName, tbl.getDbName()); + } catch (NoSuchObjectException e) { + LOG.error("Could not convert to MTable", e); + throw new InvalidObjectException("Database " + + DatabaseName.getQualified(catName, tbl.getDbName()) + " doesn't exist."); + } + + // If the table has property EXTERNAL set, update table type + // accordingly + String tableType = tbl.getTableType(); + boolean isExternal = Boolean.parseBoolean(tbl.getParameters().get("EXTERNAL")); + if (TableType.MANAGED_TABLE.toString().equals(tableType)) { + if (isExternal) { + tableType = TableType.EXTERNAL_TABLE.toString(); + } + } + if (TableType.EXTERNAL_TABLE.toString().equals(tableType)) { + if (!isExternal) { + tableType = TableType.MANAGED_TABLE.toString(); + } + } + + PrincipalType ownerPrincipalType = tbl.getOwnerType(); + String ownerType = (ownerPrincipalType == null) ? PrincipalType.USER.name() : ownerPrincipalType.name(); + + // A new table is always created with a new column descriptor + return new MTable(normalizeIdentifier(tbl.getTableName()), mdb, + convertToMStorageDescriptor(tbl.getSd()), tbl.getOwner(), ownerType, tbl + .getCreateTime(), tbl.getLastAccessTime(), tbl.getRetention(), + convertToMFieldSchemas(tbl.getPartitionKeys()), tbl.getParameters(), + tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(), + tableType); + } + + public static List convertToFieldSchemas(List mkeys) { + List keys = null; + if (mkeys != null) { + keys = new ArrayList<>(); + for (MFieldSchema part : mkeys) { + keys.add(new FieldSchema(part.getName(), part.getType(), part + .getComment())); + } + } + return keys; + } + + private static List convertToMOrders(List keys) { + List mkeys = null; + if (keys != null) { + mkeys = new ArrayList<>(); + for (Order part : keys) { + mkeys.add(new MOrder(normalizeIdentifier(part.getCol()), part.getOrder())); + } + } + return mkeys; + } + + private static List convertToOrders(List mkeys) { + List keys = null; + if (mkeys != null) { + keys = new ArrayList<>(); + for (MOrder part : mkeys) { + keys.add(new Order(part.getCol(), part.getOrder())); + } + } + return keys; + } + + private static StorageDescriptor convertToStorageDescriptor( + MStorageDescriptor msd, boolean noFS, boolean isAcidTable, Configuration conf) throws MetaException { + if (msd == null) { + return null; + } + List mFieldSchemas; + if (noFS) { + mFieldSchemas = Collections.emptyList(); + } else { + mFieldSchemas = msd.getCD() == null ? null : msd.getCD().getCols(); + } + List orderList = (isAcidTable) ? Collections.emptyList() : convertToOrders(msd.getSortCols()); + List bucList = convertList(msd.getBucketCols()); + SkewedInfo skewedInfo = null; + + Map sdParams = isAcidTable ? Collections.emptyMap() : convertMap(msd.getParameters(), conf); + StorageDescriptor sd = new StorageDescriptor(convertToFieldSchemas(mFieldSchemas), + msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd + .isCompressed(), msd.getNumBuckets(), + (!isAcidTable) ? convertToSerDeInfo(msd.getSerDeInfo(), conf, true) + : new SerDeInfo(msd.getSerDeInfo().getName(), msd.getSerDeInfo().getSerializationLib(), Collections.emptyMap()), + bucList , orderList, sdParams); + if (!isAcidTable) { + skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()), + convertToSkewedValues(msd.getSkewedColValues()), + covertToSkewedMap(msd.getSkewedColValueLocationMaps())); + } else { + skewedInfo = new SkewedInfo(Collections.emptyList(), Collections.emptyList(), + Collections.emptyMap()); + } + sd.setSkewedInfo(skewedInfo); + sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); + return sd; + } + + /** + * Convert a list of MStringList to a list of list string + */ + private static List> convertToSkewedValues(List mLists) { + List> lists = null; + if (mLists != null) { + lists = new ArrayList<>(); + for (MStringList element : mLists) { + lists.add(new ArrayList<>(element.getInternalList())); + } + } + return lists; + } + + private static List convertToMStringLists(List> mLists) { + List lists = null ; + if (null != mLists) { + lists = new ArrayList<>(); + for (List mList : mLists) { + lists.add(new MStringList(mList)); + } + } + return lists; + } + + /** + * Convert a MStringList Map to a Map + */ + private static Map, String> covertToSkewedMap(Map mMap) { + Map, String> map = null; + if (mMap != null) { + map = new HashMap<>(); + Set keys = mMap.keySet(); + for (MStringList key : keys) { + map.put(new ArrayList<>(key.getInternalList()), mMap.get(key)); + } + } + return map; + } + + /** + * Covert a Map to a MStringList Map + */ + private static Map covertToMapMStringList(Map, String> mMap) { + Map map = null; + if (mMap != null) { + map = new HashMap<>(); + Set> keys = mMap.keySet(); + for (List key : keys) { + map.put(new MStringList(key), mMap.get(key)); + } + } + return map; + } + + /** + * Converts a storage descriptor to a db-backed storage descriptor. Creates a + * new db-backed column descriptor object for this SD. + * @param sd the storage descriptor to wrap in a db-backed object + * @return the storage descriptor db-backed object + */ + private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd) + throws MetaException { + if (sd == null) { + return null; + } + MColumnDescriptor mcd = createNewMColumnDescriptor(convertToMFieldSchemas(sd.getCols())); + return convertToMStorageDescriptor(sd, mcd); + } + + /** + * Converts a storage descriptor to a db-backed storage descriptor. It points the + * storage descriptor's column descriptor to the one passed as an argument, + * so it does not create a new mcolumn descriptor object. + * @param sd the storage descriptor to wrap in a db-backed object + * @param mcd the db-backed column descriptor + * @return the db-backed storage descriptor object + */ + private static MStorageDescriptor convertToMStorageDescriptor(StorageDescriptor sd, + MColumnDescriptor mcd) throws MetaException { + if (sd == null) { + return null; + } + return new MStorageDescriptor(mcd, sd + .getLocation(), sd.getInputFormat(), sd.getOutputFormat(), sd + .isCompressed(), sd.getNumBuckets(), convertToMSerDeInfo(sd + .getSerdeInfo()), sd.getBucketCols(), + convertToMOrders(sd.getSortCols()), sd.getParameters(), + (null == sd.getSkewedInfo()) ? null + : sd.getSkewedInfo().getSkewedColNames(), + convertToMStringLists((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() + .getSkewedColValues()), + covertToMapMStringList((null == sd.getSkewedInfo()) ? null : sd.getSkewedInfo() + .getSkewedColValueLocationMaps()), sd.isStoredAsSubDirectories()); + } + + private MCreationMetadata convertToMCreationMetadata(CreationMetadata m) + throws MetaException { + if (m == null) { + return null; + } + assert !m.isSetMaterializationTime(); + try { + Set tablesUsed = new HashSet<>(); + if (m.isSetSourceTables()) { + for (SourceTable sourceTable : m.getSourceTables()) { + tablesUsed.add(convertToSourceTable(m.getCatName(), sourceTable)); + } + } else { + for (String fullyQualifiedName : m.getTablesUsed()) { + tablesUsed.add(convertToSourceTable(m.getCatName(), fullyQualifiedName)); + } + } + return new MCreationMetadata(normalizeIdentifier(m.getCatName()), normalizeIdentifier(m.getDbName()), + normalizeIdentifier(m.getTblName()), tablesUsed, m.getValidTxnList(), System.currentTimeMillis()); + } catch (NoSuchObjectException nse) { + throw new MetaException(nse.getMessage()); + } + } + + private MMVSource convertToSourceTable(String catalog, SourceTable sourceTable) + throws NoSuchObjectException { + Table table = sourceTable.getTable(); + MTable mtbl = baseStore.ensureGetMTable(catalog, table.getDbName(), table.getTableName()); + MMVSource source = new MMVSource(); + source.setTable(mtbl); + source.setInsertedCount(sourceTable.getInsertedCount()); + source.setUpdatedCount(sourceTable.getUpdatedCount()); + source.setDeletedCount(sourceTable.getDeletedCount()); + return source; + } + + /** + * This method resets the stats to 0 and supports only backward compatibility with clients does not + * send {@link SourceTable} instances. + * + * Use {@link ObjectStore#convertToSourceTable(String, SourceTable, RawStore)} instead. + * + * @param catalog Catalog name where source table is located + * @param fullyQualifiedTableName fully qualified name of source table + * @return {@link MMVSource} instance represents this source table. + */ + @Deprecated + private MMVSource convertToSourceTable(String catalog, String fullyQualifiedTableName) + throws NoSuchObjectException { + String[] names = fullyQualifiedTableName.split("\\."); + MTable mtbl = baseStore.ensureGetMTable(catalog, names[0], names[1]); + MMVSource source = new MMVSource(); + source.setTable(mtbl); + source.setInsertedCount(0L); + source.setUpdatedCount(0L); + source.setDeletedCount(0L); + return source; + } + + private CreationMetadata convertToCreationMetadata(MCreationMetadata s) + throws MetaException { + if (s == null) { + return null; + } + try { + Set tablesUsed = new HashSet<>(); + List sourceTables = new ArrayList<>(s.getTables().size()); + for (MMVSource mtbl : s.getTables()) { + tablesUsed.add( + Warehouse.getQualifiedName(mtbl.getTable().getDatabase().getName(), mtbl.getTable().getTableName())); + sourceTables.add(convertToSourceTable(mtbl, s.getCatalogName())); + } + CreationMetadata r = new CreationMetadata(s.getCatalogName(), s.getDbName(), s.getTblName(), tablesUsed); + r.setMaterializationTime(s.getMaterializationTime()); + if (s.getTxnList() != null) { + r.setValidTxnList(s.getTxnList()); + } + r.setSourceTables(sourceTables); + return r; + } catch (NoSuchObjectException nse) { + throw new MetaException(nse.getMessage()); + } + } + + private SourceTable convertToSourceTable(MMVSource mmvSource, String catalogName) + throws MetaException, NoSuchObjectException { + SourceTable sourceTable = new SourceTable(); + MTable mTable = mmvSource.getTable(); + Table table = + convertToTable(baseStore.ensureGetMTable(catalogName, mTable.getDatabase().getName(), mTable.getTableName()), + baseStore.getConf()); + sourceTable.setTable(table); + sourceTable.setInsertedCount(mmvSource.getInsertedCount()); + sourceTable.setUpdatedCount(mmvSource.getUpdatedCount()); + sourceTable.setDeletedCount(mmvSource.getDeletedCount()); + return sourceTable; + } + + /** + * Convert a Partition object into an MPartition, which is an object backed by the db + * If the Partition's set of columns is the same as the parent table's AND useTableCD + * is true, then this partition's storage descriptor's column descriptor will point + * to the same one as the table's storage descriptor. + * @param part the partition to convert + * @param mt the parent table object + * @return the model partition object, and null if the input partition is null. + */ + private MPartition convertToMPart(Partition part, MTable mt) + throws InvalidObjectException, MetaException { + // NOTE: we don't set writeId in this method. Write ID is only set after validating the + // existing write ID against the caller's valid list. + if (part == null) { + return null; + } + if (mt == null) { + throw new InvalidObjectException( + "Partition doesn't have a valid table or database name"); + } + + // If this partition's set of columns is the same as the parent table's, + // use the parent table's, so we do not create a duplicate column descriptor, + // thereby saving space + MStorageDescriptor msd; + if (mt.getSd() != null && mt.getSd().getCD() != null && + mt.getSd().getCD().getCols() != null && + part.getSd() != null && + convertToFieldSchemas(mt.getSd().getCD().getCols()). + equals(part.getSd().getCols())) { + msd = convertToMStorageDescriptor(part.getSd(), mt.getSd().getCD()); + } else { + msd = convertToMStorageDescriptor(part.getSd()); + } + + return new MPartition(Warehouse.makePartName(convertToFieldSchemas(mt + .getPartitionKeys()), part.getValues()), mt, part.getValues(), part + .getCreateTime(), part.getLastAccessTime(), + msd, part.getParameters()); + } + + private Partition convertToPart(String catName, String dbName, String tblName, + MPartition mpart, boolean isAcidTable, Configuration conf, GetPartitionsArgs... args) + throws MetaException { + if (mpart == null) { + return null; + } + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tblName = normalizeIdentifier(tblName); + Map params = convertMap(mpart.getParameters(), conf, args); + boolean noFS = args != null && args.length == 1 && args[0].isSkipColumnSchemaForPartition(); + Partition p = new Partition(convertList(mpart.getValues()), dbName, tblName, + mpart.getCreateTime(), mpart.getLastAccessTime(), + convertToStorageDescriptor(mpart.getSd(), noFS, isAcidTable, conf), params); + p.setCatName(catName); + if(mpart.getWriteId()>0) { + p.setWriteId(mpart.getWriteId()); + }else { + p.setWriteId(-1L); + } + return p; + } + + private List convertToParts(String catName, String dbName, String tblName, + List mparts, boolean isAcidTable, Configuration conf, GetPartitionsArgs args) + throws MetaException { + List parts = new ArrayList<>(mparts.size()); + for (MPartition mp : mparts) { + parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable, conf, args)); + Deadline.checkTimeout(); + } + return parts; + } + + /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */ + private static List convertList(List dnList) { + return (dnList == null) ? null : Lists.newArrayList(dnList); + } } From 21ab29e0c5ab97ae19707398370668d87c420a83 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 06:55:36 +0800 Subject: [PATCH 03/22] fix-1 --- .../hadoop/hive/metastore/RawStore.java | 74 ++- .../metastore/iface/ConstraintStore.java | 171 ++++++ .../metastore/impl/ConstraintStoreImpl.java | 538 ++++++++---------- 3 files changed, 460 insertions(+), 323 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 0633d01ef903..eadb234111fd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -119,6 +119,7 @@ import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; +import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MTable; @@ -1865,8 +1866,10 @@ default int getDatabaseCount() throws MetaException { * @return list of primary key columns or an empty list if the table does not have a primary key * @throws MetaException error accessing the RDBMS */ - List getPrimaryKeys(PrimaryKeysRequest request) - throws MetaException; + default List getPrimaryKeys(PrimaryKeysRequest request) + throws MetaException { + return unwrap(ConstraintStore.class).getPrimaryKeys(request); + } /** * SQLForeignKey represents a single foreign key column. @@ -1877,8 +1880,10 @@ List getPrimaryKeys(PrimaryKeysRequest request) * matches the arguments the results here will be all mixed together into a single list. * @throws MetaException error access the RDBMS. */ - List getForeignKeys(ForeignKeysRequest request) - throws MetaException; + default List getForeignKeys(ForeignKeysRequest request) + throws MetaException { + return unwrap(ConstraintStore.class).getForeignKeys(request); + } /** * SQLUniqueConstraint represents a single unique constraint column. @@ -1888,7 +1893,9 @@ List getForeignKeys(ForeignKeysRequest request) * @return list of unique constraints * @throws MetaException error access the RDBMS. */ - List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException; + default List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException { + return unwrap(ConstraintStore.class).getUniqueConstraints(request); + } /** * SQLNotNullConstraint represents a single not null constraint column. @@ -1898,7 +1905,9 @@ List getForeignKeys(ForeignKeysRequest request) * @return list of not null constraints * @throws MetaException error accessing the RDBMS. */ - List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException; + default List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException { + return unwrap(ConstraintStore.class).getNotNullConstraints(request); + } /** * SQLDefaultConstraint represents a single default constraint column. @@ -1908,7 +1917,9 @@ List getForeignKeys(ForeignKeysRequest request) * @return list of default values defined on the table. * @throws MetaException error accessing the RDBMS */ - List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException; + default List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException { + return unwrap(ConstraintStore.class).getDefaultConstraints(request); + } /** * SQLCheckConstraint represents a single check constraint column. @@ -1918,7 +1929,9 @@ List getForeignKeys(ForeignKeysRequest request) * @return ccheck constraints for this table * @throws MetaException error accessing the RDBMS */ - List getCheckConstraints(CheckConstraintsRequest request) throws MetaException; + default List getCheckConstraints(CheckConstraintsRequest request) throws MetaException { + return unwrap(ConstraintStore.class).getCheckConstraints(request); + } /** * Get table constraints @@ -1927,8 +1940,10 @@ List getForeignKeys(ForeignKeysRequest request) * @throws MetaException * @throws NoSuchObjectException */ - SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request) - throws MetaException, NoSuchObjectException; + default SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request) + throws MetaException, NoSuchObjectException { + return unwrap(ConstraintStore.class).getAllTableConstraints(request); + } /** * Create a table with constraints @@ -1938,7 +1953,10 @@ SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request * @throws InvalidObjectException one of the provided objects is malformed. * @throws MetaException error accessing the RDBMS */ - SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) throws InvalidObjectException, MetaException; + default SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) + throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).createTableWithConstraints(tbl, constraints); + } /** * Drop a constraint, any constraint. I have no idea why add and get each have separate @@ -1965,8 +1983,10 @@ default void dropConstraint(String catName, String dbName, String tableName, * false and there is no constraint of this name an exception will be thrown. * @throws NoSuchObjectException no constraint of this name exists and missingOk = false */ - void dropConstraint(String catName, String dbName, String tableName, String constraintName, - boolean missingOk) throws NoSuchObjectException; + default void dropConstraint(String catName, String dbName, String tableName, String constraintName, + boolean missingOk) throws NoSuchObjectException { + unwrap(ConstraintStore.class).dropConstraint(new TableName(catName, dbName, tableName), constraintName, missingOk); + } /** * Add a primary key to a table. @@ -1975,7 +1995,9 @@ void dropConstraint(String catName, String dbName, String tableName, String cons * @throws InvalidObjectException The SQLPrimaryKeys list is malformed * @throws MetaException error accessing the RDMBS */ - List addPrimaryKeys(List pks) throws InvalidObjectException, MetaException; + default List addPrimaryKeys(List pks) throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addPrimaryKeys(pks); + } /** * Add a foreign key to a table. @@ -1984,7 +2006,9 @@ void dropConstraint(String catName, String dbName, String tableName, String cons * @throws InvalidObjectException the specification is malformed. * @throws MetaException error accessing the RDBMS. */ - List addForeignKeys(List fks) throws InvalidObjectException, MetaException; + default List addForeignKeys(List fks) throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addForeignKeys(fks); + } /** * Add unique constraints to a table. @@ -1993,7 +2017,9 @@ void dropConstraint(String catName, String dbName, String tableName, String cons * @throws InvalidObjectException the specification is malformed. * @throws MetaException error accessing the RDBMS. */ - List addUniqueConstraints(List uks) throws InvalidObjectException, MetaException; + default List addUniqueConstraints(List uks) throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addUniqueConstraints(uks); + } /** * Add not null constraints to a table. @@ -2002,7 +2028,10 @@ void dropConstraint(String catName, String dbName, String tableName, String cons * @throws InvalidObjectException the specification is malformed. * @throws MetaException error accessing the RDBMS. */ - List addNotNullConstraints(List nns) throws InvalidObjectException, MetaException; + default List addNotNullConstraints(List nns) + throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addNotNullConstraints(nns); + } /** * Add default values to a table definition. @@ -2011,8 +2040,10 @@ void dropConstraint(String catName, String dbName, String tableName, String cons * @throws InvalidObjectException the specification is malformed. * @throws MetaException error accessing the RDBMS. */ - List addDefaultConstraints(List dv) - throws InvalidObjectException, MetaException; + default List addDefaultConstraints(List dv) + throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addDefaultConstraints(dv); + } /** * Add check constraints to a table. @@ -2021,7 +2052,10 @@ List addDefaultConstraints(List dv) * @throws InvalidObjectException the specification is malformed * @throws MetaException error accessing the RDBMS */ - List addCheckConstraints(List cc) throws InvalidObjectException, MetaException; + default List addCheckConstraints(List cc) + throws InvalidObjectException, MetaException { + return unwrap(ConstraintStore.class).addCheckConstraints(cc); + } /** * Gets the unique id of the backing datastore for the metadata. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java index 8032ecde335e..be0e1e509698 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java @@ -1,8 +1,179 @@ package org.apache.hadoop.hive.metastore.metastore.iface; +import java.util.List; + +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.api.AllTableConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest; +import org.apache.hadoop.hive.metastore.api.SQLAllTableConstraints; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; import org.apache.hadoop.hive.metastore.metastore.impl.ConstraintStoreImpl; @MetaDescriptor(alias = "constraint", defaultImpl = ConstraintStoreImpl.class) public interface ConstraintStore { + /** + * SQLPrimaryKey represents a single primary key column. + * Since a table can have one or more primary keys ( in case of composite primary key ), + * this method returns List<SQLPrimaryKey> + * @param request primary key request + * @return list of primary key columns or an empty list if the table does not have a primary key + * @throws MetaException error accessing the RDBMS + */ + List getPrimaryKeys(PrimaryKeysRequest request) + throws MetaException; + + /** + * SQLForeignKey represents a single foreign key column. + * Since a table can have one or more foreign keys ( in case of composite foreign key ), + * this method returns List<SQLForeignKey> + * @param request ForeignKeysRequest object + * @return List of all matching foreign key columns. Note that if more than one foreign key + * matches the arguments the results here will be all mixed together into a single list. + * @throws MetaException error access the RDBMS. + */ + List getForeignKeys(ForeignKeysRequest request) + throws MetaException; + + /** + * SQLUniqueConstraint represents a single unique constraint column. + * Since a table can have one or more unique constraint ( in case of composite unique constraint ), + * this method returns List<SQLUniqueConstraint> + * @param request UniqueConstraintsRequest object. + * @return list of unique constraints + * @throws MetaException error access the RDBMS. + */ + List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException; + + /** + * SQLNotNullConstraint represents a single not null constraint column. + * Since a table can have one or more not null constraint ( in case of composite not null constraint ), + * this method returns List<SQLNotNullConstraint> + * @param request NotNullConstraintsRequest object. + * @return list of not null constraints + * @throws MetaException error accessing the RDBMS. + */ + List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException; + + /** + * SQLDefaultConstraint represents a single default constraint column. + * Since a table can have one or more default constraint ( in case of composite default constraint ), + * this method returns List<SQLDefaultConstraint> + * @param request DefaultConstraintsRequest object. + * @return list of default values defined on the table. + * @throws MetaException error accessing the RDBMS + */ + List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException; + + /** + * SQLCheckConstraint represents a single check constraint column. + * Since a table can have one or more check constraint ( in case of composite check constraint ), + * this method returns List<SQLCheckConstraint> + * @param request CheckConstraintsRequest object. + * @return ccheck constraints for this table + * @throws MetaException error accessing the RDBMS + */ + List getCheckConstraints(CheckConstraintsRequest request) throws MetaException; + + /** + * Get table constraints + * @param request AllTableConstraintsRequest object + * @return all constraints for this table + * @throws MetaException + * @throws NoSuchObjectException + */ + SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest request) + throws MetaException, NoSuchObjectException; + + /** + * Create a table with constraints + * @param tbl table definition + * @param constraints wrapper of all table constraints + * @return list of constraint names + * @throws InvalidObjectException one of the provided objects is malformed. + * @throws MetaException error accessing the RDBMS + */ + SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) throws InvalidObjectException, MetaException; + + /** + * Drop a constraint, any constraint. I have no idea why add and get each have separate + * methods for each constraint type but drop has only one. + * @param tableName table name + * @param constraintName name of the constraint + * @param missingOk if true, it is not an error if there is no constraint of this name. If + * false and there is no constraint of this name an exception will be thrown. + * @throws NoSuchObjectException no constraint of this name exists and missingOk = false + */ + void dropConstraint(TableName tableName, String constraintName, + boolean missingOk) throws NoSuchObjectException; + + /** + * Add a primary key to a table. + * @param pks Columns in the primary key. + * @return the name of the constraint, as a list of strings. + * @throws InvalidObjectException The SQLPrimaryKeys list is malformed + * @throws MetaException error accessing the RDMBS + */ + List addPrimaryKeys(List pks) throws InvalidObjectException, MetaException; + + /** + * Add a foreign key to a table. + * @param fks foreign key specification + * @return foreign key name. + * @throws InvalidObjectException the specification is malformed. + * @throws MetaException error accessing the RDBMS. + */ + List addForeignKeys(List fks) throws InvalidObjectException, MetaException; + + /** + * Add unique constraints to a table. + * @param uks unique constraints specification + * @return unique constraint names. + * @throws InvalidObjectException the specification is malformed. + * @throws MetaException error accessing the RDBMS. + */ + List addUniqueConstraints(List uks) throws InvalidObjectException, MetaException; + + /** + * Add not null constraints to a table. + * @param nns not null constraint specifications + * @return constraint names. + * @throws InvalidObjectException the specification is malformed. + * @throws MetaException error accessing the RDBMS. + */ + List addNotNullConstraints(List nns) throws InvalidObjectException, MetaException; + + /** + * Add default values to a table definition. + * @param dv list of default values + * @return constraint names + * @throws InvalidObjectException the specification is malformed. + * @throws MetaException error accessing the RDBMS. + */ + List addDefaultConstraints(List dv) + throws InvalidObjectException, MetaException; + + /** + * Add check constraints to a table. + * @param cc check constraints to add + * @return list of constraint names + * @throws InvalidObjectException the specification is malformed + * @throws MetaException error accessing the RDBMS + */ + List addCheckConstraints(List cc) throws InvalidObjectException, MetaException; + } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java index 67b129f40b77..bb86ce0d4e34 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -14,9 +14,11 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.api.AllTableConstraintsRequest; import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; @@ -37,6 +39,7 @@ import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; +import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MFieldSchema; @@ -47,40 +50,35 @@ public class ConstraintStoreImpl extends RawStoreAware implements ConstraintStore { + private Configuration conf; + @Override public SQLAllTableConstraints createTableWithConstraints(Table tbl, SQLAllTableConstraints constraints) throws InvalidObjectException, MetaException { - boolean success = false; - try { - openTransaction(); - createTable(tbl); - // Add constraints. - // We need not do a deep retrieval of the Table Column Descriptor while persisting the - // constraints since this transaction involving create table is not yet committed. - if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) { - constraints.setForeignKeys(addForeignKeys(constraints.getForeignKeys(), false, constraints.getPrimaryKeys(), - constraints.getUniqueConstraints())); - } - if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) { - constraints.setPrimaryKeys(addPrimaryKeys(constraints.getPrimaryKeys(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) { - constraints.setUniqueConstraints(addUniqueConstraints(constraints.getUniqueConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) { - constraints.setNotNullConstraints(addNotNullConstraints(constraints.getNotNullConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) { - constraints.setDefaultConstraints(addDefaultConstraints(constraints.getDefaultConstraints(), false)); - } - if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) { - constraints.setCheckConstraints(addCheckConstraints(constraints.getCheckConstraints(), false)); - } - success = commitTransaction(); - return constraints; - } finally { - rollbackAndCleanup(success, null); + baseStore.unwrap(TableStore.class).createTable(tbl); + // Add constraints. + // We need not do a deep retrieval of the Table Column Descriptor while persisting the + // constraints since this transaction involving create table is not yet committed. + if (CollectionUtils.isNotEmpty(constraints.getForeignKeys())) { + constraints.setForeignKeys(addForeignKeys(constraints.getForeignKeys(), false, constraints.getPrimaryKeys(), + constraints.getUniqueConstraints())); + } + if (CollectionUtils.isNotEmpty(constraints.getPrimaryKeys())) { + constraints.setPrimaryKeys(addPrimaryKeys(constraints.getPrimaryKeys(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getUniqueConstraints())) { + constraints.setUniqueConstraints(addUniqueConstraints(constraints.getUniqueConstraints(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getNotNullConstraints())) { + constraints.setNotNullConstraints(addNotNullConstraints(constraints.getNotNullConstraints(), false)); + } + if (CollectionUtils.isNotEmpty(constraints.getDefaultConstraints())) { + constraints.setDefaultConstraints(addDefaultConstraints(constraints.getDefaultConstraints(), false)); } + if (CollectionUtils.isNotEmpty(constraints.getCheckConstraints())) { + constraints.setCheckConstraints(addCheckConstraints(constraints.getCheckConstraints(), false)); + } + return constraints; } private List listAllTableConstraintsWithOptionalConstraintName( @@ -92,7 +90,8 @@ private List listAllTableConstraintsWithOptionalConstraintName( List mConstraints = null; List constraintNames = new ArrayList<>(); - try (QueryWrapper queryForConstraintName = new QueryWrapper(pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where " + try (QueryWrapper queryForConstraintName = + new QueryWrapper(pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where " + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname && " + "parentTable.database.catalogName == pcatname) || " + "(childTable != null && childTable.tableName == ctblname &&" + @@ -127,23 +126,15 @@ private List listAllTableConstraintsWithOptionalConstraintName( return mConstraints; } - private boolean constraintNameAlreadyExists(MTable table, String constraintName) { - boolean commited = false; + private boolean constraintNameAlreadyExists(MTable table, String constraintName) { Query constraintExistsQuery = null; - String constraintNameIfExists = null; - try { - openTransaction(); - constraintName = normalizeIdentifier(constraintName); - constraintExistsQuery = pm.newQuery(MConstraint.class, - "parentTable == parentTableP && constraintName == constraintNameP"); - constraintExistsQuery.declareParameters("MTable parentTableP, java.lang.String constraintNameP"); - constraintExistsQuery.setUnique(true); - constraintExistsQuery.setResult("constraintName"); - constraintNameIfExists = (String) constraintExistsQuery.executeWithArray(table, constraintName); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, constraintExistsQuery); - } + constraintName = normalizeIdentifier(constraintName); + constraintExistsQuery = pm.newQuery(MConstraint.class, + "parentTable == parentTableP && constraintName == constraintNameP"); + constraintExistsQuery.declareParameters("MTable parentTableP, java.lang.String constraintNameP"); + constraintExistsQuery.setUnique(true); + constraintExistsQuery.setResult("constraintName"); + String constraintNameIfExists = (String) constraintExistsQuery.executeWithArray(table, constraintName); return constraintNameIfExists != null && !constraintNameIfExists.isEmpty(); } @@ -806,66 +797,49 @@ protected List getJdoResult( } private List getPrimaryKeysViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; List primaryKeys = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == cat_name &&" - + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + - "java.lang.String cat_name"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - primaryKeys = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPK = (MConstraint) i.next(); - List cols = currPK.getParentColumn() != null ? - currPK.getParentColumn().getCols() : currPK.getParentTable().getPartitionKeys(); - int enableValidateRely = currPK.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, - tblName, - cols.get(currPK.getParentIntegerIndex()).getName(), - currPK.getPosition(), - currPK.getConstraintName(), enable, validate, rely); - keyCol.setCatName(catName); - primaryKeys.add(keyCol); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == cat_name &&" + + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + + "java.lang.String cat_name"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + primaryKeys = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPK = (MConstraint) i.next(); + List cols = currPK.getParentColumn() != null ? + currPK.getParentColumn().getCols() : currPK.getParentTable().getPartitionKeys(); + int enableValidateRely = currPK.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, + tblName, + cols.get(currPK.getParentIntegerIndex()).getName(), + currPK.getPosition(), + currPK.getConstraintName(), enable, validate, rely); + keyCol.setCatName(catName); + primaryKeys.add(keyCol); } return primaryKeys; } private String getPrimaryKeyConstraintName(String catName, String dbName, String tblName) { - boolean commited = false; String ret = null; - Query query = null; - - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + - "java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPK = (MConstraint) i.next(); - ret = currPK.getConstraintName(); - break; - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.PRIMARY_KEY_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, " + + "java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPK = (MConstraint) i.next(); + ret = currPK.getConstraintName(); + break; } return ret; } @@ -920,86 +894,78 @@ protected List getJdoResult( private List getForeignKeysViaJdo(String catName, String parentDbName, String parentTblName, String foreignDbName, String foreignTblName) { - boolean commited = false; List foreignKeys = null; Collection constraints = null; - Query query = null; Map tblToConstraint = new HashMap<>(); - try { - openTransaction(); - String queryText = - " parentTable.database.catalogName == catName1 &&" + "childTable.database.catalogName == catName2 && " + ( - parentTblName != null ? "parentTable.tableName == parent_tbl_name && " : "") + ( - parentDbName != null ? " parentTable.database.name == parent_db_name && " : "") + ( - foreignTblName != null ? " childTable.tableName == foreign_tbl_name && " : "") + ( - foreignDbName != null ? " childTable.database.name == foreign_db_name && " : "") - + " constraintType == MConstraint.FOREIGN_KEY_CONSTRAINT"; - queryText = queryText.trim(); - query = pm.newQuery(MConstraint.class, queryText); - String paramText = "java.lang.String catName1, java.lang.String catName2" + ( - parentTblName == null ? "" : ", java.lang.String parent_tbl_name") + ( - parentDbName == null ? "" : " , java.lang.String parent_db_name") + ( - foreignTblName == null ? "" : ", java.lang.String foreign_tbl_name") + ( - foreignDbName == null ? "" : " , java.lang.String foreign_db_name"); - query.declareParameters(paramText); - List params = new ArrayList<>(); - params.add(catName); - params.add(catName); // This is not a mistake, catName is in the where clause twice - if (parentTblName != null) { - params.add(parentTblName); - } - if (parentDbName != null) { - params.add(parentDbName); - } - if (foreignTblName != null) { - params.add(foreignTblName); - } - if (foreignDbName != null) { - params.add(foreignDbName); - } - constraints = (Collection) query.executeWithArray(params.toArray(new String[0])); - - pm.retrieveAll(constraints); - foreignKeys = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currPKFK = (MConstraint) i.next(); - List parentCols = currPKFK.getParentColumn() != null ? - currPKFK.getParentColumn().getCols() : currPKFK.getParentTable().getPartitionKeys(); - List childCols = currPKFK.getChildColumn() != null ? - currPKFK.getChildColumn().getCols() : currPKFK.getChildTable().getPartitionKeys(); - int enableValidateRely = currPKFK.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - String consolidatedtblName = - currPKFK.getParentTable().getDatabase().getName() + "." + - currPKFK.getParentTable().getTableName(); - String pkName; - if (tblToConstraint.containsKey(consolidatedtblName)) { - pkName = tblToConstraint.get(consolidatedtblName); - } else { - pkName = getPrimaryKeyConstraintName(currPKFK.getParentTable().getDatabase().getCatalogName(), - currPKFK.getParentTable().getDatabase().getName(), - currPKFK.getParentTable().getTableName()); - tblToConstraint.put(consolidatedtblName, pkName); - } - SQLForeignKey fk = new SQLForeignKey( + String queryText = + " parentTable.database.catalogName == catName1 &&" + "childTable.database.catalogName == catName2 && " + ( + parentTblName != null ? "parentTable.tableName == parent_tbl_name && " : "") + ( + parentDbName != null ? " parentTable.database.name == parent_db_name && " : "") + ( + foreignTblName != null ? " childTable.tableName == foreign_tbl_name && " : "") + ( + foreignDbName != null ? " childTable.database.name == foreign_db_name && " : "") + + " constraintType == MConstraint.FOREIGN_KEY_CONSTRAINT"; + queryText = queryText.trim(); + Query query = pm.newQuery(MConstraint.class, queryText); + String paramText = "java.lang.String catName1, java.lang.String catName2" + ( + parentTblName == null ? "" : ", java.lang.String parent_tbl_name") + ( + parentDbName == null ? "" : " , java.lang.String parent_db_name") + ( + foreignTblName == null ? "" : ", java.lang.String foreign_tbl_name") + ( + foreignDbName == null ? "" : " , java.lang.String foreign_db_name"); + query.declareParameters(paramText); + List params = new ArrayList<>(); + params.add(catName); + params.add(catName); // This is not a mistake, catName is in the where clause twice + if (parentTblName != null) { + params.add(parentTblName); + } + if (parentDbName != null) { + params.add(parentDbName); + } + if (foreignTblName != null) { + params.add(foreignTblName); + } + if (foreignDbName != null) { + params.add(foreignDbName); + } + constraints = (Collection) query.executeWithArray(params.toArray(new String[0])); + + pm.retrieveAll(constraints); + foreignKeys = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currPKFK = (MConstraint) i.next(); + List parentCols = currPKFK.getParentColumn() != null ? + currPKFK.getParentColumn().getCols() : currPKFK.getParentTable().getPartitionKeys(); + List childCols = currPKFK.getChildColumn() != null ? + currPKFK.getChildColumn().getCols() : currPKFK.getChildTable().getPartitionKeys(); + int enableValidateRely = currPKFK.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + String consolidatedtblName = + currPKFK.getParentTable().getDatabase().getName() + "." + + currPKFK.getParentTable().getTableName(); + String pkName; + if (tblToConstraint.containsKey(consolidatedtblName)) { + pkName = tblToConstraint.get(consolidatedtblName); + } else { + pkName = getPrimaryKeyConstraintName(currPKFK.getParentTable().getDatabase().getCatalogName(), currPKFK.getParentTable().getDatabase().getName(), - currPKFK.getParentTable().getTableName(), - parentCols.get(currPKFK.getParentIntegerIndex()).getName(), - currPKFK.getChildTable().getDatabase().getName(), - currPKFK.getChildTable().getTableName(), - childCols.get(currPKFK.getChildIntegerIndex()).getName(), - currPKFK.getPosition(), - currPKFK.getUpdateRule(), - currPKFK.getDeleteRule(), - currPKFK.getConstraintName(), pkName, enable, validate, rely); - fk.setCatName(catName); - foreignKeys.add(fk); + currPKFK.getParentTable().getTableName()); + tblToConstraint.put(consolidatedtblName, pkName); } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + SQLForeignKey fk = new SQLForeignKey( + currPKFK.getParentTable().getDatabase().getName(), + currPKFK.getParentTable().getTableName(), + parentCols.get(currPKFK.getParentIntegerIndex()).getName(), + currPKFK.getChildTable().getDatabase().getName(), + currPKFK.getChildTable().getTableName(), + childCols.get(currPKFK.getChildIntegerIndex()).getName(), + currPKFK.getPosition(), + currPKFK.getUpdateRule(), + currPKFK.getDeleteRule(), + currPKFK.getConstraintName(), pkName, enable, validate, rely); + fk.setCatName(catName); + foreignKeys.add(fk); } return foreignKeys; } @@ -1037,33 +1003,25 @@ protected List getJdoResult(ObjectStore.GetHelper getUniqueConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; List uniqueConstraints = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name && parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.UNIQUE_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - uniqueConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - uniqueConstraints.add(new SQLUniqueConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getPosition(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name && parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.UNIQUE_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + uniqueConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + uniqueConstraints.add(new SQLUniqueConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getPosition(), + currConstraint.getConstraintName(), enable, validate, rely)); } return uniqueConstraints; } @@ -1141,68 +1099,52 @@ protected List getJdoResult(ObjectStore.GetHelper getCheckConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; List checkConstraints= null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName && constraintType == MConstraint.CHECK_CONSTRAINT"); - query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - checkConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - checkConstraints.add(new SQLCheckConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), - currConstraint.getDefaultValue(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName && constraintType == MConstraint.CHECK_CONSTRAINT"); + query.declareParameters("java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + checkConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + checkConstraints.add(new SQLCheckConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), + currConstraint.getDefaultValue(), + currConstraint.getConstraintName(), enable, validate, rely)); } return checkConstraints; } private List getDefaultConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; List defaultConstraints= null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName &&" - + " constraintType == MConstraint.DEFAULT_CONSTRAINT"); - query.declareParameters( - "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - defaultConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - defaultConstraints.add(new SQLDefaultConstraint(catName, dbName, tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getDefaultValue(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName &&" + + " constraintType == MConstraint.DEFAULT_CONSTRAINT"); + query.declareParameters( + "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + defaultConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + defaultConstraints.add(new SQLDefaultConstraint(catName, dbName, tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), currConstraint.getDefaultValue(), + currConstraint.getConstraintName(), enable, validate, rely)); } return defaultConstraints; } @@ -1230,35 +1172,27 @@ protected List getJdoResult(ObjectStore.GetHelper getNotNullConstraintsViaJdo(String catName, String dbName, String tblName) { - boolean commited = false; List notNullConstraints = null; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MConstraint.class, - "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" - + " parentTable.database.catalogName == catName && constraintType == MConstraint.NOT_NULL_CONSTRAINT"); - query.declareParameters( - "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); - Collection constraints = (Collection) query.execute(tblName, dbName, catName); - pm.retrieveAll(constraints); - notNullConstraints = new ArrayList<>(); - for (Iterator i = constraints.iterator(); i.hasNext();) { - MConstraint currConstraint = (MConstraint) i.next(); - List cols = currConstraint.getParentColumn() != null ? - currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); - int enableValidateRely = currConstraint.getEnableValidateRely(); - boolean enable = (enableValidateRely & 4) != 0; - boolean validate = (enableValidateRely & 2) != 0; - boolean rely = (enableValidateRely & 1) != 0; - notNullConstraints.add(new SQLNotNullConstraint(catName, dbName, - tblName, - cols.get(currConstraint.getParentIntegerIndex()).getName(), - currConstraint.getConstraintName(), enable, validate, rely)); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + Query query = pm.newQuery(MConstraint.class, + "parentTable.tableName == tbl_name && parentTable.database.name == db_name &&" + + " parentTable.database.catalogName == catName && constraintType == MConstraint.NOT_NULL_CONSTRAINT"); + query.declareParameters( + "java.lang.String tbl_name, java.lang.String db_name, java.lang.String catName"); + Collection constraints = (Collection) query.execute(tblName, dbName, catName); + pm.retrieveAll(constraints); + notNullConstraints = new ArrayList<>(); + for (Iterator i = constraints.iterator(); i.hasNext();) { + MConstraint currConstraint = (MConstraint) i.next(); + List cols = currConstraint.getParentColumn() != null ? + currConstraint.getParentColumn().getCols() : currConstraint.getParentTable().getPartitionKeys(); + int enableValidateRely = currConstraint.getEnableValidateRely(); + boolean enable = (enableValidateRely & 4) != 0; + boolean validate = (enableValidateRely & 2) != 0; + boolean rely = (enableValidateRely & 1) != 0; + notNullConstraints.add(new SQLNotNullConstraint(catName, dbName, + tblName, + cols.get(currConstraint.getParentIntegerIndex()).getName(), + currConstraint.getConstraintName(), enable, validate, rely)); } return notNullConstraints; } @@ -1276,8 +1210,6 @@ public SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest String catName = request.getCatName(); String dbName = request.getDbName(); String tblName = request.getTblName(); - debugLog("Get all table constraints for the table - " + catName + "." + dbName + "." + tblName - + " in class ObjectStore.java"); SQLAllTableConstraints sqlAllTableConstraints = new SQLAllTableConstraints(); PrimaryKeysRequest primaryKeysRequest = new PrimaryKeysRequest(dbName, tblName); primaryKeysRequest.setCatName(catName); @@ -1298,24 +1230,18 @@ public SQLAllTableConstraints getAllTableConstraints(AllTableConstraintsRequest } @Override - public void dropConstraint(String catName, String dbName, String tableName, - String constraintName, boolean missingOk) + public void dropConstraint(TableName tableName, String constraintName, boolean missingOk) throws NoSuchObjectException { - boolean success = false; - try { - openTransaction(); - - List tabConstraints = - listAllTableConstraintsWithOptionalConstraintName(catName, dbName, tableName, constraintName); - if (CollectionUtils.isNotEmpty(tabConstraints)) { - pm.deletePersistentAll(tabConstraints); - } else if (!missingOk) { - throw new NoSuchObjectException("The constraint: " + constraintName + - " does not exist for the associated table: " + dbName + "." + tableName); - } - success = commitTransaction(); - } finally { - rollbackAndCleanup(success, null); + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); + List tabConstraints = + listAllTableConstraintsWithOptionalConstraintName(catName, dbName, tblName, constraintName); + if (CollectionUtils.isNotEmpty(tabConstraints)) { + pm.deletePersistentAll(tabConstraints); + } else if (!missingOk) { + throw new NoSuchObjectException("The constraint: " + constraintName + + " does not exist for the associated table: " + dbName + "." + tblName); } } @@ -1344,6 +1270,12 @@ private static int getColumnIndexFromTableColumns(List cols, Strin return -1; } + @Override + public void setBaseStore(RawStore store) { + super.setBaseStore(store); + this.conf = store.getConf(); + } + class AttachedMTableInfo { MTable mtbl; MColumnDescriptor mcd; From f052b8bc75aa2df60bde994815719b169b629fbc Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 07:22:33 +0800 Subject: [PATCH 04/22] getForeignKeysInternal --- .../metastore/impl/ConstraintStoreImpl.java | 179 +++++++----------- 1 file changed, 65 insertions(+), 114 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java index bb86ce0d4e34..c2a1d8321cce 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.metastore.metastore.impl; import javax.jdo.Query; @@ -8,7 +26,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import org.apache.commons.collections4.CollectionUtils; @@ -23,6 +40,7 @@ import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -37,6 +55,7 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; +import org.apache.hadoop.hive.metastore.metastore.GetListHelper; import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; @@ -49,7 +68,6 @@ import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; public class ConstraintStoreImpl extends RawStoreAware implements ConstraintStore { - private Configuration conf; @Override @@ -190,6 +208,7 @@ private List addForeignKeys(List foreignKeys, bool if (childTable == null) { throw new InvalidObjectException("Child table not found: " + fkTableName); } + MColumnDescriptor childCD = retrieveCD ? nChildTable.mcd : childTable.getSd().getCD(); final List childCols = childCD == null || childCD.getCols() == null ? new ArrayList<>() : new ArrayList<>(childCD.getCols()); @@ -781,16 +800,15 @@ private List getPrimaryKeysInternal(final String catName, throws MetaException, NoSuchObjectException { final String dbName = dbNameInput != null ? normalizeIdentifier(dbNameInput) : null; final String tblName = normalizeIdentifier(tblNameInput); - return new ObjectStore.GetListHelper(catName, dbName, tblName, true, true) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) throws MetaException { - return directSql.getPrimaryKeys(catName, dbName, tblName); + protected List getSqlResult() throws MetaException { + return getDirectSql().getPrimaryKeys(catName, dbName, tblName); } @Override - protected List getJdoResult( - ObjectStore.GetHelper> ctx) throws MetaException, NoSuchObjectException { + protected List getJdoResult() throws MetaException, NoSuchObjectException { return getPrimaryKeysViaJdo(catName, dbName, tblName); } }.run(false); @@ -849,8 +867,7 @@ public List getForeignKeys(ForeignKeysRequest request) throws Met try { return getForeignKeysInternal(request.getCatName(), request.getParent_db_name(), request.getParent_tbl_name() , - request.getForeign_db_name(),request.getForeign_tbl_name(), true, - true); + request.getForeign_db_name(),request.getForeign_tbl_name()); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } @@ -858,8 +875,7 @@ public List getForeignKeys(ForeignKeysRequest request) throws Met private List getForeignKeysInternal( final String catName, final String parent_db_name_input, final String parent_tbl_name_input, - final String foreign_db_name_input, final String foreign_tbl_name_input, boolean allowSql, - boolean allowJdo) throws MetaException, NoSuchObjectException { + final String foreign_db_name_input, final String foreign_tbl_name_input) throws MetaException, NoSuchObjectException { final String parent_db_name = (parent_db_name_input != null) ? normalizeIdentifier(parent_db_name_input) : null; final String parent_tbl_name = (parent_tbl_name_input != null) ? normalizeIdentifier(parent_tbl_name_input) : null; final String foreign_db_name = (foreign_db_name_input != null) ? normalizeIdentifier(foreign_db_name_input) : null; @@ -875,17 +891,16 @@ private List getForeignKeysInternal( db_name = foreign_db_name; tbl_name = foreign_tbl_name; } - return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) throws MetaException { - return directSql.getForeignKeys(catName, parent_db_name, + protected List getSqlResult() throws MetaException { + return getDirectSql().getForeignKeys(catName, parent_db_name, parent_tbl_name, foreign_db_name, foreign_tbl_name); } @Override - protected List getJdoResult( - ObjectStore.GetHelper> ctx) throws MetaException, NoSuchObjectException { + protected List getJdoResult() throws MetaException, NoSuchObjectException { return getForeignKeysViaJdo(catName, parent_db_name, parent_tbl_name, foreign_db_name, foreign_tbl_name); } @@ -974,28 +989,28 @@ private List getForeignKeysViaJdo(String catName, String parentDb public List getUniqueConstraints(UniqueConstraintsRequest request) throws MetaException { try { return getUniqueConstraintsInternal(request.getCatName(), - request.getDb_name(),request.getTbl_name(), true, true); + request.getDb_name(),request.getTbl_name()); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } } private List getUniqueConstraintsInternal( - String catNameInput, final String db_name_input, final String tbl_name_input, - boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + String catNameInput, final String db_name_input, final String tbl_name_input) + throws MetaException, NoSuchObjectException { final String catName = normalizeIdentifier(catNameInput); final String db_name = normalizeIdentifier(db_name_input); final String tbl_name = normalizeIdentifier(tbl_name_input); - return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) + protected List getSqlResult() throws MetaException { - return directSql.getUniqueConstraints(catName, db_name, tbl_name); + return getDirectSql().getUniqueConstraints(catName, db_name, tbl_name); } @Override - protected List getJdoResult(ObjectStore.GetHelper> ctx) + protected List getJdoResult() throws MetaException, NoSuchObjectException { return getUniqueConstraintsViaJdo(catName, db_name, tbl_name); } @@ -1029,7 +1044,7 @@ private List getUniqueConstraintsViaJdo(String catName, Str @Override public List getNotNullConstraints(NotNullConstraintsRequest request) throws MetaException { try { - return getNotNullConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + return getNotNullConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name()); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } @@ -1038,7 +1053,7 @@ public List getNotNullConstraints(NotNullConstraintsReques @Override public List getDefaultConstraints(DefaultConstraintsRequest request) throws MetaException { try { - return getDefaultConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + return getDefaultConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name()); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } @@ -1047,52 +1062,43 @@ public List getDefaultConstraints(DefaultConstraintsReques @Override public List getCheckConstraints(CheckConstraintsRequest request) throws MetaException { try { - return getCheckConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name(), true, true); + return getCheckConstraintsInternal(request.getCatName(),request.getDb_name(),request.getTbl_name()); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } } - private List getDefaultConstraintsInternal( - String catName, final String db_name_input, final String tbl_name_input, boolean allowSql, - boolean allowJdo) throws MetaException, NoSuchObjectException { - catName = normalizeIdentifier(catName); - final String db_name = normalizeIdentifier(db_name_input); - final String tbl_name = normalizeIdentifier(tbl_name_input); - return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { - + private List getDefaultConstraintsInternal(String catalog, String dbName, String tableName) + throws MetaException, NoSuchObjectException { + final String catName = normalizeIdentifier(catalog); + final String db_name = normalizeIdentifier(dbName); + final String tbl_name = normalizeIdentifier(tableName); + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) - throws MetaException { - return directSql.getDefaultConstraints(catName, db_name, tbl_name); + protected List getSqlResult() throws MetaException { + return getDirectSql().getDefaultConstraints(catName, db_name, tbl_name); } - @Override - protected List getJdoResult(ObjectStore.GetHelper> ctx) - throws MetaException, NoSuchObjectException { + protected List getJdoResult() + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { return getDefaultConstraintsViaJdo(catName, db_name, tbl_name); } }.run(false); } protected List getCheckConstraintsInternal(String catName, final String db_name_input, - final String tbl_name_input, boolean allowSql, - boolean allowJdo) - throws MetaException, NoSuchObjectException { + final String tbl_name_input) throws MetaException, NoSuchObjectException { final String db_name = normalizeIdentifier(db_name_input); final String tbl_name = normalizeIdentifier(tbl_name_input); - return new ObjectStore.GetListHelper(normalizeIdentifier(catName), db_name, tbl_name, - allowSql, allowJdo) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) - throws MetaException { - return directSql.getCheckConstraints(catName, db_name, tbl_name); + protected List getSqlResult() throws MetaException { + return getDirectSql().getCheckConstraints(catName, db_name, tbl_name); } - @Override - protected List getJdoResult(ObjectStore.GetHelper> ctx) - throws MetaException, NoSuchObjectException { + protected List getJdoResult() + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { return getCheckConstraintsViaJdo(catName, db_name, tbl_name); } }.run(false); @@ -1150,23 +1156,21 @@ private List getDefaultConstraintsViaJdo(String catName, S } protected List getNotNullConstraintsInternal(String catName, final String db_name_input, - final String tbl_name_input, boolean allowSql, boolean allowJdo) - throws MetaException, NoSuchObjectException { - catName = normalizeIdentifier(catName); + final String tbl_name_input) throws MetaException, NoSuchObjectException { + final String cat_name = normalizeIdentifier(catName); final String db_name = normalizeIdentifier(db_name_input); final String tbl_name = normalizeIdentifier(tbl_name_input); - return new ObjectStore.GetListHelper(catName, db_name, tbl_name, allowSql, allowJdo) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult(ObjectStore.GetHelper> ctx) - throws MetaException { - return directSql.getNotNullConstraints(catName, db_name, tbl_name); + protected List getSqlResult() throws MetaException { + return getDirectSql().getNotNullConstraints(cat_name, db_name, tbl_name); } @Override - protected List getJdoResult(ObjectStore.GetHelper> ctx) - throws MetaException, NoSuchObjectException { - return getNotNullConstraintsViaJdo(catName, db_name, tbl_name); + protected List getJdoResult() + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + return getNotNullConstraintsViaJdo(cat_name, db_name, tbl_name); } }.run(false); } @@ -1275,57 +1279,4 @@ public void setBaseStore(RawStore store) { super.setBaseStore(store); this.conf = store.getConf(); } - - class AttachedMTableInfo { - MTable mtbl; - MColumnDescriptor mcd; - - public AttachedMTableInfo() {} - - public AttachedMTableInfo(MTable mtbl, MColumnDescriptor mcd) { - this.mtbl = mtbl; - this.mcd = mcd; - } - } - - private AttachedMTableInfo getMTable(String catName, String db, String table, - boolean retrieveCD) { - AttachedMTableInfo nmtbl = new AttachedMTableInfo(); - MTable mtbl = null; - boolean commited = false; - Query query = null; - try { - openTransaction(); - catName = normalizeIdentifier(Optional.ofNullable(catName).orElse(getDefaultCatalog(conf))); - db = normalizeIdentifier(db); - table = normalizeIdentifier(table); - query = pm.newQuery(MTable.class, - "tableName == table && database.name == db && database.catalogName == catname"); - query.declareParameters( - "java.lang.String table, java.lang.String db, java.lang.String catname"); - query.setUnique(true); - if (LOG.isDebugEnabled()) { - LOG.debug("Executing getMTable for {}", - TableName.getQualified(catName, db, table)); - } - mtbl = (MTable) query.execute(table, db, catName); - pm.retrieve(mtbl); - // Retrieving CD can be expensive and unnecessary, so do it only when required. - if (mtbl != null && retrieveCD) { - pm.retrieve(mtbl.getSd()); - pm.retrieveAll(mtbl.getSd().getCD()); - nmtbl.mcd = mtbl.getSd().getCD(); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - nmtbl.mtbl = mtbl; - return nmtbl; - } - - private MTable getMTable(String catName, String db, String table) { - AttachedMTableInfo nmtbl = getMTable(catName, db, table, false); - return nmtbl.mtbl; - } } From aab20d4c3b00e6f7a6e2ed985f894acbb1204cb9 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 07:37:34 +0800 Subject: [PATCH 05/22] col stats --- .../hadoop/hive/metastore/ObjectStore.java | 6 + .../hadoop/hive/metastore/RawStore.java | 100 +++++++--- .../metastore/iface/ColStatsStore.java | 178 ++++++++++++++++++ 3 files changed, 254 insertions(+), 30 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 6f8ad482be6a..94dab79b539d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -67,6 +67,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Catalog; @@ -4916,6 +4917,11 @@ public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperat } } + @Override + public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) { + + } + private void recoverInvalidScheduledQueryState(int timeoutSecs) { int maxLastUpdateTime = (int) (System.currentTimeMillis() / 1000) - timeoutSecs; try (QueryWrapper q = new QueryWrapper(pm.newQuery(MScheduledQuery.class))) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index eadb234111fd..12905bad655e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1298,9 +1298,11 @@ default Map updateTableColumnStatistics(ColumnStatistics colStat * @throws InvalidInputException unable to record the stats for the table */ @Deprecated - Map updatePartitionColumnStatistics(ColumnStatistics statsObj, + default Map updatePartitionColumnStatistics(ColumnStatistics statsObj, List partVals, String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + throw new UnsupportedOperationException(); + } default Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics statsObj, List partVals, @@ -1321,8 +1323,10 @@ default Map updatePartitionColumnStatistics(Table table, MTable * @throws MetaException error accessing the RDBMS * */ - List getTableColumnStatistics(String catName, String dbName, String tableName, - List colName) throws MetaException, NoSuchObjectException; + default List getTableColumnStatistics(String catName, String dbName, String tableName, + List colName) throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class).getTableColumnStatistics(new TableName(catName, dbName, tableName), colName); + } /** * Returns the relevant column statistics for a given column in a given table in a given database @@ -1337,8 +1341,11 @@ List getTableColumnStatistics(String catName, String dbName, S * @throws MetaException error accessing the RDBMS * */ - ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName, - List colName, String engine) throws MetaException, NoSuchObjectException; + default ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName, + List colName, String engine) throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .getTableColumnStatistics(new TableName(catName, dbName, tableName), colName, engine); + } /** * Returns the relevant column statistics for a given column in a given table in a given database @@ -1354,10 +1361,13 @@ ColumnStatistics getTableColumnStatistics(String catName, String dbName, String * @throws MetaException error accessing the RDBMS * */ - ColumnStatistics getTableColumnStatistics( + default ColumnStatistics getTableColumnStatistics( String catName, String dbName, String tableName, List colName, String engine, String writeIdList) - throws MetaException, NoSuchObjectException; + throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .getTableColumnStatistics(new TableName(catName, dbName, tableName), colName, engine, writeIdList); + } /** * Get statistics for a partition for a set of columns. @@ -1370,9 +1380,12 @@ ColumnStatistics getTableColumnStatistics( * @throws MetaException error accessing the RDBMS * @throws NoSuchObjectException no such partition. */ - List> getPartitionColumnStatistics( + default List> getPartitionColumnStatistics( String catName, String dbName, String tblName, List partNames, List colNames) - throws MetaException, NoSuchObjectException; + throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .getPartitionColumnStatistics(new TableName(catName, dbName, tblName), partNames, colNames); + } /** * Get statistics for a partition for a set of columns. @@ -1386,9 +1399,12 @@ List> getPartitionColumnStatistics( * @throws MetaException error accessing the RDBMS * @throws NoSuchObjectException no such partition. */ - List getPartitionColumnStatistics( + default List getPartitionColumnStatistics( String catName, String dbName, String tblName, List partNames, List colNames, - String engine) throws MetaException, NoSuchObjectException; + String engine) throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .getPartitionColumnStatistics(new TableName(catName, dbName, tblName), partNames, colNames, engine); + } /** * Get statistics for a partition for a set of columns. @@ -1403,11 +1419,14 @@ List getPartitionColumnStatistics( * @throws MetaException error accessing the RDBMS * @throws NoSuchObjectException no such partition. */ - List getPartitionColumnStatistics( + default List getPartitionColumnStatistics( String catName, String dbName, String tblName, List partNames, List colNames, String engine, String writeIdList) - throws MetaException, NoSuchObjectException; + throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .getPartitionColumnStatistics(new TableName(catName, dbName, tblName), partNames, colNames, engine, writeIdList); + } /** * Deletes column statistics if present associated with a given db, table, partition and a list of cols. If @@ -1425,9 +1444,12 @@ List getPartitionColumnStatistics( * @throws InvalidObjectException error dropping the stats * @throws InvalidInputException bad input, such as null table or database name. */ - boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, + default boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, List partNames, List colNames, String engine) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return unwrap(ColStatsStore.class) + .deletePartitionColumnStatistics(new TableName(catName, dbName, tableName), partNames, colNames, engine); + } /** * Delete statistics for a single column, a list of columns or all columns in a table. @@ -1442,9 +1464,11 @@ boolean deletePartitionColumnStatistics(String catName, String dbName, String ta * @throws InvalidObjectException error dropping the stats * @throws InvalidInputException bad inputs, such as null table name. */ - boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, + default boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, List colNames, String engine) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return unwrap(ColStatsStore.class).deleteTableColumnStatistics(new TableName(catName, dbName, tableName), colNames, engine); + } long cleanupEvents(); @@ -1697,8 +1721,11 @@ List getFunctionsRequest(String catName, String dbName, String pattern, * @throws MetaException error accessing RDBMS * @throws NoSuchObjectException no such table or partition */ - AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, - List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException; + default AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, + List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .get_aggr_stats_for(new TableName(catName, dbName, tblName), partNames, colNames, engine); + } /** * Get aggregated stats for a table or partition(s). @@ -1714,10 +1741,13 @@ AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, * @throws MetaException error accessing RDBMS * @throws NoSuchObjectException no such table or partition */ - AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, + default AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, List partNames, List colNames, String engine, String writeIdList) - throws MetaException, NoSuchObjectException; + throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class) + .get_aggr_stats_for(new TableName(catName, dbName, tblName), partNames, colNames, engine, writeIdList); + } /** * Get column stats for all partitions of all tables in the database @@ -1727,8 +1757,10 @@ AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, * @throws MetaException error accessing RDBMS * @throws NoSuchObjectException no such database */ - List getPartitionColStatsForDatabase(String catName, String dbName) - throws MetaException, NoSuchObjectException; + default List getPartitionColStatsForDatabase(String catName, String dbName) + throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class).getPartitionColStatsForDatabase(catName, dbName); + } /** * Get the next notification event. @@ -2260,12 +2292,18 @@ List getSchemaVersionsByColumns(String colName, String colNamespa /** Removes outdated statistics. */ int deleteRuntimeStats(int maxRetainSecs) throws MetaException; - List getTableNamesWithStats() throws MetaException, NoSuchObjectException; + default List getTableNamesWithStats() throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class).getTableNamesWithStats(); + } - List getAllTableNamesForStats() throws MetaException, NoSuchObjectException; + default List getAllTableNamesForStats() throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class).getAllTableNamesForStats(); + } - Map> getPartitionColsWithStats(String catName, String dbName, - String tableName) throws MetaException, NoSuchObjectException; + default Map> getPartitionColsWithStats(String catName, String dbName, + String tableName) throws MetaException, NoSuchObjectException { + return unwrap(ColStatsStore.class).getPartitionColsWithStats(new TableName(catName, dbName, tableName)); + } /** * Remove older notification events. @@ -2334,11 +2372,13 @@ void scheduledQueryProgress(ScheduledQueryProgressInfo info) */ ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest replicationMetricsRequest); - Map> updatePartitionColumnStatisticsInBatch( + default Map> updatePartitionColumnStatisticsInBatch( Map partColStatsMap, Table tbl, List listeners, String validWriteIds, long writeId) - throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return unwrap(ColStatsStore.class).updatePartitionColumnStatisticsInBatch(partColStatsMap, tbl, listeners, validWriteIds, writeId); + } int deleteReplicationMetrics(int maxRetainSecs); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java index ed2d582487cb..78254dacc94f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java @@ -20,6 +20,9 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; @@ -29,6 +32,7 @@ import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; import org.apache.hadoop.hive.metastore.metastore.impl.ColStatsStoreImpl; import org.apache.hadoop.hive.metastore.model.MTable; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; @MetaDescriptor(alias = "stats", defaultImpl = ColStatsStoreImpl.class) public interface ColStatsStore { @@ -47,4 +51,178 @@ Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics statsObj, List partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + + /** + * Returns the relevant column statistics for a given column in a given table in a given database + * if such statistics exist. + * @param tableName name of the table + * @param colName names of the columns for which statistics is requested + * @return Relevant column statistics for the column for the given table + * @throws NoSuchObjectException No such table + * @throws MetaException error accessing the RDBMS + * + */ + List getTableColumnStatistics(TableName tableName, + List colName) throws MetaException, NoSuchObjectException; + + /** + * Returns the relevant column statistics for a given column in a given table in a given database + * if such statistics exist. + * @param tableName name of the table + * @param colName names of the columns for which statistics is requested + * @param engine engine requesting the statistics + * @return Relevant column statistics for the column for the given table + * @throws NoSuchObjectException No such table + * @throws MetaException error accessing the RDBMS + * + */ + ColumnStatistics getTableColumnStatistics(TableName tableName, + List colName, String engine) throws MetaException, NoSuchObjectException; + + /** + * Returns the relevant column statistics for a given column in a given table in a given database + * if such statistics exist. + * @param tableName name of the table + * @param colName names of the columns for which statistics is requested + * @param engine engine requesting the statistics + * @param writeIdList string format of valid writeId transaction list + * @return Relevant column statistics for the column for the given table + * @throws NoSuchObjectException No such table + * @throws MetaException error accessing the RDBMS + * + */ + ColumnStatistics getTableColumnStatistics( + TableName tableName, + List colName, String engine, String writeIdList) + throws MetaException, NoSuchObjectException; + + /** + * Get statistics for a partition for a set of columns. + * @param tableName table name. + * @param partNames list of partition names. These are names so must be key1=val1[/key2=val2...] + * @param colNames list of columns to get stats for + * @return list of statistics objects + * @throws MetaException error accessing the RDBMS + * @throws NoSuchObjectException no such partition. + */ + List> getPartitionColumnStatistics( + TableName tableName, List partNames, List colNames) + throws MetaException, NoSuchObjectException; + + /** + * Get statistics for a partition for a set of columns. + * @param tableName table name. + * @param partNames list of partition names. These are names so must be key1=val1[/key2=val2...] + * @param colNames list of columns to get stats for + * @param engine engine requesting the statistics + * @return list of statistics objects + * @throws MetaException error accessing the RDBMS + * @throws NoSuchObjectException no such partition. + */ + List getPartitionColumnStatistics( + TableName tableName, List partNames, List colNames, + String engine) throws MetaException, NoSuchObjectException; + + /** + * Get statistics for a partition for a set of columns. + * @param tableName table name. + * @param partNames list of partition names. These are names so must be key1=val1[/key2=val2...] + * @param colNames list of columns to get stats for + * @param engine engine requesting the statistics + * @param writeIdList string format of valid writeId transaction list + * @return list of statistics objects + * @throws MetaException error accessing the RDBMS + * @throws NoSuchObjectException no such partition. + */ + List getPartitionColumnStatistics( + TableName tableName, + List partNames, List colNames, + String engine, String writeIdList) + throws MetaException, NoSuchObjectException; + + /** + * Deletes column statistics if present associated with a given db, table, partition and a list of cols. If + * null is passed instead of a colName, stats when present for all columns associated + * with a given db, table and partition are deleted. + * @param tableName table name. + * @param partNames partition names. + * @param colNames a list of column names. + * @param engine engine for which we want to delete statistics + * @return Boolean indicating the outcome of the operation + * @throws NoSuchObjectException no such partition + * @throws MetaException error access the RDBMS + * @throws InvalidObjectException error dropping the stats + * @throws InvalidInputException bad input, such as null table or database name. + */ + boolean deletePartitionColumnStatistics(TableName tableName, + List partNames, List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + + /** + * Delete statistics for a single column, a list of columns or all columns in a table. + * @param tableName table name + * @param colNames a list of column names. Null to delete stats for all columns in the table. + * @param engine engine for which we want to delete statistics + * @return true if the statistics were deleted. + * @throws NoSuchObjectException no such table or column. + * @throws MetaException error access the RDBMS. + * @throws InvalidObjectException error dropping the stats + * @throws InvalidInputException bad inputs, such as null table name. + */ + boolean deleteTableColumnStatistics(TableName tableName, + List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + + /** + * Get aggregated stats for a table or partition(s). + * @param tableName table name. + * @param partNames list of partition names. These are the names of the partitions, not + * values. + * @param colNames list of column names + * @param engine engine requesting the statistics + * @return aggregated stats + * @throws MetaException error accessing RDBMS + * @throws NoSuchObjectException no such table or partition + */ + AggrStats get_aggr_stats_for(TableName tableName, + List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException; + + /** + * Get aggregated stats for a table or partition(s). + * @param partNames list of partition names. These are the names of the partitions, not + * values. + * @param colNames list of column names + * @param engine engine requesting the statistics + * @param writeIdList string format of valid writeId transaction list + * @return aggregated stats + * @throws MetaException error accessing RDBMS + * @throws NoSuchObjectException no such table or partition + */ + AggrStats get_aggr_stats_for(TableName tableName, + List partNames, List colNames, + String engine, String writeIdList) + throws MetaException, NoSuchObjectException; + + Map> updatePartitionColumnStatisticsInBatch( + Map partColStatsMap, + Table tbl, List listeners, + String validWriteIds, long writeId) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException; + + /** + * Get column stats for all partitions of all tables in the database + * @param catName catalog name + * @param dbName database name + * @return List of column stats objects for all partitions of all tables in the database + * @throws MetaException error accessing RDBMS + * @throws NoSuchObjectException no such database + */ + List getPartitionColStatsForDatabase(String catName, String dbName) + throws MetaException, NoSuchObjectException; + + List getTableNamesWithStats() throws MetaException, NoSuchObjectException; + + List getAllTableNamesForStats() throws MetaException, NoSuchObjectException; + + Map> getPartitionColsWithStats(TableName tableName) throws MetaException, NoSuchObjectException; } From 1102849e2d939a4fe363d62dd8b5402be12dbe79 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 08:05:13 +0800 Subject: [PATCH 06/22] part=2 --- .../hadoop/hive/metastore/RawStore.java | 7 +- .../metastore/impl/ColStatsStoreImpl.java | 741 ++++++++---------- 2 files changed, 348 insertions(+), 400 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 12905bad655e..9cc76a57d19b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; @@ -1301,7 +1302,11 @@ default Map updateTableColumnStatistics(ColumnStatistics colStat default Map updatePartitionColumnStatistics(ColumnStatistics statsObj, List partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - throw new UnsupportedOperationException(); + ColumnStatisticsDesc statsDesc = statsObj.getStatsDesc(); + Table table = unwrap(TableStore.class) + .getTable(new TableName(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()), null, -1); + MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); + return updatePartitionColumnStatistics(table, mTable, statsObj, partVals, validWriteIds, writeId); } default Map updatePartitionColumnStatistics(Table table, MTable mTable, diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 277f1e545b4b..649af3f61a72 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -72,8 +72,11 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; +import org.apache.hadoop.hive.metastore.metastore.GetHelper; +import org.apache.hadoop.hive.metastore.metastore.GetListHelper; import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; +import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; import org.apache.hadoop.hive.metastore.model.MTable; @@ -81,39 +84,48 @@ import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.RetryingExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers; public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { + private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class); @Override public List getTableNamesWithStats() throws MetaException, NoSuchObjectException { - return new ObjectStore.GetListHelper(null, null, null, true, false) { + return new GetListHelper (this, null) { @Override - protected List getSqlResult( - ObjectStore.GetHelper> ctx) throws MetaException { - return directSql.getTableNamesWithStats(); + protected List getSqlResult() throws MetaException { + return getDirectSql().getTableNamesWithStats(); } @Override - protected List getJdoResult( - ObjectStore.GetHelper> ctx) throws MetaException { + protected boolean canUseJdoQuery() throws MetaException { + return false; + } + + @Override + protected List getJdoResult() throws MetaException { throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? } }.run(false); } @Override - public Map> getPartitionColsWithStats(String catName, String dbName, String tableName) + public Map> getPartitionColsWithStats(TableName tableName) throws MetaException, NoSuchObjectException { - return new ObjectStore.GetHelper>>(catName, dbName, null, true, false) { + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); + return new GetHelper>>(this, null) { @Override - protected Map> getSqlResult( - ObjectStore.GetHelper>> ctx) throws MetaException { + protected Map> getSqlResult() throws MetaException { try { - return directSql.getColAndPartNamesWithStats(catName, dbName, tableName); + return getDirectSql().getColAndPartNamesWithStats(catName, dbName, tblName); } catch (Throwable ex) { LOG.error("DirectSQL failed", ex); throw new MetaException(ex.getMessage()); @@ -121,8 +133,12 @@ protected Map> getSqlResult( } @Override - protected Map> getJdoResult( - ObjectStore.GetHelper>> ctx) throws MetaException { + protected boolean canUseJdoQuery() throws MetaException { + return false; + } + + @Override + protected Map> getJdoResult() throws MetaException { throw new UnsupportedOperationException("UnsupportedOperationException"); // TODO: implement? } @@ -135,42 +151,32 @@ protected String describeResult() { @Override public List getAllTableNamesForStats() throws MetaException, NoSuchObjectException { - return new ObjectStore.GetListHelper(null, null, null, true, false) { + return new GetListHelper(this, null) { @Override - protected List getSqlResult( - ObjectStore.GetHelper> ctx) throws MetaException { - return directSql.getAllTableNamesForStats(); + protected List getSqlResult() throws MetaException { + return getDirectSql().getAllTableNamesForStats(); } @Override - protected List getJdoResult( - ObjectStore.GetHelper> ctx) throws MetaException { - boolean commited = false; - Query query = null; + protected List getJdoResult() throws MetaException { List result = new ArrayList<>(); - openTransaction(); - try { - String paramStr = "", whereStr = ""; - for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) { - if (i != 0) { - paramStr += ", "; - whereStr += "||"; - } - paramStr += "java.lang.String tt" + i; - whereStr += " tableType == tt" + i; + String paramStr = "", whereStr = ""; + for (int i = 0; i < MetaStoreDirectSql.STATS_TABLE_TYPES.length; ++i) { + if (i != 0) { + paramStr += ", "; + whereStr += "||"; } - query = pm.newQuery(MTable.class, whereStr); - query.declareParameters(paramStr); - Collection tbls = (Collection) query.executeWithArray( - query, MetaStoreDirectSql.STATS_TABLE_TYPES); - pm.retrieveAll(tbls); - for (MTable tbl : tbls) { - result.add(new TableName( - tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName())); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); + paramStr += "java.lang.String tt" + i; + whereStr += " tableType == tt" + i; + } + Query query = pm.newQuery(MTable.class, whereStr); + query.declareParameters(paramStr); + Collection tbls = (Collection) query.executeWithArray( + query, MetaStoreDirectSql.STATS_TABLE_TYPES); + pm.retrieveAll(tbls); + for (MTable tbl : tbls) { + result.add(new TableName( + tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName())); } return result; } @@ -180,7 +186,7 @@ protected List getJdoResult( private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStats) throws MetaException { - Preconditions.checkState(this.currentTransaction.isActive()); + Preconditions.checkState(baseStore.isActiveTransaction()); String colName = mStatsObj.getColName(); @@ -203,7 +209,7 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, String partName = mStatsObj.getPartition().getPartitionName(); String colName = mStatsObj.getColName(); - Preconditions.checkState(this.currentTransaction.isActive()); + Preconditions.checkState(this.baseStore.isActiveTransaction()); LOG.info("Updating partition level column statistics for table=" + TableName.getQualified(catName, dbName, tableName) + @@ -232,192 +238,169 @@ private void writeMPartitionColumnStatistics(Table table, Partition partition, @Override public Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - boolean committed = false; List statsObjs = colStats.getStatsObj(); ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); long start = System.currentTimeMillis(); String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); - try { - openTransaction(); - // DataNucleus objects get detached all over the place for no (real) reason. - // So let's not use them anywhere unless absolutely necessary. - MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); - int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); - long sleepInterval = MetastoreConf.getTimeVar(conf, - MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); - Map result = new RetryingExecutor<>(maxRetries, () -> { - AtomicReference exceptionRef = new AtomicReference<>(); - String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); - executePlainSQL( - sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()), - true, - exception -> { - rollbackTransactionToSavePoint(savePoint); - exceptionRef.set(exception); - }); - if (exceptionRef.get() != null) { - throw new RetryingExecutor.RetryException(exceptionRef.get()); - } - pm.refresh(mTable); - Table table = convertToTable(mTable, conf); - List colNames = new ArrayList<>(); - for (ColumnStatisticsObj statsObj : statsObjs) { - colNames.add(statsObj.getColName()); - } + // DataNucleus objects get detached all over the place for no (real) reason. + // So let's not use them anywhere unless absolutely necessary. + MTable mTable = baseStore.ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName()); + int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); + long sleepInterval = MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + Map result = new RetryingExecutor<>(maxRetries, () -> { + AtomicReference exceptionRef = new AtomicReference<>(); + String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); + setTransactionSavePoint(savePoint); + executePlainSQL( + sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()), + true, + exception -> { + rollbackTransactionToSavePoint(savePoint); + exceptionRef.set(exception); + }); + if (exceptionRef.get() != null) { + throw new RetryingExecutor.RetryException(exceptionRef.get()); + } + pm.refresh(mTable); + Table table = convertToTable(mTable, conf); + List colNames = new ArrayList<>(); + for (ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } - Map oldStats = Maps.newHashMap(); - List stats = getMTableColumnStatistics(table, colNames, colStats.getEngine()); - for (MTableColumnStatistics cStat : stats) { - oldStats.put(cStat.getColName(), cStat); - } + Map oldStats = Maps.newHashMap(); + List stats = getMTableColumnStatistics(table, colNames, colStats.getEngine()); + for (MTableColumnStatistics cStat : stats) { + oldStats.put(cStat.getColName(), cStat); + } - for (ColumnStatisticsObj statsObj : statsObjs) { - MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc, - statsObj, colStats.getEngine()); - writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); - // There is no need to add colname again, otherwise we will get duplicate colNames. - } + for (ColumnStatisticsObj statsObj : statsObjs) { + MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(mTable, statsDesc, + statsObj, colStats.getEngine()); + writeMTableColumnStatistics(table, mStatsObj, oldStats.get(statsObj.getColName())); + // There is no need to add colname again, otherwise we will get duplicate colNames. + } - // Set the table properties - // No need to check again if it exists. - String dbname = table.getDbName(); - String name = table.getTableName(); - MTable oldt = mTable; - Map newParams = new HashMap<>(table.getParameters()); - StatsSetupConst.setColumnStatsState(newParams, colNames); - boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters()); - if (isTxn) { - if (!areTxnStatsSupported) { + // Set the table properties + // No need to check again if it exists. + String dbname = table.getDbName(); + String name = table.getTableName(); + MTable oldt = mTable; + Map newParams = new HashMap<>(table.getParameters()); + StatsSetupConst.setColumnStatsState(newParams, colNames); + boolean isTxn = TxnUtils.isTransactionalTable(oldt.getParameters()); + if (isTxn) { + if (!areTxnStatsSupported) { + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + } else { + String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams, + writeId, validWriteIds, true); + if (errorMsg != null) { + throw new MetaException(errorMsg); + } + if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) { + // Make sure we set the flag to invalid regardless of the current value. StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - } else { - String errorMsg = verifyStatsChangeCtx(TableName.getDbTable(dbname, name), oldt.getParameters(), newParams, - writeId, validWriteIds, true); - if (errorMsg != null) { - throw new MetaException(errorMsg); - } - if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) { - // Make sure we set the flag to invalid regardless of the current value. - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name); - } - oldt.setWriteId(writeId); + LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name); } + oldt.setWriteId(writeId); } - oldt.setParameters(newParams); - return newParams; - }).onRetry(e -> e instanceof RetryingExecutor.RetryException) - .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval -> - ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); - committed = commitTransaction(); - return committed ? result : null; - } finally { - LOG.debug("{} updateTableColumnStatistics took {}ms, success: {}", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), - System.currentTimeMillis() - start, committed); - rollbackAndCleanup(committed, null); - } + } + oldt.setParameters(newParams); + return newParams; + }).onRetry(e -> e instanceof RetryingExecutor.RetryException) + .commandName("updateTableColumnStatistics").sleepInterval(sleepInterval, interval -> + ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); + + LOG.debug("{} updateTableColumnStatistics took {}ms", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), + System.currentTimeMillis() - start); + return result; } @Override public Map updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats, List partVals, String validWriteIds, long writeId) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - boolean committed = false; long start = System.currentTimeMillis(); List statsObjs = colStats.getStatsObj(); ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf); - try { - openTransaction(); - MPartition mPartition = - ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals); - if (mPartition == null) { - throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); - } + MPartition mPartition = + baseStore.ensureGetMPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), partVals); + if (mPartition == null) { + throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist."); + } - List colNames = new ArrayList<>(); - for(ColumnStatisticsObj statsObj : statsObjs) { - colNames.add(statsObj.getColName()); + List colNames = new ArrayList<>(); + for(ColumnStatisticsObj statsObj : statsObjs) { + colNames.add(statsObj.getColName()); + } + int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); + long sleepInterval = MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + Map result = new RetryingExecutor<>(maxRetries, () -> { + AtomicReference exceptionRef = new AtomicReference<>(); + String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); + setTransactionSavePoint(savePoint); + executePlainSQL(sqlGenerator.addForUpdateNoWait( + "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()), + true, + exception -> { + rollbackTransactionToSavePoint(savePoint); + exceptionRef.set(exception); + }); + if (exceptionRef.get() != null) { + throw new RetryingExecutor.RetryException(exceptionRef.get()); + } + pm.refresh(mPartition); + Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), + mPartition, TxnUtils.isAcidTable(table), conf); + Map oldStats = Maps.newHashMap(); + List stats = + getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine()); + for (MPartitionColumnStatistics cStat : stats) { + oldStats.put(cStat.getColName(), cStat); } - int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); - long sleepInterval = MetastoreConf.getTimeVar(conf, - MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); - Map result = new RetryingExecutor<>(maxRetries, () -> { - AtomicReference exceptionRef = new AtomicReference<>(); - String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); - executePlainSQL(sqlGenerator.addForUpdateNoWait( - "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()), - true, - exception -> { - rollbackTransactionToSavePoint(savePoint); - exceptionRef.set(exception); - }); - if (exceptionRef.get() != null) { - throw new RetryingExecutor.RetryException(exceptionRef.get()); - } - pm.refresh(mPartition); - Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), - mPartition, TxnUtils.isAcidTable(table), conf); - Map oldStats = Maps.newHashMap(); - List stats = - getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine()); - for (MPartitionColumnStatistics cStat : stats) { - oldStats.put(cStat.getColName(), cStat); - } - for (ColumnStatisticsObj statsObj : statsObjs) { - MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, - statsDesc, statsObj, colStats.getEngine()); - writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName())); - } + for (ColumnStatisticsObj statsObj : statsObjs) { + MPartitionColumnStatistics mStatsObj = StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, + statsDesc, statsObj, colStats.getEngine()); + writeMPartitionColumnStatistics(table, partition, mStatsObj, oldStats.get(statsObj.getColName())); + } - Map newParams = new HashMap<>(mPartition.getParameters()); - StatsSetupConst.setColumnStatsState(newParams, colNames); - boolean isTxn = TxnUtils.isTransactionalTable(table); - if (isTxn) { - if (!areTxnStatsSupported) { + Map newParams = new HashMap<>(mPartition.getParameters()); + StatsSetupConst.setColumnStatsState(newParams, colNames); + boolean isTxn = TxnUtils.isTransactionalTable(table); + if (isTxn) { + if (!areTxnStatsSupported) { + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + } else { + String errorMsg = verifyStatsChangeCtx( + TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(), + newParams, writeId, validWriteIds, true); + if (errorMsg != null) { + throw new MetaException(errorMsg); + } + if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) { + // Make sure we set the flag to invalid regardless of the current value. StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - } else { - String errorMsg = verifyStatsChangeCtx( - TableName.getDbTable(statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getParameters(), - newParams, writeId, validWriteIds, true); - if (errorMsg != null) { - throw new MetaException(errorMsg); - } - if (!isCurrentStatsValidForTheQuery(mPartition.getParameters(), mPartition.getWriteId(), validWriteIds, true)) { - // Make sure we set the flag to invalid regardless of the current value. - StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); - LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName()); - } - mPartition.setWriteId(writeId); + LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the partition: {}, {} ", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), statsDesc.getPartName()); } + mPartition.setWriteId(writeId); } - mPartition.setParameters(newParams); - return newParams; - }).onRetry(e -> e instanceof RetryingExecutor.RetryException) - .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval -> - ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); - committed = commitTransaction(); - return committed ? result : null; - } finally { - LOG.debug("{} updatePartitionColumnStatistics took {}ms, success: {}", - new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), - System.currentTimeMillis() - start, committed); - rollbackAndCleanup(committed, null); - } - } - - @Override - public Map updatePartitionColumnStatistics(ColumnStatistics colStats, - List partVals, String validWriteIds, long writeId) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - Table table = getTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); - MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); - return updatePartitionColumnStatistics(table, mTable, colStats, partVals, validWriteIds, writeId); + } + mPartition.setParameters(newParams); + return newParams; + }).onRetry(e -> e instanceof RetryingExecutor.RetryException) + .commandName("updatePartitionColumnStatistics").sleepInterval(sleepInterval, interval -> + ThreadLocalRandom.current().nextLong(sleepInterval) + 30).run(); + LOG.debug("{} updatePartitionColumnStatistics took {}ms", + new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), + System.currentTimeMillis() - start); + return result; } @Override @@ -428,22 +411,25 @@ public Map> updatePartitionColumnStatisticsInBatch( String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - return new ObjectStore.GetHelper>>(tbl.getCatName(), - tbl.getDbName(), tbl.getTableName(), true, false) { + return new GetHelper>>(this, null) { @Override protected String describeResult() { return "Map of partition key to column stats if successful"; } @Override - protected Map> getSqlResult( - ObjectStore.GetHelper>> ctx) + protected Map> getSqlResult() throws MetaException { - return directSql.updatePartitionColumnStatisticsBatch(partColStatsMap, tbl, + return getDirectSql().updatePartitionColumnStatisticsBatch(partColStatsMap, tbl, listeners, validWriteIds, writeId); } + + @Override + protected boolean canUseJdoQuery() throws MetaException { + return false; + } + @Override - protected Map> getJdoResult( - ObjectStore.GetHelper>> ctx) + protected Map> getJdoResult() throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { throw new UnsupportedOperationException("Cannot update partition column statistics with JDO, make sure direct SQL is enabled"); } @@ -453,103 +439,77 @@ protected Map> getJdoResult( private List getMTableColumnStatistics(Table table, List colNames, String engine) throws MetaException { - Preconditions.checkState(this.currentTransaction.isActive()); + Preconditions.checkState(baseStore.isActiveTransaction()); if (colNames.isEmpty()) { return Collections.emptyList(); } - boolean committed = false; - try { - openTransaction(); - - validateTableCols(table, colNames); - - List result = Collections.emptyList(); - try (Query query = pm.newQuery(MTableColumnStatistics.class)) { - result = - Batchable.runBatched(batchSize, colNames, new Batchable() { - @Override - public List run(List input) - throws MetaException { - StringBuilder filter = - new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && ("); - StringBuilder paramStr = new StringBuilder( - "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); - Object[] params = new Object[input.size() + 4]; - params[0] = table.getTableName(); - params[1] = table.getDbName(); - params[2] = table.getCatName(); - params[3] = engine; - for (int i = 0; i < input.size(); ++i) { - filter.append((i == 0) ? "" : " || ").append("colName == c").append(i); - paramStr.append(", java.lang.String c").append(i); - params[i + 4] = input.get(i); - } - filter.append(")"); - query.setFilter(filter.toString()); - query.declareParameters(paramStr.toString()); - List paritial = (List) query.executeWithArray(params); - pm.retrieveAll(paritial); - return paritial; - } - }); + validateTableCols(table, colNames); + + List result = Collections.emptyList(); + Query query = pm.newQuery(MTableColumnStatistics.class); + result = + Batchable.runBatched(batchSize, colNames, new Batchable() { + @Override + public List run(List input) + throws MetaException { + StringBuilder filter = + new StringBuilder("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && engine == t4 && ("); + StringBuilder paramStr = new StringBuilder( + "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"); + Object[] params = new Object[input.size() + 4]; + params[0] = table.getTableName(); + params[1] = table.getDbName(); + params[2] = table.getCatName(); + params[3] = engine; + for (int i = 0; i < input.size(); ++i) { + filter.append((i == 0) ? "" : " || ").append("colName == c").append(i); + paramStr.append(", java.lang.String c").append(i); + params[i + 4] = input.get(i); + } + filter.append(")"); + query.setFilter(filter.toString()); + query.declareParameters(paramStr.toString()); + List paritial = (List) query.executeWithArray(params); + pm.retrieveAll(paritial); + return paritial; + } + }); - if (result.size() > colNames.size()) { - throw new MetaException("Unexpected " + result.size() + " statistics for " - + colNames.size() + " columns"); - } - result = new ArrayList<>(result); - } - committed = commitTransaction(); - return result; - } catch (Exception ex) { - LOG.error("Error retrieving statistics via jdo", ex); - if (ex instanceof MetaException) { - throw (MetaException) ex; - } - throw new MetaException(ex.getMessage()); - } finally { - rollbackAndCleanup(committed, null); + if (result.size() > colNames.size()) { + throw new MetaException("Unexpected " + result.size() + " statistics for " + + colNames.size() + " columns"); } + return new ArrayList<>(result); } @Override public List getTableColumnStatistics( - String catName, - String dbName, - String tableName, + TableName tblName, List colNames) throws MetaException, NoSuchObjectException { + String catName = normalizeIdentifier(tblName.getCat()); + String dbName = normalizeIdentifier(tblName.getDb()); + String tableName = normalizeIdentifier(tblName.getTable()); // Note: this will get stats without verifying ACID. - boolean committed = false; - Query query = null; List result = new ArrayList<>(); - - try { - openTransaction(); - query = pm.newQuery(MTableColumnStatistics.class); - query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - query.setResult("DISTINCT engine"); - Collection names = (Collection) query.execute(tableName, dbName, catName); - List engines = new ArrayList<>(); - for (Iterator i = names.iterator(); i.hasNext();) { - engines.add((String) i.next()); - } - for (String e : engines) { - ColumnStatistics cs = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, e, true, true); - if (cs != null) { - result.add(cs); - } + Query query = pm.newQuery(MTableColumnStatistics.class); + query.setFilter("table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + query.setResult("DISTINCT engine"); + Collection names = (Collection) query.execute(tableName, dbName, catName); + List engines = new ArrayList<>(); + for (Iterator i = names.iterator(); i.hasNext();) { + engines.add((String) i.next()); + } + for (String e : engines) { + ColumnStatistics cs = getTableColumnStatisticsInternal( + new TableName(catName, dbName, tableName), colNames, e); + if (cs != null) { + result.add(cs); } - committed = commitTransaction(); - return result; - } finally { - LOG.debug("Done executing getTableColumnStatistics with status : {}", - committed); - rollbackAndCleanup(committed, query); } + return result; } @Override @@ -561,30 +521,28 @@ public ColumnStatistics getTableColumnStatistics( String engine) throws MetaException, NoSuchObjectException { // Note: this will get stats without verifying ACID. return getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, true, true); + new TableName(catName, dbName, tableName), colNames, engine); } @Override public ColumnStatistics getTableColumnStatistics( - String catName, - String dbName, - String tableName, + TableName tableName, List colNames, String engine, String writeIdList) throws MetaException, NoSuchObjectException { // If the current stats in the metastore doesn't comply with // the isolation level of the query, set No to the compliance flag. Boolean isCompliant = null; + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); if (writeIdList != null) { - MTable table = this.getMTable(catName, dbName, tableName); - if (table == null) { - throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tableName) + " table not found"); - } + MTable table = baseStore.ensureGetMTable(catName, dbName, tblName); isCompliant = !TxnUtils.isTransactionalTable(table.getParameters()) || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table, writeIdList, false)); } ColumnStatistics stats = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, true, true); + new TableName(catName, dbName, tblName), colNames, engine); if (stats != null && isCompliant != null) { stats.setIsStatsCompliant(isCompliant); } @@ -630,50 +588,41 @@ protected ColumnStatistics getJdoResult(ObjectStore.GetHelper } @Override - public List> getPartitionColumnStatistics(String catName, String dbName, String tableName, + public List> getPartitionColumnStatistics(TableName tblName, List partNames, List colNames) throws MetaException, NoSuchObjectException { + String catName = normalizeIdentifier(tblName.getCat()); + String dbName = normalizeIdentifier(tblName.getDb()); + String tableName = normalizeIdentifier(tblName.getTable()); // Note: this will get stats without verifying ACID. - boolean committed = false; - Query query = null; List> result = new ArrayList<>(); - - try { - openTransaction(); - query = pm.newQuery(MPartitionColumnStatistics.class); - query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - query.setResult("DISTINCT engine"); - Collection names = (Collection) query.execute(tableName, dbName, catName); - List engines = new ArrayList<>(); - for (Iterator i = names.iterator(); i.hasNext();) { - engines.add((String) i.next()); - } - for (String e : engines) { - List cs = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, e, true, true); - if (cs != null) { - result.add(cs); - } + Query query = pm.newQuery(MPartitionColumnStatistics.class); + query.setFilter("partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); + query.setResult("DISTINCT engine"); + Collection names = (Collection) query.execute(tableName, dbName, catName); + List engines = new ArrayList<>(); + for (Iterator i = names.iterator(); i.hasNext();) { + engines.add((String) i.next()); + } + for (String e : engines) { + List cs = getPartitionColumnStatisticsInternal( + new TableName(catName, dbName, tableName), partNames, colNames, e); + if (cs != null) { + result.add(cs); } - committed = commitTransaction(); - return result; - } finally { - LOG.debug("Done executing getTableColumnStatistics with status : {}", - committed); - rollbackAndCleanup(committed, query); } + return result; } @Override - public List getPartitionColumnStatistics(String catName, String dbName, String tableName, + public List getPartitionColumnStatistics(TableName tableName, List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException { // Note: this will get stats without verifying ACID. if (CollectionUtils.isEmpty(partNames) || CollectionUtils.isEmpty(colNames)) { LOG.debug("PartNames and/or ColNames are empty"); return Collections.emptyList(); } - return getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, true, true); + return getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine); } @Override @@ -1028,81 +977,75 @@ private boolean deletePartitionColumnStatisticsViaJdo(String catName, String dbN String database = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); String catalog = normalizeIdentifier(catName); - try { - openTransaction(); - Batchable b = new Batchable() { - @Override - public List run(List input) throws Exception { - Query query = pm.newQuery(MPartitionColumnStatistics.class); - addQueryAfterUse(query); - String filter; - String parameters; - if (colNames != null && !colNames.isEmpty()) { - filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " - + "t4.contains(colName) && partition.table.database.catalogName == t5" + (engine != null ? " && engine == t6" : ""); - parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, " - + "java.util.Collection t4, java.lang.String t5" + (engine != null ? ", java.lang.String t6" : ""); - } else { - filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + - "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : ""); - parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + (engine != null ? ", java.lang.String t5" : ""); - } - query.setFilter(filter); - query.declareParameters(parameters); - List params = new ArrayList<>(); - params.add(input); - params.add(normalizeIdentifier(database)); - params.add(normalizeIdentifier(tableName)); - if (colNames != null && !colNames.isEmpty()) { - params.add(colNames); - } - params.add(catalog); - if (engine != null) { - params.add(engine); - } - List mStatsObjColl = - (List) query.executeWithArray(params.toArray()); - pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { - pm.deletePersistentAll(mStatsObjColl); - } - return null; + Batchable b = new Batchable() { + @Override + public List run(List input) throws Exception { + Query query = pm.newQuery(MPartitionColumnStatistics.class); + addQueryAfterUse(query); + String filter; + String parameters; + if (colNames != null && !colNames.isEmpty()) { + filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + + "t4.contains(colName) && partition.table.database.catalogName == t5" + (engine != null ? " && engine == t6" : ""); + parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, " + + "java.util.Collection t4, java.lang.String t5" + (engine != null ? ", java.lang.String t6" : ""); + } else { + filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + + "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : ""); + parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + (engine != null ? ", java.lang.String t5" : ""); } - }; - try { - Batchable.runBatched(batchSize, partNames, b); - } finally { - b.closeAllQueries(); + query.setFilter(filter); + query.declareParameters(parameters); + List params = new ArrayList<>(); + params.add(input); + params.add(normalizeIdentifier(database)); + params.add(normalizeIdentifier(tableName)); + if (colNames != null && !colNames.isEmpty()) { + params.add(colNames); + } + params.add(catalog); + if (engine != null) { + params.add(engine); + } + List mStatsObjColl = + (List) query.executeWithArray(params.toArray()); + pm.retrieveAll(mStatsObjColl); + if (mStatsObjColl != null) { + pm.deletePersistentAll(mStatsObjColl); + } + return null; } + }; + try { + Batchable.runBatched(batchSize, partNames, b); + } finally { + b.closeAllQueries(); + } - Batchable.runBatched(batchSize, partNames, new Batchable() { - @Override - public List run(List input) throws MetaException { - Pair> queryWithParams = getPartQueryWithParams(pm, catalog, database, tableName, - input); - try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) { - qw.setResultClass(MPartition.class); - qw.setClass(MPartition.class); - List mparts = (List) qw.executeWithMap(queryWithParams.getRight()); - for (MPartition mPart : mparts) { - Map params = mPart.getParameters(); - if (params != null && params.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { - if (colNames == null || colNames.isEmpty()) { - StatsSetupConst.clearColumnStatsState(params); - } else { - StatsSetupConst.removeColumnStatsState(params, colNames); - } - mPart.setParameters(params); + Batchable.runBatched(batchSize, partNames, new Batchable() { + @Override + public List run(List input) throws MetaException { + Pair> queryWithParams = getPartQueryWithParams(pm, catalog, database, tableName, + input); + try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) { + qw.setResultClass(MPartition.class); + qw.setClass(MPartition.class); + List mparts = (List) qw.executeWithMap(queryWithParams.getRight()); + for (MPartition mPart : mparts) { + Map params = mPart.getParameters(); + if (params != null && params.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { + if (colNames == null || colNames.isEmpty()) { + StatsSetupConst.clearColumnStatsState(params); + } else { + StatsSetupConst.removeColumnStatsState(params, colNames); } + mPart.setParameters(params); } } - return Collections.emptyList(); } - }); - ret = commitTransaction(); - } finally { - rollbackAndCleanup(ret, null); - } + return Collections.emptyList(); + } + }); return ret; } From ba964f8a7993bf5360bad8e3b16370a272f1d072 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 08:08:48 +0800 Subject: [PATCH 07/22] f --- .../metastore/impl/ColStatsStoreImpl.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 649af3f61a72..255e33961662 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -43,14 +43,18 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.Batchable; +import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.Deadline; import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.PersistenceManagerProvider; import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.StatObjectConverter; import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; import org.apache.hadoop.hive.metastore.Warehouse; @@ -87,6 +91,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING; import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; @@ -95,6 +100,22 @@ public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class); + private DatabaseProduct dbType; + protected int batchSize = NO_BATCHING; + private boolean areTxnStatsSupported = false; + private Configuration conf; + + @Override + public void setBaseStore(RawStore store) { + super.setBaseStore(store); + this.dbType = PersistenceManagerProvider.getDatabaseProduct(); + this.batchSize = MetastoreConf.getIntVar(store.getConf(), + MetastoreConf.ConfVars.RAWSTORE_PARTITION_BATCH_SIZE); + this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(), + MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED); + this.conf = store.getConf(); + } + @Override public List getTableNamesWithStats() throws MetaException, NoSuchObjectException { return new GetListHelper (this, null) { @@ -552,8 +573,8 @@ public ColumnStatistics getTableColumnStatistics( protected ColumnStatistics getTableColumnStatisticsInternal( String catName, String dbName, String tableName, final List colNames, String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName), normalizeIdentifier(tableName), allowSql, allowJdo, null) { @Override @@ -667,8 +688,8 @@ public List getPartitionColumnStatistics( protected List getPartitionColumnStatisticsInternal( String catName, String dbName, String tableName, final List partNames, final List colNames, String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetListHelper(catName, dbName, tableName, allowSql, allowJdo) { @Override protected List getSqlResult( @@ -754,11 +775,11 @@ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblNam public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, final List partNames, final List colNames, String engine) throws MetaException, NoSuchObjectException { - final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(getConf(), + final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_NDV_DENSITY_FUNCTION); - final double ndvTuner = MetastoreConf.getDoubleVar(getConf(), MetastoreConf.ConfVars.STATS_NDV_TUNER); - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + final double ndvTuner = MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER); + final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetHelper(catName, dbName, tblName, true, false) { @Override protected AggrStats getSqlResult(ObjectStore.GetHelper ctx) @@ -784,8 +805,8 @@ protected String describeResult() { @Override public List getPartitionColStatsForDatabase(String catName, String dbName) throws MetaException, NoSuchObjectException { - final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); - final boolean enableKll = MetastoreConf.getBoolVar(getConf(), MetastoreConf.ConfVars.STATS_FETCH_KLL); + final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); + final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetHelper>( catName, dbName, null, true, false) { @Override From 5f10e14a12f2f515b6065f3ce8ddf978fd841687 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 08:20:57 +0800 Subject: [PATCH 08/22] fix --- .../metastore/impl/ColStatsStoreImpl.java | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 255e33961662..94550c0ce1fb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -535,14 +535,11 @@ public List getTableColumnStatistics( @Override public ColumnStatistics getTableColumnStatistics( - String catName, - String dbName, - String tableName, + TableName tableName, List colNames, String engine) throws MetaException, NoSuchObjectException { // Note: this will get stats without verifying ACID. - return getTableColumnStatisticsInternal( - new TableName(catName, dbName, tableName), colNames, engine); + return getTableColumnStatisticsInternal(tableName, colNames, engine); } @Override @@ -571,8 +568,7 @@ public ColumnStatistics getTableColumnStatistics( } protected ColumnStatistics getTableColumnStatisticsInternal( - String catName, String dbName, String tableName, final List colNames, String engine, - boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + TableName tableName, final List colNames, String engine) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName), @@ -648,7 +644,7 @@ public List getPartitionColumnStatistics(TableName tableName, @Override public List getPartitionColumnStatistics( - String catName, String dbName, String tableName, + TableName tableName, List partNames, List colNames, String engine, String writeIdList) throws MetaException, NoSuchObjectException { @@ -686,8 +682,8 @@ public List getPartitionColumnStatistics( } protected List getPartitionColumnStatisticsInternal( - String catName, String dbName, String tableName, final List partNames, final List colNames, - String engine, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + TableName tableName, final List partNames, final List colNames, + String engine) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); return new ObjectStore.GetListHelper(catName, dbName, tableName, allowSql, allowJdo) { @@ -732,7 +728,7 @@ protected List getJdoResult(ObjectStore.GetHelper partNames, final List colNames, String engine, String writeIdList) throws MetaException, NoSuchObjectException { // If the current stats in the metastore doesn't comply with @@ -772,7 +768,7 @@ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblNam } @Override - public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, + public AggrStats get_aggr_stats_for(TableName tableName, final List partNames, final List colNames, String engine) throws MetaException, NoSuchObjectException { final boolean useDensityFunctionForNDVEstimation = MetastoreConf.getBoolVar(conf, @@ -964,7 +960,7 @@ protected Integer getJdoResult(ObjectStore.GetHelper ctx) throws MetaEx } @Override - public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, + public boolean deletePartitionColumnStatistics(TableName tableName, List partNames, List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { if (partNames == null || partNames.isEmpty()) { @@ -1071,7 +1067,7 @@ public List run(List input) throws MetaException { } @Override - public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, + public boolean deleteTableColumnStatistics(TableName tableName, List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); @@ -1252,10 +1248,9 @@ public static boolean isCurrentStatsValidForTheQuery( return false; } - private abstract class GetStatHelper extends ObjectStore.GetHelper { - public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql, - boolean allowJdo, String writeIdList) throws MetaException { - super(catalogName, dbName, tblName, allowSql, allowJdo); + private abstract class GetStatHelper extends GetHelper { + public GetStatHelper(TableName tableName, RawStoreAware baseStore) throws MetaException { + super(baseStore, tableName); } @Override From 195dd77fc38f559a3ce9911f5fa86234cd918614 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 08:44:46 +0800 Subject: [PATCH 09/22] fix --- .../hadoop/hive/metastore/ObjectStore.java | 4 - .../metastore/impl/ColStatsStoreImpl.java | 377 +++++++++--------- 2 files changed, 180 insertions(+), 201 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 94dab79b539d..f9523e4c4a58 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -129,7 +129,6 @@ import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats; import org.apache.hadoop.hive.metastore.metastore.iface.PrivilegeStore; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.metrics.Metrics; @@ -234,7 +233,6 @@ private enum TXN_STATUS { protected PersistenceManager pm = null; protected SQLGenerator sqlGenerator = null; private MetaStoreDirectSql directSql = null; - private DirectSqlAggrStats directSqlAggrStats; protected DatabaseProduct dbType = null; protected Configuration conf; private volatile int openTrasactionCalls = 0; @@ -273,7 +271,6 @@ public void setConf(Configuration conf) { // most recent instance of the pmf pm = null; directSql = null; - directSqlAggrStats = null; openTrasactionCalls = 0; currentTransaction = null; transactionStatus = TXN_STATUS.NO_STATE; @@ -314,7 +311,6 @@ private void initialize() { String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema"); schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null); directSql = new MetaStoreDirectSql(pm, conf, schema); - directSqlAggrStats = new DirectSqlAggrStats(pm,conf,schema); } } if (propertyStore == null) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 94550c0ce1fb..d0203bfff6e8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.directsql.DirectSqlAggrStats; import org.apache.hadoop.hive.metastore.directsql.DirectSqlDeleteStats; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.metastore.GetHelper; @@ -85,6 +86,7 @@ import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; +import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.RetryingExecutor; @@ -104,7 +106,9 @@ public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { protected int batchSize = NO_BATCHING; private boolean areTxnStatsSupported = false; private Configuration conf; - + private DirectSqlAggrStats directSqlAggrStats; + private SQLGenerator sqlGenerator; + @Override public void setBaseStore(RawStore store) { super.setBaseStore(store); @@ -114,6 +118,10 @@ public void setBaseStore(RawStore store) { this.areTxnStatsSupported = MetastoreConf.getBoolVar(baseStore.getConf(), MetastoreConf.ConfVars.HIVE_TXN_STATS_ENABLED); this.conf = store.getConf(); + this.sqlGenerator = new SQLGenerator(dbType, conf); + String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema"); + schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null); + this.directSqlAggrStats = new DirectSqlAggrStats(pm, conf, schema); } @Override @@ -571,15 +579,17 @@ protected ColumnStatistics getTableColumnStatisticsInternal( TableName tableName, final List colNames, String engine) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - return new ObjectStore.GetStatHelper(normalizeIdentifier(catName), normalizeIdentifier(dbName), - normalizeIdentifier(tableName), allowSql, allowJdo, null) { + return new GetStatHelper(this, tableName) { @Override - protected ColumnStatistics getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + protected ColumnStatistics getSqlResult() throws MetaException { + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); return directSqlAggrStats.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll); } @Override - protected ColumnStatistics getJdoResult(ObjectStore.GetHelper ctx) throws MetaException { + protected ColumnStatistics getJdoResult() throws MetaException { List mStats = getMTableColumnStatistics(getTable(), colNames, engine); if (mStats.isEmpty()) { @@ -652,8 +662,7 @@ public List getPartitionColumnStatistics( LOG.debug("PartNames and/or ColNames are empty"); return Collections.emptyList(); } - List allStats = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, true, true); + List allStats = getPartitionColumnStatisticsInternal(tableName, partNames, colNames, engine); if (writeIdList != null) { if (!areTxnStatsSupported) { for (ColumnStatistics cs : allStats) { @@ -662,13 +671,13 @@ public List getPartitionColumnStatistics( } else { // TODO: this could be improved to get partitions in bulk for (ColumnStatistics cs : allStats) { - MPartition mpart = ensureGetMPartition(new TableName(catName, dbName, tableName), + MPartition mpart = baseStore.ensureGetMPartition(tableName, Warehouse.getPartValuesFromPartName(cs.getStatsDesc().getPartName())); if (mpart == null || !isCurrentStatsValidForTheQuery(mpart.getParameters(), mpart.getWriteId(), writeIdList, false)) { if (mpart != null) { - LOG.debug("The current metastore transactional partition column statistics for {}.{}.{} " - + "(write ID {}) are not valid for current query ({} {})", dbName, tableName, + LOG.debug("The current metastore transactional partition column statistics for {} {} " + + "(write ID {}) are not valid for current query ({})", tableName, mpart.getPartitionName(), mpart.getWriteId(), writeIdList); } cs.setIsStatsCompliant(false); @@ -686,15 +695,17 @@ protected List getPartitionColumnStatisticsInternal( String engine) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - return new ObjectStore.GetListHelper(catName, dbName, tableName, allowSql, allowJdo) { + return new GetListHelper(this, tableName) { @Override - protected List getSqlResult( - ObjectStore.GetHelper> ctx) throws MetaException { + protected List getSqlResult() throws MetaException { + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); return directSqlAggrStats.getPartitionStats( catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll); } @Override - protected List getJdoResult(ObjectStore.GetHelper> ctx) + protected List getJdoResult() throws MetaException, NoSuchObjectException { List mStats = getMPartitionColumnStatistics(getTable(), partNames, colNames, engine); @@ -738,7 +749,7 @@ public AggrStats get_aggr_stats_for(TableName tableName, return null; } - Table table = getTable(catName, dbName, tblName); + Table table = baseStore.unwrap(TableStore.class).getTable(tableName, null, -1); boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); if (isTxn && !areTxnStatsSupported) { return null; @@ -758,13 +769,13 @@ public AggrStats get_aggr_stats_for(TableName tableName, if (!isCurrentStatsValidForTheQuery(part.getParameters(), part.getWriteId(), writeIdList, false)) { String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); LOG.debug("The current metastore transactional partition column " - + "statistics for {}.{}.{} is not valid for the current query", - dbName, tblName, partName); + + "statistics for {} {} is not valid for the current query", + tableName, partName); return null; } } } - return get_aggr_stats_for(catName, dbName, tblName, partNames, colNames, engine); + return get_aggr_stats_for(tableName, partNames, colNames, engine); } @Override @@ -776,15 +787,24 @@ public AggrStats get_aggr_stats_for(TableName tableName, final double ndvTuner = MetastoreConf.getDoubleVar(conf, MetastoreConf.ConfVars.STATS_NDV_TUNER); final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - return new ObjectStore.GetHelper(catName, dbName, tblName, true, false) { + return new GetHelper(this, tableName) { @Override - protected AggrStats getSqlResult(ObjectStore.GetHelper ctx) + protected AggrStats getSqlResult() throws MetaException { - return directSql.aggrColStatsForPartitions(catName, dbName, tblName, partNames, + String catName = normalizeIdentifier(tableName.getCat()); + String dbName = normalizeIdentifier(tableName.getDb()); + String tblName = normalizeIdentifier(tableName.getTable()); + return getDirectSql().aggrColStatsForPartitions(catName, dbName, tblName, partNames, colNames, engine, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector, enableKll); } + @Override - protected AggrStats getJdoResult(ObjectStore.GetHelper ctx) + protected boolean canUseJdoQuery() throws MetaException { + return false; + } + + @Override + protected AggrStats getJdoResult() throws MetaException, NoSuchObjectException { // This is fast path for query optimizations, if we can find this info // quickly using @@ -803,17 +823,19 @@ public List getPartitionColStats throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - return new ObjectStore.GetHelper>( - catName, dbName, null, true, false) { + return new GetHelper>(this, null) { @Override - protected List getSqlResult( - ObjectStore.GetHelper> ctx) throws MetaException { + protected List getSqlResult() throws MetaException { return directSqlAggrStats.getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll); } @Override - protected List getJdoResult( - ObjectStore.GetHelper> ctx) + protected boolean canUseJdoQuery() throws MetaException { + return false; + } + + @Override + protected List getJdoResult() throws MetaException, NoSuchObjectException { // This is fast path for query optimizations, if we can find this info // quickly using directSql, do it. No point in failing back to slow path @@ -825,162 +847,129 @@ protected List getJdoResult( protected String describeResult() { return null; } - }.run(true); + }.run(false); } private List getMPartitionColumnStatistics(Table table, List partNames, List colNames, String engine) throws MetaException { - boolean committed = false; - + // We are not going to verify SD for each partition. Just verify for the + // table. TODO: we need verify the partition column instead try { - openTransaction(); - // We are not going to verify SD for each partition. Just verify for the - // table. TODO: we need verify the partition column instead - try { - validateTableCols(table, colNames); - } catch (MetaException me) { - LOG.warn("The table does not have the same column definition as its partition."); - } - List result = Collections.emptyList(); - try (Query query = pm.newQuery(MPartitionColumnStatistics.class)) { - String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"; - String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && partition.table.database.catalogName == t3 && engine == t4 && ("; - Object[] params = new Object[colNames.size() + partNames.size() + 4]; - int i = 0; - params[i++] = table.getTableName(); - params[i++] = table.getDbName(); - params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf); - params[i++] = engine; - int firstI = i; - for (String s : partNames) { - filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i; - paramStr += ", java.lang.String p" + i; - params[i++] = s; - } - filter += ") && ("; - firstI = i; - for (String s : colNames) { - filter += ((i == firstI) ? "" : " || ") + "colName == c" + i; - paramStr += ", java.lang.String c" + i; - params[i++] = s; - } - filter += ")"; - query.setFilter(filter); - query.declareParameters(paramStr); - query.setOrdering("partition.partitionName ascending"); - result = (List) query.executeWithArray(params); - pm.retrieveAll(result); - result = new ArrayList<>(result); - } catch (Exception ex) { - LOG.error("Error retrieving statistics via jdo", ex); - throw new MetaException(ex.getMessage()); - } - committed = commitTransaction(); - return result; - } finally { - if (!committed) { - rollbackTransaction(); - return Collections.emptyList(); - } + validateTableCols(table, colNames); + } catch (MetaException me) { + LOG.warn("The table does not have the same column definition as its partition."); } + List result = Collections.emptyList(); + Query query = pm.newQuery(MPartitionColumnStatistics.class); + String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.lang.String t4"; + String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && " + + "partition.table.database.catalogName == t3 && engine == t4 && ("; + Object[] params = new Object[colNames.size() + partNames.size() + 4]; + int i = 0; + params[i++] = table.getTableName(); + params[i++] = table.getDbName(); + params[i++] = table.isSetCatName() ? table.getCatName() : getDefaultCatalog(conf); + params[i++] = engine; + int firstI = i; + for (String s : partNames) { + filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i; + paramStr += ", java.lang.String p" + i; + params[i++] = s; + } + filter += ") && ("; + firstI = i; + for (String s : colNames) { + filter += ((i == firstI) ? "" : " || ") + "colName == c" + i; + paramStr += ", java.lang.String c" + i; + params[i++] = s; + } + filter += ")"; + query.setFilter(filter); + query.declareParameters(paramStr); + query.setOrdering("partition.partitionName ascending"); + result = (List) query.executeWithArray(params); + pm.retrieveAll(result); + return new ArrayList<>(result); } @Override - public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) { - + public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) + throws MetaException, NoSuchObjectException { String catName = tn.getCat(); String dbName = tn.getDb(); String tableName = tn.getTable(); - Query query = null; dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); catName = normalizeIdentifier(catName); if (tableName == null) { throw new RuntimeException("Table name is null."); } - boolean ret = false; - try { - openTransaction(); - MTable mTable = getMTable(catName, dbName, tableName); - - query = pm.newQuery(MPartitionColumnStatistics.class); - - String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4"; - String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4"; - - query.setFilter(filter); - query.declareParameters(parameters); + MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName); + Query query = pm.newQuery(MPartitionColumnStatistics.class); + String filter = "partition.table.database.name == t2 && partition.table.tableName == t3 && partition.table.database.catalogName == t4"; + String parameters = "java.lang.String t2, java.lang.String t3, java.lang.String t4"; - Long number = query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), - normalizeIdentifier(catName)); + query.setFilter(filter); + query.declareParameters(parameters); + Long number = query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), + normalizeIdentifier(catName)); - new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { - private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown(); + new GetHelper(this, tn) { + private final MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown(); - @Override - protected String describeResult() { - return "Partition count"; - } - - @Override - protected boolean canUseDirectSql(ObjectStore.GetHelper ctx) throws MetaException { - return true; - } + @Override + protected String describeResult() { + return "Partition count"; + } - @Override - protected Integer getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { - directSql.deleteColumnStatsState(getTable().getId()); - return 0; - } + @Override + protected Integer getSqlResult() throws MetaException { + getDirectSql().deleteColumnStatsState(getTable().getId()); + return 0; + } - @Override - protected Integer getJdoResult(ObjectStore.GetHelper ctx) throws MetaException, NoSuchObjectException { - try { - List parts = getPartitions(catName, dbName, tableName, - GetPartitionsArgs.getAllPartitions()); - for (Partition part : parts) { - Partition newPart = new Partition(part); - StatsSetupConst.clearColumnStatsState(newPart.getParameters()); - alterPartition(catName, dbName, tableName, part.getValues(), newPart, writeIdList); - } - return parts.size(); - } catch (InvalidObjectException e) { - LOG.error("error updating parts", e); - return -1; + @Override + protected Integer getJdoResult() throws MetaException, NoSuchObjectException { + try { + List parts = baseStore.unwrap(TableStore.class).getPartitions(tn, + GetPartitionsArgs.getAllPartitions()); + for (Partition part : parts) { + Partition newPart = new Partition(part); + StatsSetupConst.clearColumnStatsState(newPart.getParameters()); + baseStore.unwrap(TableStore.class).alterPartition(tn, part.getValues(), newPart, writeIdList); } + return parts.size(); + } catch (InvalidObjectException e) { + LOG.error("error updating parts", e); + return -1; } - }.run(true); - - ret = commitTransaction(); - } catch (Exception e) { - LOG.error("Couldn't clear stats for table", e); - } finally { - rollbackAndCleanup(ret, query); - } + } + }.run(true); } @Override - public boolean deletePartitionColumnStatistics(TableName tableName, + public boolean deletePartitionColumnStatistics(TableName tblName, List partNames, List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { if (partNames == null || partNames.isEmpty()) { throw new InvalidInputException("No partition specified for dropping the statistics"); } - dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); - catName = normalizeIdentifier(catName); + String catName = normalizeIdentifier(tblName.getCat()); + String dbName = normalizeIdentifier(tblName.getDb()); + String tableName = normalizeIdentifier(tblName.getTable()); List cols = normalizeIdentifiers(colNames); - return new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { + return new GetHelper(this, null) { @Override protected String describeResult() { return "delete partition column stats"; } @Override - protected Boolean getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { - DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); + protected Boolean getSqlResult() throws MetaException { + DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(getDirectSql(), pm); return deleteStats.deletePartitionColumnStats(catName, dbName, tableName, partNames, cols, engine); } @Override - protected Boolean getJdoResult(ObjectStore.GetHelper ctx) + protected Boolean getJdoResult() throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { return deletePartitionColumnStatisticsViaJdo(catName, dbName, tableName, partNames, cols, engine); } @@ -1070,84 +1059,78 @@ public List run(List input) throws MetaException { public boolean deleteTableColumnStatistics(TableName tableName, List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - dbName = org.apache.commons.lang3.StringUtils.defaultString(dbName, Warehouse.DEFAULT_DATABASE_NAME); if (tableName == null) { throw new InvalidInputException("Table name is null."); } List cols = normalizeIdentifiers(colNames); - return new ObjectStore.GetHelper(catName, dbName, tableName, true, true) { + return new GetHelper(this, tableName) { @Override protected String describeResult() { return "delete table column stats"; } @Override - protected Boolean getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { - DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(directSql, pm); + protected Boolean getSqlResult() throws MetaException { + DirectSqlDeleteStats deleteStats = new DirectSqlDeleteStats(getDirectSql(), pm); return deleteStats.deleteTableColumnStatistics(getTable(), cols, engine); } @Override - protected Boolean getJdoResult(ObjectStore.GetHelper ctx) + protected Boolean getJdoResult() throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - return deleteTableColumnStatisticsViaJdo(catName, dbName, tableName, cols, engine); + return deleteTableColumnStatisticsViaJdo(tableName, cols, engine); } }.run(true); } - private boolean deleteTableColumnStatisticsViaJdo(String catName, String dbName, String tableName, + private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - boolean ret = false; - Query query = null; - try { - openTransaction(); - List mStatsObjColl; - // Note: this does not verify ACID state; called internally when removing cols/etc. - // Also called via an unused metastore API that checks for ACID tables. - query = pm.newQuery(MTableColumnStatistics.class); - String filter; - String parameters; - if (colNames != null && !colNames.isEmpty()) { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + (engine != null ? " && engine == t5" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + (engine != null ? ", java.lang.String t5" : ""); - } else { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + (engine != null ? " && engine == t4" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + (engine != null ? ", java.lang.String t4" : ""); - } + String catName = normalizeIdentifier(tblName.getCat()); + String dbName = normalizeIdentifier(tblName.getDb()); + String tableName = normalizeIdentifier(tblName.getTable()); + List mStatsObjColl; + // Note: this does not verify ACID state; called internally when removing cols/etc. + // Also called via an unused metastore API that checks for ACID tables. + Query query = pm.newQuery(MTableColumnStatistics.class); + String filter; + String parameters; + if (colNames != null && !colNames.isEmpty()) { + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + (engine != null ? " && engine == t5" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + (engine != null ? ", java.lang.String t5" : ""); + } else { + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + (engine != null ? " && engine == t4" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + (engine != null ? ", java.lang.String t4" : ""); + } - query.setFilter(filter); - query.declareParameters(parameters); - List params = new ArrayList<>(); - params.add(normalizeIdentifier(tableName)); - params.add(normalizeIdentifier(dbName)); - params.add(catName == null ? null : normalizeIdentifier(catName)); - if (colNames != null && !colNames.isEmpty()) { - params.add(colNames); - } - if (engine != null) { - params.add(engine); - } - mStatsObjColl = (List) query.executeWithArray(params.toArray()); - pm.retrieveAll(mStatsObjColl); - if (mStatsObjColl != null) { - pm.deletePersistentAll(mStatsObjColl); - } + query.setFilter(filter); + query.declareParameters(parameters); + List params = new ArrayList<>(); + params.add(normalizeIdentifier(tableName)); + params.add(normalizeIdentifier(dbName)); + params.add(catName == null ? null : normalizeIdentifier(catName)); + if (colNames != null && !colNames.isEmpty()) { + params.add(colNames); + } + if (engine != null) { + params.add(engine); + } + mStatsObjColl = (List) query.executeWithArray(params.toArray()); + pm.retrieveAll(mStatsObjColl); + if (mStatsObjColl != null) { + pm.deletePersistentAll(mStatsObjColl); + } - MTable mTable = getMTable(catName, dbName, tableName); - if (mTable != null) { - Map tableParams = mTable.getParameters(); - if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { - if (colNames == null || colNames.isEmpty()) { - StatsSetupConst.clearColumnStatsState(tableParams); - } else { - StatsSetupConst.removeColumnStatsState(tableParams, colNames); - } - mTable.setParameters(tableParams); + MTable mTable = getMTable(catName, dbName, tableName); + if (mTable != null) { + Map tableParams = mTable.getParameters(); + if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { + if (colNames == null || colNames.isEmpty()) { + StatsSetupConst.clearColumnStatsState(tableParams); + } else { + StatsSetupConst.removeColumnStatsState(tableParams, colNames); } + mTable.setParameters(tableParams); } - ret = commitTransaction(); - } finally { - rollbackAndCleanup(ret, query); } - return ret; + return true; } private void executePlainSQL(String sql, From 6e79f346743a2392344296dcc918b7d954fadfab Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 08:55:51 +0800 Subject: [PATCH 10/22] fix --- .../hadoop/hive/metastore/ObjectStore.java | 84 ++++++++++- .../hive/metastore/metastore/GetHelper.java | 4 +- .../metastore/iface/ColStatsStore.java | 3 + .../metastore/iface/ConstraintStore.java | 18 +++ .../metastore/metastore/iface/TableStore.java | 15 ++ .../metastore/metastore/iface/WLMStore.java | 18 +++ .../metastore/impl/ColStatsStoreImpl.java | 121 ++++++---------- .../metastore/impl/ConstraintStoreImpl.java | 21 +-- .../metastore/impl/TableStoreImpl.java | 134 +++--------------- .../hive/metastore/TestHiveMetaStore.java | 7 +- .../hive/metastore/VerifyingObjectStore.java | 40 ++++-- 11 files changed, 244 insertions(+), 221 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index f9523e4c4a58..275d71c198a2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -61,12 +61,15 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -239,7 +242,6 @@ private enum TXN_STATUS { private Transaction currentTransaction = null; private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; private Counter directSqlErrors; - private boolean areTxnStatsSupported = false; private PropertyStore propertyStore; private Map, Object> cachedImpls = new HashMap<>(); @@ -261,7 +263,6 @@ public Configuration getConf() { public void setConf(Configuration conf) { isInitialized = false; this.conf = conf; - this.areTxnStatsSupported = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_TXN_STATS_ENABLED); configureSSL(conf); PersistenceManagerProvider.updatePmfProperties(conf); @@ -1329,6 +1330,49 @@ public static Map convertMap(Map dnMap, Configur return parameters; } + public static Pair> getPartQueryWithParams( + PersistenceManager pm, + String catName, String dbName, String tblName, + List partNames) { + Query query = pm.newQuery(); + Map params = new HashMap<>(); + String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); + query.setFilter(filterStr); + LOG.debug(" JDOQL filter is {}", filterStr); + query.declareParameters(makeParameterDeclarationString(params)); + return Pair.of(query, params); + } + + public static String getJDOFilterStrForPartitionNames(String catName, String dbName, String tblName, + List partNames, Map params) { + StringBuilder sb = new StringBuilder( + "table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); + params.put("t1", normalizeIdentifier(tblName)); + params.put("t2", normalizeIdentifier(dbName)); + params.put("t3", normalizeIdentifier(catName)); + int n = 0; + for (Iterator itr = partNames.iterator(); itr.hasNext(); ) { + String pn = "p" + n; + n++; + String part = itr.next(); + params.put(pn, part); + sb.append("partitionName == ").append(pn); + sb.append(" || "); + } + sb.setLength(sb.length() - 4); // remove the last " || " + sb.append(')'); + return sb.toString(); + } + + public static String makeParameterDeclarationString(Map params) { + //Create the parameter declaration string + StringBuilder paramDecl = new StringBuilder(); + for (String key : params.keySet()) { + paramDecl.append(", java.lang.String ").append(key); + } + return paramDecl.toString(); + } + public static List convertToMFieldSchemas(List keys) { List mkeys = null; if (keys != null) { @@ -1662,6 +1706,42 @@ public static String verifyStatsChangeCtx(String fullTableName, Map statsParams, long statsWriteId, String queryValidWriteIdList, + boolean isCompleteStatsWriter) throws MetaException { + + // Note: can be changed to debug/info to verify the calls. + LOG.debug("isCurrentStatsValidForTheQuery with stats write ID {}; query {}; writer: {} params {}", + statsWriteId, queryValidWriteIdList, isCompleteStatsWriter, statsParams); + // return true since the stats does not seem to be transactional. + if (statsWriteId < 1) { + return true; + } + // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is + // written by an aborted transaction but TXNS has no entry for the transaction + // after compaction. Don't check for a complete stats writer - it may replace invalid stats. + if (!isCompleteStatsWriter && !StatsSetupConst.areBasicStatsUptoDate(statsParams)) { + return false; + } + + if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid. + ValidWriteIdList list4TheQuery = ValidReaderWriteIdList.fromValue(queryValidWriteIdList); + // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), + // that means it cannot possibly be a concurrent write. If it's not valid (we are not + // allowed to see it), that means it's either concurrent or aborted, same thing for us. + if (list4TheQuery.isWriteIdValid(statsWriteId)) { + return true; + } + // Updater is also allowed to overwrite stats from aborted txns, as long as they are not concurrent. + if (isCompleteStatsWriter && list4TheQuery.isWriteIdAborted(statsWriteId)) { + return true; + } + } + + return false; + } + @Override public String getMetastoreDbUuid() throws MetaException { String ret = getGuidFromDB(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java index 296e4654f879..4c911e919dbc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java @@ -195,13 +195,13 @@ private void handleDirectSqlError(Exception ex, String savePoint) throws MetaExc doUseDirectSql = false; } - private void setTransactionSavePoint(String savePoint) { + public void setTransactionSavePoint(String savePoint) { if (savePoint != null) { ((JDOTransaction) pm.currentTransaction()).setSavepoint(savePoint); } } - private void rollbackTransactionToSavePoint(String savePoint) { + public void rollbackTransactionToSavePoint(String savePoint) { if (savePoint != null) { ((JDOTransaction) pm.currentTransaction()).rollbackToSavepoint(savePoint); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java index 78254dacc94f..52b7aa23c6b3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java @@ -225,4 +225,7 @@ List getPartitionColStatsForData List getAllTableNamesForStats() throws MetaException, NoSuchObjectException; Map> getPartitionColsWithStats(TableName tableName) throws MetaException, NoSuchObjectException; + + public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) + throws MetaException, NoSuchObjectException; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java index be0e1e509698..018139235d43 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.metastore.metastore.iface; import java.util.List; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TableStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TableStore.java index bf180f8766ea..fdad367433ae 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TableStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TableStore.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; +import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; @@ -474,4 +475,18 @@ long updateParameterWithExpectedValue(Table table, String key, String expectedVa throws MetaException, NoSuchObjectException; MPartition ensureGetMPartition(TableName tableName, List partVals) throws MetaException; + + class AttachedMTableInfo { + public MTable mtbl; + public MColumnDescriptor mcd; + + public AttachedMTableInfo() {} + + public AttachedMTableInfo(MTable mtbl, MColumnDescriptor mcd) { + this.mtbl = mtbl; + this.mcd = mcd; + } + } + + AttachedMTableInfo getMTable(TableName tableName, boolean retrieveCD); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java index 11f984339637..57716f17ce68 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.metastore.metastore.iface; // work load management public interface WLMStore { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index d0203bfff6e8..6d07b8886e2e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -46,12 +46,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.Batchable; import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.Deadline; -import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.PersistenceManagerProvider; import org.apache.hadoop.hive.metastore.QueryWrapper; import org.apache.hadoop.hive.metastore.RawStore; @@ -94,6 +91,8 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.hive.metastore.Batchable.NO_BATCHING; +import static org.apache.hadoop.hive.metastore.ObjectStore.getPartQueryWithParams; +import static org.apache.hadoop.hive.metastore.ObjectStore.isCurrentStatsValidForTheQuery; import static org.apache.hadoop.hive.metastore.ObjectStore.verifyStatsChangeCtx; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; @@ -277,22 +276,25 @@ public Map updateTableColumnStatistics(ColumnStatistics colStats int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); long sleepInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + GetHelper transactionHelper = createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), + statsDesc.getTableName())); Map result = new RetryingExecutor<>(maxRetries, () -> { AtomicReference exceptionRef = new AtomicReference<>(); String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); + transactionHelper.setTransactionSavePoint(savePoint); executePlainSQL( sqlGenerator.addForUpdateNoWait("SELECT \"TBL_ID\" FROM \"TBLS\" WHERE \"TBL_ID\" = " + mTable.getId()), true, exception -> { - rollbackTransactionToSavePoint(savePoint); + transactionHelper.rollbackTransactionToSavePoint(savePoint); exceptionRef.set(exception); }); if (exceptionRef.get() != null) { throw new RetryingExecutor.RetryException(exceptionRef.get()); } pm.refresh(mTable); - Table table = convertToTable(mTable, conf); + Table table = baseStore.unwrap(TableStore.class).getTable(new TableName(catName, statsDesc.getDbName(), + statsDesc.getTableName()), null, -1); List colNames = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { colNames.add(statsObj.getColName()); @@ -328,7 +330,7 @@ public Map updateTableColumnStatistics(ColumnStatistics colStats if (errorMsg != null) { throw new MetaException(errorMsg); } - if (!isCurrentStatsValidForTheQuery(oldt, validWriteIds, true)) { + if (!isCurrentStatsValidForTheQuery(oldt.getParameters(), oldt.getWriteId(), validWriteIds, true)) { // Make sure we set the flag to invalid regardless of the current value. StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + dbname + "." + name); @@ -369,23 +371,25 @@ public Map updatePartitionColumnStatistics(Table table, MTable m int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); long sleepInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); + GetHelper helper = + createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName())); Map result = new RetryingExecutor<>(maxRetries, () -> { AtomicReference exceptionRef = new AtomicReference<>(); String savePoint = "ups_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); - setTransactionSavePoint(savePoint); + helper.setTransactionSavePoint(savePoint); executePlainSQL(sqlGenerator.addForUpdateNoWait( "SELECT \"PART_ID\" FROM \"PARTITIONS\" WHERE \"PART_ID\" = " + mPartition.getId()), true, exception -> { - rollbackTransactionToSavePoint(savePoint); + helper.rollbackTransactionToSavePoint(savePoint); exceptionRef.set(exception); }); if (exceptionRef.get() != null) { throw new RetryingExecutor.RetryException(exceptionRef.get()); } pm.refresh(mPartition); - Partition partition = convertToPart(catName, statsDesc.getDbName(), statsDesc.getTableName(), - mPartition, TxnUtils.isAcidTable(table), conf); + Partition partition = baseStore.unwrap(TableStore.class) + .getPartition(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), mPartition.getValues(), null); Map oldStats = Maps.newHashMap(); List stats = getMPartitionColumnStatistics(table, Lists.newArrayList(statsDesc.getPartName()), colNames, colStats.getEngine()); @@ -565,7 +569,7 @@ public ColumnStatistics getTableColumnStatistics( if (writeIdList != null) { MTable table = baseStore.ensureGetMTable(catName, dbName, tblName); isCompliant = !TxnUtils.isTransactionalTable(table.getParameters()) - || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table, writeIdList, false)); + || (areTxnStatsSupported && isCurrentStatsValidForTheQuery(table.getParameters(), table.getWriteId(), writeIdList, false)); } ColumnStatistics stats = getTableColumnStatisticsInternal( new TableName(catName, dbName, tblName), colNames, engine); @@ -579,7 +583,7 @@ protected ColumnStatistics getTableColumnStatisticsInternal( TableName tableName, final List colNames, String engine) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR); final boolean enableKll = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - return new GetStatHelper(this, tableName) { + return new GetStatHelper(tableName, this) { @Override protected ColumnStatistics getSqlResult() throws MetaException { String catName = normalizeIdentifier(tableName.getCat()); @@ -760,7 +764,8 @@ public AggrStats get_aggr_stats_for(TableName tableName, GetProjectionsSpec ps = new GetProjectionsSpec(); ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%'); ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values")); - List parts = getPartitionSpecsByFilterAndProjection(table, ps, fs); + List parts = baseStore.unwrap(TableStore.class) + .getPartitionSpecsByFilterAndProjection(table, ps, fs); // Loop through the given "partNames" list // checking isolation-level-compliance of each partition column stats. @@ -1118,7 +1123,7 @@ private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, pm.deletePersistentAll(mStatsObjColl); } - MTable mTable = getMTable(catName, dbName, tableName); + MTable mTable = baseStore.ensureGetMTable(catName, dbName, tableName); if (mTable != null) { Map tableParams = mTable.getParameters(); if (tableParams != null && tableParams.containsKey(StatsSetupConst.COLUMN_STATS_ACCURATE)) { @@ -1165,72 +1170,6 @@ private void executePlainSQL(String sql, } } - /** - * Return true if the current statistics in the Metastore is valid - * for the query of the given "txnId" and "queryValidWriteIdList". - * - * Note that a statistics entity is valid iff - * the stats is written by the current query or - * the conjunction of the following two are true: - * ~ COLUMN_STATE_ACCURATE(CSA) state is true - * ~ Isolation-level (snapshot) compliant with the query - * @param tbl MTable of the stats entity - * @param queryValidWriteIdList valid writeId list of the query - * @Precondition "tbl" should be retrieved from the TBLS table. - */ - private boolean isCurrentStatsValidForTheQuery(MTable tbl, String queryValidWriteIdList, - boolean isCompleteStatsWriter) throws MetaException { - return isCurrentStatsValidForTheQuery(tbl.getParameters(), tbl.getWriteId(), - queryValidWriteIdList, isCompleteStatsWriter); - } - - /** - * Return true if the current statistics in the Metastore is valid - * for the query of the given "txnId" and "queryValidWriteIdList". - * - * Note that a statistics entity is valid iff - * the stats is written by the current query or - * the conjunction of the following two are true: - * ~ COLUMN_STATE_ACCURATE(CSA) state is true - * ~ Isolation-level (snapshot) compliant with the query - * @param queryValidWriteIdList valid writeId list of the query - */ - // TODO: move to somewhere else - public static boolean isCurrentStatsValidForTheQuery( - Map statsParams, long statsWriteId, String queryValidWriteIdList, - boolean isCompleteStatsWriter) throws MetaException { - - // Note: can be changed to debug/info to verify the calls. - LOG.debug("isCurrentStatsValidForTheQuery with stats write ID {}; query {}; writer: {} params {}", - statsWriteId, queryValidWriteIdList, isCompleteStatsWriter, statsParams); - // return true since the stats does not seem to be transactional. - if (statsWriteId < 1) { - return true; - } - // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is - // written by an aborted transaction but TXNS has no entry for the transaction - // after compaction. Don't check for a complete stats writer - it may replace invalid stats. - if (!isCompleteStatsWriter && !StatsSetupConst.areBasicStatsUptoDate(statsParams)) { - return false; - } - - if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid. - ValidWriteIdList list4TheQuery = ValidReaderWriteIdList.fromValue(queryValidWriteIdList); - // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), - // that means it cannot possibly be a concurrent write. If it's not valid (we are not - // allowed to see it), that means it's either concurrent or aborted, same thing for us. - if (list4TheQuery.isWriteIdValid(statsWriteId)) { - return true; - } - // Updater is also allowed to overwrite stats from aborted txns, as long as they are not concurrent. - if (isCompleteStatsWriter && list4TheQuery.isWriteIdAborted(statsWriteId)) { - return true; - } - } - - return false; - } - private abstract class GetStatHelper extends GetHelper { public GetStatHelper(TableName tableName, RawStoreAware baseStore) throws MetaException { super(baseStore, tableName); @@ -1242,6 +1181,25 @@ protected String describeResult() { } } + private GetHelper createSubTransactionHelper(TableName tableName) throws MetaException { + return new GetHelper(this, tableName) { + @Override + protected String describeResult() { + return "A helper for managing sub-transaction"; + } + + @Override + protected String getSqlResult() { + throw new UnsupportedOperationException(); + } + + @Override + protected String getJdoResult() { + throw new UnsupportedOperationException(); + } + }; + } + @VisibleForTesting public void validateTableCols(Table table, List colNames) throws MetaException { List colList = table.getSd().getCols(); @@ -1260,4 +1218,5 @@ public void validateTableCols(Table table, List colNames) throws MetaExc } } + } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java index c2a1d8321cce..d45f9c76fd53 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -33,7 +33,6 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.QueryWrapper; import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.api.AllTableConstraintsRequest; @@ -203,7 +202,8 @@ private List addForeignKeys(List foreignKeys, bool final String fkTableName = normalizeIdentifier(foreignKeys.get(i).getFktable_name()); // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. // For instance, this is the case when we are creating the table. - final ObjectStore.AttachedMTableInfo nChildTable = getMTable(catName, fkTableDB, fkTableName, retrieveCD); + final TableStore.AttachedMTableInfo nChildTable = + baseStore.unwrap(TableStore.class).getMTable(new TableName(catName, fkTableDB, fkTableName), retrieveCD); final MTable childTable = nChildTable.mtbl; if (childTable == null) { throw new InvalidObjectException("Child table not found: " + fkTableName); @@ -221,7 +221,7 @@ private List addForeignKeys(List foreignKeys, bool // For primary keys, we retrieve the column descriptors if retrieveCD is true (which means // it is an alter table statement) or if it is a create table statement but we are // referencing another table instead of self for the primary key. - final ObjectStore.AttachedMTableInfo nParentTable; + final TableStore.AttachedMTableInfo nParentTable; final MTable parentTable; MColumnDescriptor parentCD; final List parentCols; @@ -236,7 +236,8 @@ private List addForeignKeys(List foreignKeys, bool existingTablePrimaryKeys = primaryKeys; existingTableUniqueConstraints = uniqueConstraints; } else { - nParentTable = getMTable(catName, pkTableDB, pkTableName, true); + nParentTable = + baseStore.unwrap(TableStore.class).getMTable(new TableName(catName, pkTableDB, pkTableName), true); parentTable = nParentTable.mtbl; if (parentTable == null) { throw new InvalidObjectException("Parent table not found: " + pkTableName); @@ -443,7 +444,8 @@ private List addPrimaryKeys(List pks, boolean retr // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. // For instance, this is the case when we are creating the table. - ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + TableStore.AttachedMTableInfo nParentTable = baseStore.unwrap(TableStore.class) + .getMTable(new TableName(catName, tableDB, tableName), retrieveCD); MTable parentTable = nParentTable.mtbl; if (parentTable == null) { throw new InvalidObjectException("Parent table not found: " + tableName); @@ -526,7 +528,8 @@ private List addUniqueConstraints(List // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. // For instance, this is the case when we are creating the table. - ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + TableStore.AttachedMTableInfo nParentTable = baseStore.unwrap(TableStore.class) + .getMTable(new TableName(catName, tableDB, tableName), retrieveCD); MTable parentTable = nParentTable.mtbl; if (parentTable == null) { throw new InvalidObjectException("Parent table not found: " + tableName); @@ -641,7 +644,8 @@ private MConstraint addConstraint(String catName, String tableDB, String tableNa String constraintName = null; // If retrieveCD is false, we do not need to do a deep retrieval of the Table Column Descriptor. // For instance, this is the case when we are creating the table. - ObjectStore.AttachedMTableInfo nParentTable = getMTable(catName, tableDB, tableName, retrieveCD); + TableStore.AttachedMTableInfo nParentTable = baseStore.unwrap(TableStore.class) + .getMTable(new TableName(catName, tableDB, tableName), retrieveCD); MTable parentTable = nParentTable.mtbl; if (parentTable == null) { throw new InvalidObjectException("Parent table not found: " + tableName); @@ -729,7 +733,8 @@ private List addNotNullConstraints(List statsParams, long statsWriteId, String queryValidWriteIdList, - boolean isCompleteStatsWriter) throws MetaException { - - // Note: can be changed to debug/info to verify the calls. - LOG.debug("isCurrentStatsValidForTheQuery with stats write ID {}; query {}; writer: {} params {}", - statsWriteId, queryValidWriteIdList, isCompleteStatsWriter, statsParams); - // return true since the stats does not seem to be transactional. - if (statsWriteId < 1) { - return true; - } - // This COLUMN_STATS_ACCURATE(CSA) state checking also includes the case that the stats is - // written by an aborted transaction but TXNS has no entry for the transaction - // after compaction. Don't check for a complete stats writer - it may replace invalid stats. - if (!isCompleteStatsWriter && !StatsSetupConst.areBasicStatsUptoDate(statsParams)) { - return false; - } - - if (queryValidWriteIdList != null) { // Can be null when stats are being reset to invalid. - ValidWriteIdList list4TheQuery = ValidReaderWriteIdList.fromValue(queryValidWriteIdList); - // Just check if the write ID is valid. If it's valid (i.e. we are allowed to see it), - // that means it cannot possibly be a concurrent write. If it's not valid (we are not - // allowed to see it), that means it's either concurrent or aborted, same thing for us. - if (list4TheQuery.isWriteIdValid(statsWriteId)) { - return true; - } - // Updater is also allowed to overwrite stats from aborted txns, as long as they are not concurrent. - if (isCompleteStatsWriter && list4TheQuery.isWriteIdAborted(statsWriteId)) { - return true; - } - } - - return false; - } - @Override public boolean addPartitions(TableName tableName, List parts) throws InvalidObjectException, MetaException { String catName = normalizeIdentifier(tableName.getCat()); @@ -852,7 +807,7 @@ public Partition getPartition(TableName tabName, List part_vals, String } MPartition mpart = getMPartition(catName, dbName, tableName, part_vals, table); part = convertToPart(catName, dbName, tableName, mpart, - TxnUtils.isAcidTable(table.getParameters()), conf); + TxnUtils.isAcidTable(table.getParameters())); if (part == null) { throw new NoSuchObjectException("partition values=" + part_vals.toString()); @@ -924,7 +879,7 @@ protected List getSqlResult() throws MetaException { protected List getJdoResult() throws MetaException { try { return convertToParts(catName, dbName, tblName, - listMPartitions(catName, dbName, tblName, args.getMax()), false, conf, args); + listMPartitions(catName, dbName, tblName, args.getMax()), false, args); } catch (Exception e) { LOG.error("Failed to convert to parts", e); throw new MetaException(e.getMessage()); @@ -1773,7 +1728,7 @@ private List getPartitionsViaOrmFilter(String catName, String dbName, pm.retrieveAll(mparts); // TODO: why is this inconsistent with what we get by names? LOG.debug("Done retrieving all objects for getPartitionsViaOrmFilter"); List results = - convertToParts(catName, dbName, tblName, mparts, isAcidTable, conf, args); + convertToParts(catName, dbName, tblName, mparts, isAcidTable, args); return results; } @@ -1804,7 +1759,7 @@ public List run(List input) throws MetaException { List mparts = (List) query.executeWithMap(queryWithParams.getRight()); List partitions = convertToParts(catName, dbName, tblName, mparts, - isAcidTable, conf, args); + isAcidTable, args); return partitions; } @@ -1958,7 +1913,7 @@ private Partition alterPartitionNoTxn(String catName, String dbname, } oldCd.set(oldCD); - return convertToPart(catName, dbname, name, oldp, TxnUtils.isAcidTable(table.getParameters()), conf); + return convertToPart(catName, dbname, name, oldp, TxnUtils.isAcidTable(table.getParameters())); } @Override @@ -2213,7 +2168,7 @@ protected List getJdoResult() throws MetaException { try { List mparts = listMPartitionsWithProjection(fieldNames, jdoFilter, params); return convertToParts(table.getCatName(), table.getDbName(), table.getTableName(), - mparts, false, conf, new GetPartitionsArgs.GetPartitionsArgsBuilder() + mparts, false, new GetPartitionsArgs.GetPartitionsArgsBuilder() .excludeParamKeyPattern(excludeParamKeyPattern) .includeParamKeyPattern(includeParamKeyPattern) .build()); @@ -2282,7 +2237,7 @@ protected List getJdoResult() args.getPart_vals(), args.getMax(), null); boolean isAcidTable = TxnUtils.isAcidTable(getTable()); for (MPartition o : parts) { - Partition part = convertToPart(catName, dbName, tblName, o, isAcidTable, conf, args); + Partition part = convertToPart(catName, dbName, tblName, o, isAcidTable, args); result.add(part); } return result; @@ -2735,7 +2690,7 @@ public Partition getPartitionWithAuth(TableName tableName, List partVals } MTable mtbl = mpart.getTable(); - Partition part = convertToPart(catName, dbName, tblName, mpart, TxnUtils.isAcidTable(mtbl.getParameters()), conf); + Partition part = convertToPart(catName, dbName, tblName, mpart, TxnUtils.isAcidTable(mtbl.getParameters())); if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { String partName = Warehouse.makePartName(convertToFieldSchemas(mtbl .getPartitionKeys()), partVals); @@ -2999,49 +2954,6 @@ private void putPersistentPrivObjects(MTable mtbl, List toPersistPrivObj } } - private Pair> getPartQueryWithParams( - PersistenceManager pm, - String catName, String dbName, String tblName, - List partNames) { - Query query = pm.newQuery(); - Map params = new HashMap<>(); - String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); - query.setFilter(filterStr); - LOG.debug(" JDOQL filter is {}", filterStr); - query.declareParameters(makeParameterDeclarationString(params)); - return Pair.of(query, params); - } - - private String getJDOFilterStrForPartitionNames(String catName, String dbName, String tblName, - List partNames, Map params) { - StringBuilder sb = new StringBuilder( - "table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); - params.put("t1", normalizeIdentifier(tblName)); - params.put("t2", normalizeIdentifier(dbName)); - params.put("t3", normalizeIdentifier(catName)); - int n = 0; - for (Iterator itr = partNames.iterator(); itr.hasNext(); ) { - String pn = "p" + n; - n++; - String part = itr.next(); - params.put(pn, part); - sb.append("partitionName == ").append(pn); - sb.append(" || "); - } - sb.setLength(sb.length() - 4); // remove the last " || " - sb.append(')'); - return sb.toString(); - } - - private String makeParameterDeclarationString(Map params) { - //Create the parameter declaration string - StringBuilder paramDecl = new StringBuilder(); - for (String key : params.keySet()) { - paramDecl.append(", java.lang.String ").append(key); - } - return paramDecl.toString(); - } - private Table convertToTable(MTable mtbl, Configuration conf) throws MetaException { if (mtbl == null) { return null; @@ -3336,8 +3248,6 @@ private MMVSource convertToSourceTable(String catalog, SourceTable sourceTable) * This method resets the stats to 0 and supports only backward compatibility with clients does not * send {@link SourceTable} instances. * - * Use {@link ObjectStore#convertToSourceTable(String, SourceTable, RawStore)} instead. - * * @param catalog Catalog name where source table is located * @param fullyQualifiedTableName fully qualified name of source table * @return {@link MMVSource} instance represents this source table. @@ -3436,7 +3346,7 @@ private MPartition convertToMPart(Partition part, MTable mt) } private Partition convertToPart(String catName, String dbName, String tblName, - MPartition mpart, boolean isAcidTable, Configuration conf, GetPartitionsArgs... args) + MPartition mpart, boolean isAcidTable, GetPartitionsArgs... args) throws MetaException { if (mpart == null) { return null; @@ -3459,11 +3369,11 @@ private Partition convertToPart(String catName, String dbName, String tblName, } private List convertToParts(String catName, String dbName, String tblName, - List mparts, boolean isAcidTable, Configuration conf, GetPartitionsArgs args) + List mparts, boolean isAcidTable, GetPartitionsArgs args) throws MetaException { List parts = new ArrayList<>(mparts.size()); for (MPartition mp : mparts) { - parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable, conf, args)); + parts.add(convertToPart(catName, dbName, tblName, mp, isAcidTable, args)); Deadline.checkTimeout(); } return parts; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index b6c579c9d052..3153cd1a3942 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -61,6 +61,8 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.dataconnector.jdbc.AbstractJDBCConnectorProvider; import org.apache.hadoop.hive.metastore.handler.AddPartitionsHandler; +import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; +import org.apache.hadoop.hive.metastore.metastore.impl.ColStatsStoreImpl; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo; @@ -3610,8 +3612,9 @@ public void testValidateTableCols() throws Throwable { List expectedCols = Lists.newArrayList(); expectedCols.add("name"); ObjectStore objStore = new ObjectStore(); + ColStatsStoreImpl impl = (ColStatsStoreImpl) objStore.unwrap(ColStatsStore.class); try { - objStore.validateTableCols(tbl, expectedCols); + impl.validateTableCols(tbl, expectedCols); } catch (MetaException ex) { throw new RuntimeException(ex); } @@ -3619,7 +3622,7 @@ public void testValidateTableCols() throws Throwable { expectedCols.add("doesntExist"); boolean exceptionFound = false; try { - objStore.validateTableCols(tbl, expectedCols); + impl.validateTableCols(tbl, expectedCols); } catch (MetaException ex) { assertEquals(ex.getMessage(), "Column doesntExist doesn't exist in table comptbl in database compdb"); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java index 65e7996b6e02..fd01623d9558 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java @@ -38,7 +38,9 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; +import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.utils.DirectSqlConfigurator; @@ -123,12 +125,14 @@ public List getPartitions( @Override public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName, List colNames, String engine) throws MetaException, NoSuchObjectException { - ColumnStatistics sqlResult = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, true, false); - ColumnStatistics jdoResult = getTableColumnStatisticsInternal( - catName, dbName, tableName, colNames, engine, false, true); - verifyObjects(sqlResult, jdoResult, ColumnStatistics.class); - return sqlResult; + ColStatsStore colStatsStore = unwrap(ColStatsStore.class); + try (DirectSqlConfigurator configurator = new DirectSqlConfigurator(conf, false)) { + ColumnStatistics jdoResult = colStatsStore.getTableColumnStatistics(new TableName(catName, dbName, tableName), colNames, engine); + configurator.tryDirectSql(true); + ColumnStatistics sqlResult = colStatsStore.getTableColumnStatistics(new TableName(catName, dbName, tableName), colNames, engine); + verifyObjects(sqlResult, jdoResult, ColumnStatistics.class); + return sqlResult; + } } @Override @@ -141,11 +145,15 @@ public List alterPartitions(String catName, String dbName, String tbl boolean success = false; try { openTransaction(); - MTable table = ensureGetMTable(catName, dbName, tblName); if (writeId > 0) { newParts.forEach(newPart -> newPart.setWriteId(writeId)); } - List partCols = convertToFieldSchemas(table.getPartitionKeys()); + TableName tableName = new TableName(catName, dbName, tblName); + Table table = unwrap(TableStore.class).getTable(tableName, queryWriteIdList, writeId); + if (table == null) { + throw new NoSuchObjectException("Table " + table + " doesn't exist"); + } + List partCols = table.getPartitionKeys(); List partNames = new ArrayList<>(); for (List partVal : part_vals) { partNames.add(Warehouse.makePartName(partCols, partVal)); @@ -186,12 +194,16 @@ public List alterPartitions(String catName, String dbName, String tbl public List getPartitionColumnStatistics(String catName, String dbName, String tableName, List partNames, List colNames, String engine) throws MetaException, NoSuchObjectException { - List sqlResult = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, true, false); - List jdoResult = getPartitionColumnStatisticsInternal( - catName, dbName, tableName, partNames, colNames, engine, false, true); - verifyLists(sqlResult, jdoResult, ColumnStatistics.class); - return sqlResult; + ColStatsStore colStatsStore = unwrap(ColStatsStore.class); + try (DirectSqlConfigurator configurator = new DirectSqlConfigurator(conf, false)) { + List jdoResult = + colStatsStore.getPartitionColumnStatistics(new TableName(catName, dbName, tableName), partNames, colNames, engine); + configurator.tryDirectSql(true); + List sqlResult = + colStatsStore.getPartitionColumnStatistics(new TableName(catName, dbName, tableName), partNames, colNames, engine); + verifyLists(sqlResult, jdoResult, ColumnStatistics.class); + return sqlResult; + } } private void verifyObjects( From 720ba614bf12366489d3ad3f7c97bc21402278a8 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 10:25:57 +0800 Subject: [PATCH 11/22] work load management --- .../hadoop/hive/metastore/ObjectStore.java | 1198 ----------------- .../hadoop/hive/metastore/RawStore.java | 118 +- .../hive/metastore/cache/CachedStore.java | 128 -- .../metastore/iface/ColStatsStore.java | 2 +- .../metastore/metastore/iface/WLMStore.java | 85 ++ .../metastore/impl/ColStatsStoreImpl.java | 2 - .../metastore/impl/WLMStoreImpl.java | 1047 ++++++++++++++ 7 files changed, 1215 insertions(+), 1365 deletions(-) create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 275d71c198a2..a272f12a79a7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.net.InetAddress; -import java.sql.SQLIntegrityConstraintViolationException; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -36,14 +35,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -70,7 +66,6 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Catalog; @@ -97,7 +92,6 @@ import org.apache.hadoop.hive.metastore.api.QueryState; import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.ReplicationMetricList; import org.apache.hadoop.hive.metastore.api.GetReplicationMetricsRequest; import org.apache.hadoop.hive.metastore.api.ReplicationMetrics; @@ -119,16 +113,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableParamsUpdate; import org.apache.hadoop.hive.metastore.api.Type; -import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMMapping; -import org.apache.hadoop.hive.metastore.api.WMNullablePool; -import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMPool; -import org.apache.hadoop.hive.metastore.api.WMPoolTrigger; -import org.apache.hadoop.hive.metastore.api.WMResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; -import org.apache.hadoop.hive.metastore.api.WMTrigger; -import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; @@ -151,7 +135,6 @@ import org.apache.hadoop.hive.metastore.model.MPackage; import org.apache.hadoop.hive.metastore.model.MPartitionEvent; import org.apache.hadoop.hive.metastore.model.MResourceUri; -import org.apache.hadoop.hive.metastore.model.MRuntimeStat; import org.apache.hadoop.hive.metastore.model.MScheduledExecution; import org.apache.hadoop.hive.metastore.model.MScheduledQuery; import org.apache.hadoop.hive.metastore.model.MSchemaVersion; @@ -159,12 +142,6 @@ import org.apache.hadoop.hive.metastore.model.MStoredProc; import org.apache.hadoop.hive.metastore.model.MType; import org.apache.hadoop.hive.metastore.model.MVersionTable; -import org.apache.hadoop.hive.metastore.model.MWMMapping; -import org.apache.hadoop.hive.metastore.model.MWMMapping.EntityType; -import org.apache.hadoop.hive.metastore.model.MWMPool; -import org.apache.hadoop.hive.metastore.model.MWMResourcePlan; -import org.apache.hadoop.hive.metastore.model.MWMResourcePlan.Status; -import org.apache.hadoop.hive.metastore.model.MWMTrigger; import org.apache.hadoop.hive.metastore.model.MReplicationMetrics; import org.apache.hadoop.hive.metastore.properties.CachingPropertyStore; import org.apache.hadoop.hive.metastore.properties.PropertyStore; @@ -175,7 +152,6 @@ import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.datanucleus.ExecutionContext; import org.datanucleus.api.jdo.JDOPersistenceManager; import org.datanucleus.api.jdo.JDOTransaction; @@ -191,7 +167,6 @@ import com.cronutils.model.time.ExecutionTime; import com.cronutils.parser.CronParser; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; /** * This class is the interface between the application logic and the database @@ -203,8 +178,6 @@ public class ObjectStore implements RawStore, Configurable { protected int batchSize = NO_BATCHING; - private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern( - "yyyy_MM_dd_HH_mm_ss"); /** * Verify the schema only once per JVM since the db connection info is static */ @@ -3308,1172 +3281,6 @@ protected void rollbackAndCleanup(boolean success, Query query) { } } - private void checkForConstraintException(Exception e, String msg) throws AlreadyExistsException { - if (getConstraintException(e) != null) { - LOG.error(msg, e); - throw new AlreadyExistsException(msg); - } - } - - private Throwable getConstraintException(Throwable t) { - while (t != null) { - if (t instanceof SQLIntegrityConstraintViolationException) { - return t; - } - t = t.getCause(); - } - return null; - } - - @Override - public void createResourcePlan( - WMResourcePlan resourcePlan, String copyFromName, int defaultPoolSize) - throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException { - boolean commited = false; - String rpName = normalizeIdentifier(resourcePlan.getName()); - if (rpName.isEmpty()) { - throw new InvalidObjectException("Resource name cannot be empty."); - } - MWMResourcePlan rp = null; - if (copyFromName == null) { - Integer queryParallelism = null; - if (resourcePlan.isSetQueryParallelism()) { - queryParallelism = resourcePlan.getQueryParallelism(); - if (queryParallelism <= 0) { - throw new InvalidObjectException("Query parallelism should be positive."); - } - } - rp = new MWMResourcePlan(rpName, queryParallelism, Status.DISABLED); - } else { - rp = new MWMResourcePlan(rpName, null, Status.DISABLED); - } - rp.setNs(resourcePlan.getNs()); - try { - openTransaction(); - pm.makePersistent(rp); - if (copyFromName != null) { - String ns = getNsOrDefault(resourcePlan.getNs()); - MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, ns, false); - if (copyFrom == null) { - throw new NoSuchObjectException(copyFromName + " in " + ns); - } - copyRpContents(rp, copyFrom); - } else { - // TODO: ideally, this should be moved outside to HiveMetaStore to be shared between - // all the RawStore-s. Right now there's no method to create a pool. - if (defaultPoolSize > 0) { - MWMPool defaultPool = new MWMPool(rp, "default", 1.0, defaultPoolSize, null); - pm.makePersistent(defaultPool); - rp.setPools(Sets.newHashSet(defaultPool)); - rp.setDefaultPool(defaultPool); - } - } - commited = commitTransaction(); - } catch (InvalidOperationException e) { - throw new RuntimeException(e); - } catch (Exception e) { - checkForConstraintException(e, "Resource plan already exists: "); - throw e; - } finally { - rollbackAndCleanup(commited, null); - } - } - - private void copyRpContents(MWMResourcePlan dest, MWMResourcePlan src) { - dest.setQueryParallelism(src.getQueryParallelism()); - dest.setNs(src.getNs()); - Map pools = new HashMap<>(); - Map> triggersToPools = new HashMap<>(); - for (MWMPool copyPool : src.getPools()) { - MWMPool pool = new MWMPool(dest, copyPool.getPath(), copyPool.getAllocFraction(), - copyPool.getQueryParallelism(), copyPool.getSchedulingPolicy()); - pm.makePersistent(pool); - pools.put(copyPool.getPath(), pool); - if (copyPool.getTriggers() != null) { - for (MWMTrigger trigger : copyPool.getTriggers()) { - Set p2t = triggersToPools.get(trigger.getName()); - if (p2t == null) { - p2t = new HashSet<>(); - triggersToPools.put(trigger.getName(), p2t); - } - p2t.add(pool); - pool.setTriggers(new HashSet<>()); - } - } - } - dest.setPools(new HashSet<>(pools.values())); - if (src.getDefaultPool() != null) { - dest.setDefaultPool(pools.get(src.getDefaultPool().getPath())); - } - Set mappings = new HashSet<>(); - for (MWMMapping copyMapping : src.getMappings()) { - MWMPool pool = null; - if (copyMapping.getPool() != null) { - pool = pools.get(copyMapping.getPool().getPath()); - } - MWMMapping mapping = new MWMMapping(dest, copyMapping.getEntityType(), - copyMapping.getEntityName(), pool, copyMapping.getOrdering()); - pm.makePersistent(mapping); - mappings.add(mapping); - } - dest.setMappings(mappings); - Set triggers = new HashSet<>(); - for (MWMTrigger copyTrigger : src.getTriggers()) { - Set p2t = triggersToPools.get(copyTrigger.getName()); - if (p2t == null) { - p2t = new HashSet<>(); - } - MWMTrigger trigger = new MWMTrigger(dest, copyTrigger.getName(), - copyTrigger.getTriggerExpression(), copyTrigger.getActionExpression(), p2t, - copyTrigger.getIsInUnmanaged()); - pm.makePersistent(trigger); - for (MWMPool pool : p2t) { - pool.getTriggers().add(trigger); - } - triggers.add(trigger); - } - dest.setTriggers(triggers); - } - - private WMResourcePlan fromMResourcePlan(MWMResourcePlan mplan) { - if (mplan == null) { - return null; - } - WMResourcePlan rp = new WMResourcePlan(); - rp.setName(mplan.getName()); - rp.setNs(mplan.getNs()); - rp.setStatus(WMResourcePlanStatus.valueOf(mplan.getStatus().name())); - if (mplan.getQueryParallelism() != null) { - rp.setQueryParallelism(mplan.getQueryParallelism()); - } - if (mplan.getDefaultPool() != null) { - rp.setDefaultPoolPath(mplan.getDefaultPool().getPath()); - } - return rp; - } - - private WMFullResourcePlan fullFromMResourcePlan(MWMResourcePlan mplan) { - if (mplan == null) { - return null; - } - WMFullResourcePlan rp = new WMFullResourcePlan(); - rp.setPlan(fromMResourcePlan(mplan)); - for (MWMPool mPool : mplan.getPools()) { - rp.addToPools(fromMPool(mPool, mplan.getName())); - for (MWMTrigger mTrigger : mPool.getTriggers()) { - rp.addToPoolTriggers(new WMPoolTrigger(mPool.getPath(), mTrigger.getName())); - } - } - for (MWMTrigger mTrigger : mplan.getTriggers()) { - rp.addToTriggers(fromMWMTrigger(mTrigger, mplan.getName())); - } - for (MWMMapping mMapping : mplan.getMappings()) { - rp.addToMappings(fromMMapping(mMapping, mplan.getName())); - } - return rp; - } - - private WMPool fromMPool(MWMPool mPool, String rpName) { - WMPool result = new WMPool(rpName, mPool.getPath()); - assert mPool.getAllocFraction() != null; - result.setAllocFraction(mPool.getAllocFraction()); - assert mPool.getQueryParallelism() != null; - result.setQueryParallelism(mPool.getQueryParallelism()); - result.setSchedulingPolicy(mPool.getSchedulingPolicy()); - result.setNs(mPool.getResourcePlan().getNs()); - return result; - } - - private WMMapping fromMMapping(MWMMapping mMapping, String rpName) { - WMMapping result = new WMMapping(rpName, - mMapping.getEntityType().toString(), mMapping.getEntityName()); - if (mMapping.getPool() != null) { - result.setPoolPath(mMapping.getPool().getPath()); - } - if (mMapping.getOrdering() != null) { - result.setOrdering(mMapping.getOrdering()); - } - result.setNs(mMapping.getResourcePlan().getNs()); - return result; - } - - private String getNsOrDefault(String ns) { - // This is only needed for old clients not setting NS in requests. - // Not clear how to handle this... this is properly a HS2 config but metastore needs its default - // value for backward compat, and we don't want it configurable separately because it's also - // used in upgrade scripts, were it cannot be configured. - return normalizeIdentifier(ns == null ? "default" : ns); - } - - @Override - public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { - boolean commited = false; - try { - openTransaction(); - WMFullResourcePlan fullRp = fullFromMResourcePlan(getMWMResourcePlan(name, ns, false)); - commited = commitTransaction(); - return fullRp; - } catch (InvalidOperationException e) { - // Should not happen, edit check is false. - throw new RuntimeException(e); - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck) - throws NoSuchObjectException, InvalidOperationException { - return getMWMResourcePlan(name, ns, editCheck, true); - } - - private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck, boolean mustExist) - throws NoSuchObjectException, InvalidOperationException { - MWMResourcePlan resourcePlan; - boolean commited = false; - Query query = null; - - name = normalizeIdentifier(name); - try { - query = createGetResourcePlanQuery(); - ns = getNsOrDefault(ns); - resourcePlan = (MWMResourcePlan) query.execute(name, ns); - pm.retrieve(resourcePlan); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - if (mustExist && resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); - } - if (editCheck && resourcePlan != null - && resourcePlan.getStatus() != MWMResourcePlan.Status.DISABLED) { - throw new InvalidOperationException("Resource plan must be disabled to edit it."); - } - return resourcePlan; - } - - private Query createGetResourcePlanQuery() { - openTransaction(); - Query query = pm.newQuery(MWMResourcePlan.class, "name == rpname && ns == nsname"); - query.declareParameters("java.lang.String rpname, java.lang.String nsname"); - query.setUnique(true); - return query; - } - - @Override - public List getAllResourcePlans(String ns) throws MetaException { - List resourcePlans = new ArrayList(); - boolean commited = false; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MWMResourcePlan.class, "ns == nsname"); - query.declareParameters("java.lang.String nsname"); - List mplans = (List) query.execute(getNsOrDefault(ns)); - pm.retrieveAll(mplans); - commited = commitTransaction(); - if (mplans != null) { - for (MWMResourcePlan mplan : mplans) { - resourcePlans.add(fromMResourcePlan(mplan)); - } - } - } finally { - rollbackAndCleanup(commited, query); - } - return resourcePlans; - } - - @Override - public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan changes, - boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - name = name == null ? null : normalizeIdentifier(name); - if (isReplace && name == null) { - throw new InvalidOperationException("Cannot replace without specifying the source plan"); - } - boolean commited = false; - Query query = null; - // This method only returns the result when activating a resource plan. - // We could also add a boolean flag to be specified by the caller to see - // when the result might be needed. - WMFullResourcePlan result = null; - try { - openTransaction(); - if (isReplace) { - result = handleAlterReplace(name, ns, changes); - } else { - result = handleSimpleAlter(name, ns, changes, canActivateDisabled, canDeactivate); - } - - commited = commitTransaction(); - return result; - } catch (Exception e) { - checkForConstraintException(e, "Resource plan name should be unique: "); - throw e; - } finally { - rollbackAndCleanup(commited, query); - } - } - - private WMFullResourcePlan handleSimpleAlter(String name, String ns, WMNullableResourcePlan changes, - boolean canActivateDisabled, boolean canDeactivate) - throws InvalidOperationException, NoSuchObjectException, MetaException { - MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan(ns) - : getMWMResourcePlan(name, ns, !changes.isSetStatus()); - boolean hasNsChange = changes.isSetNs() && !changes.getNs().equals(getNsOrDefault(plan.getNs())); - if (hasNsChange) { - throw new InvalidOperationException("Cannot change ns; from " + getNsOrDefault(plan.getNs()) - + " to " + changes.getNs()); - } - boolean hasNameChange = changes.isSetName() && !changes.getName().equals(name); - // Verify that field changes are consistent with what Hive does. Note: we could handle this. - if (changes.isSetIsSetQueryParallelism() - || changes.isSetIsSetDefaultPoolPath() || hasNameChange) { - if (changes.isSetStatus()) { - throw new InvalidOperationException("Cannot change values during status switch."); - } else if (plan.getStatus() != MWMResourcePlan.Status.DISABLED) { - throw new InvalidOperationException("Resource plan must be disabled to edit it."); - } - } - - // Handle rename and other changes. - if (changes.isSetName()) { - String newName = normalizeIdentifier(changes.getName()); - if (newName.isEmpty()) { - throw new InvalidOperationException("Cannot rename to empty value."); - } - if (!newName.equals(plan.getName())) { - plan.setName(newName); - } - } - if (changes.isSetIsSetQueryParallelism() && changes.isIsSetQueryParallelism()) { - if (changes.isSetQueryParallelism()) { - if (changes.getQueryParallelism() <= 0) { - throw new InvalidOperationException("queryParallelism should be positive."); - } - plan.setQueryParallelism(changes.getQueryParallelism()); - } else { - plan.setQueryParallelism(null); - } - } - if (changes.isSetIsSetDefaultPoolPath() && changes.isIsSetDefaultPoolPath()) { - if (changes.isSetDefaultPoolPath()) { - MWMPool pool = getPool(plan, changes.getDefaultPoolPath()); - plan.setDefaultPool(pool); - } else { - plan.setDefaultPool(null); - } - } - - // Handle the status change. - if (changes.isSetStatus()) { - return switchStatus(name, plan, - changes.getStatus().name(), canActivateDisabled, canDeactivate); - } - return null; - } - - private WMFullResourcePlan handleAlterReplace(String name, String ns, WMNullableResourcePlan changes) - throws InvalidOperationException, NoSuchObjectException, MetaException { - // Verify that field changes are consistent with what Hive does. Note: we could handle this. - if (changes.isSetQueryParallelism() || changes.isSetDefaultPoolPath()) { - throw new InvalidOperationException("Cannot change values during replace."); - } - boolean isReplacingSpecific = changes.isSetName(); - boolean isReplacingActive = (changes.isSetStatus() - && changes.getStatus() == WMResourcePlanStatus.ACTIVE); - if (isReplacingActive == isReplacingSpecific) { - throw new InvalidOperationException("Must specify a name, or the active plan; received " - + changes.getName() + ", " + (changes.isSetStatus() ? changes.getStatus() : null)); - } - if (name == null) { - throw new InvalidOperationException("Invalid replace - no name specified"); - } - ns = getNsOrDefault(ns); - MWMResourcePlan replacedPlan = isReplacingSpecific - ? getMWMResourcePlan(changes.getName(), ns, false) : getActiveMWMResourcePlan(ns); - MWMResourcePlan plan = getMWMResourcePlan(name, ns, false); - - if (replacedPlan.getName().equals(plan.getName())) { - throw new InvalidOperationException("A plan cannot replace itself"); - } - String oldNs = getNsOrDefault(replacedPlan.getNs()), newNs = getNsOrDefault(plan.getNs()); - if (!oldNs.equals(newNs)) { - throw new InvalidOperationException("Cannot change the namespace; replacing " - + oldNs + " with " + newNs); - } - - // We will inherit the name and status from the plan we are replacing. - String newName = replacedPlan.getName(); - int i = 0; - String copyName = generateOldPlanName(newName, i); - while (true) { - MWMResourcePlan dup = getMWMResourcePlan(copyName, ns, false, false); - if (dup == null) { - break; - } - // Note: this can still conflict with parallel transactions. We do not currently handle - // parallel changes from two admins (by design :(). - copyName = generateOldPlanName(newName, ++i); - } - replacedPlan.setName(copyName); - plan.setName(newName); - plan.setStatus(replacedPlan.getStatus()); - replacedPlan.setStatus(MWMResourcePlan.Status.DISABLED); - // TODO: add a configurable option to skip the history and just drop it? - return plan.getStatus() == Status.ACTIVE ? fullFromMResourcePlan(plan) : null; - } - - private String generateOldPlanName(String newName, int i) { - if (MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) { - // Do not use datetime in tests to avoid result changes. - return newName + "_old_" + i; - } else { - return newName + "_old_" - + LocalDateTime.now().format(YMDHMS_FORMAT) + (i == 0 ? "" : ("_" + i)); - } - } - - @Override - public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { - // Note: fullFromMResroucePlan needs to be called inside the txn, otherwise we could have - // deduplicated this with getActiveMWMResourcePlan. - boolean commited = false; - Query query = null; - WMFullResourcePlan result = null; - try { - query = createActivePlanQuery(); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( - Status.ACTIVE.toString(), getNsOrDefault(ns)); - if (mResourcePlan != null) { - result = fullFromMResourcePlan(mResourcePlan); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return result; - } - - private MWMResourcePlan getActiveMWMResourcePlan(String ns) { - boolean commited = false; - Query query = null; - MWMResourcePlan result = null; - try { - query = createActivePlanQuery(); - result = (MWMResourcePlan) query.execute( - Status.ACTIVE.toString(), getNsOrDefault(ns)); - pm.retrieve(result); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - return result; - } - - private Query createActivePlanQuery() { - openTransaction(); - Query query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus && ns == nsname"); - query.declareParameters("java.lang.String activeStatus, java.lang.String nsname"); - query.setUnique(true); - return query; - } - - private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePlan, String status, - boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException { - Status currentStatus = mResourcePlan.getStatus(); - Status newStatus = null; - try { - newStatus = Status.valueOf(status); - } catch (IllegalArgumentException e) { - throw new InvalidOperationException("Invalid status: " + status); - } - - if (newStatus == currentStatus) { - return null; - } - - boolean doActivate = false, doValidate = false; - switch (currentStatus) { - case ACTIVE: // No status change for active resource plan, first activate another plan. - if (!canDeactivate) { - throw new InvalidOperationException( - "Resource plan " + name - + " is active; activate another plan first, or disable workload management."); - } - break; - case DISABLED: - assert newStatus == Status.ACTIVE || newStatus == Status.ENABLED; - doValidate = true; - doActivate = (newStatus == Status.ACTIVE); - if (doActivate && !canActivateDisabled) { - throw new InvalidOperationException("Resource plan " + name - + " is disabled and should be enabled before activation (or in the same command)"); - } - break; - case ENABLED: - if (newStatus == Status.DISABLED) { - mResourcePlan.setStatus(newStatus); - return null; // A simple case. - } - assert newStatus == Status.ACTIVE; - doActivate = true; - break; - default: throw new AssertionError("Unexpected status " + currentStatus); - } - if (doValidate) { - // Note: this may use additional inputs from the caller, e.g. maximum query - // parallelism in the cluster based on physical constraints. - WMValidateResourcePlanResponse response = getResourcePlanErrors(mResourcePlan); - if (!response.getErrors().isEmpty()) { - throw new InvalidOperationException( - "ResourcePlan: " + name + " is invalid: " + response.getErrors()); - } - } - if (doActivate) { - // Deactivate currently active resource plan. - deactivateActiveResourcePlan(mResourcePlan.getNs()); - mResourcePlan.setStatus(newStatus); - return fullFromMResourcePlan(mResourcePlan); - } else { - mResourcePlan.setStatus(newStatus); - } - return null; - } - - private void deactivateActiveResourcePlan(String ns) { - boolean commited = false; - Query query = null; - try { - query = createActivePlanQuery(); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( - Status.ACTIVE.toString(), getNsOrDefault(ns)); - // We may not have an active resource plan in the start. - if (mResourcePlan != null) { - mResourcePlan.setStatus(Status.ENABLED); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - private static class PoolData { - double totalChildrenAllocFraction = 0; - boolean found = false; - boolean hasChildren = false; - } - - private PoolData getPoolData(Map poolInfo, String poolPath) { - PoolData poolData = poolInfo.get(poolPath); - if (poolData == null) { - poolData = new PoolData(); - poolInfo.put(poolPath, poolData); - } - return poolData; - } - - private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mResourcePlan) { - WMValidateResourcePlanResponse response = new WMValidateResourcePlanResponse(); - response.setErrors(new ArrayList()); - response.setWarnings(new ArrayList()); - Integer rpParallelism = mResourcePlan.getQueryParallelism(); - if (rpParallelism != null && rpParallelism < 1) { - response.addToErrors("Query parallelism should for resource plan be positive. Got: " + - rpParallelism); - } - int totalQueryParallelism = 0; - Map poolInfo = new HashMap<>(); - for (MWMPool pool : mResourcePlan.getPools()) { - PoolData currentPoolData = getPoolData(poolInfo, pool.getPath()); - currentPoolData.found = true; - String parent = getParentPath(pool.getPath(), ""); - PoolData parentPoolData = getPoolData(poolInfo, parent); - parentPoolData.hasChildren = true; - parentPoolData.totalChildrenAllocFraction += pool.getAllocFraction(); - if (pool.getQueryParallelism() != null && pool.getQueryParallelism() < 1) { - response.addToErrors("Invalid query parallelism for pool: " + pool.getPath()); - } else { - totalQueryParallelism += pool.getQueryParallelism(); - } - if (!MetaStoreUtils.isValidSchedulingPolicy(pool.getSchedulingPolicy())) { - response.addToErrors("Invalid scheduling policy " + pool.getSchedulingPolicy() + - " for pool: " + pool.getPath()); - } - } - if (rpParallelism != null) { - if (rpParallelism < totalQueryParallelism) { - response.addToErrors("Sum of all pools' query parallelism: " + totalQueryParallelism + - " exceeds resource plan query parallelism: " + rpParallelism); - } else if (rpParallelism != totalQueryParallelism) { - response.addToWarnings("Sum of all pools' query parallelism: " + totalQueryParallelism + - " is less than resource plan query parallelism: " + rpParallelism); - } - } - for (Entry entry : poolInfo.entrySet()) { - final PoolData poolData = entry.getValue(); - final boolean isRoot = entry.getKey().isEmpty(); - // Special case for root parent - if (isRoot) { - poolData.found = true; - if (!poolData.hasChildren) { - response.addToErrors("Root has no children"); - // TODO: change fractions to use decimal? somewhat brittle - } else if (Math.abs(1.0 - poolData.totalChildrenAllocFraction) > 0.00001) { - response.addToErrors("Sum of root children pools' alloc fraction should be 1.0 got: " + - poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); - } - } - if (!poolData.found) { - response.addToErrors("Pool does not exists but has children: " + entry.getKey()); - } - if (poolData.hasChildren) { - - if (!isRoot && (poolData.totalChildrenAllocFraction - 1.0) > 0.00001) { - response.addToErrors("Sum of children pools' alloc fraction should be less than 1 got: " - + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); - } - } - } - // trigger and action expressions are not validated here, since counters are not - // available and grammar check is there in the language itself. - return response; - } - - @Override - public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) - throws NoSuchObjectException, InvalidObjectException, MetaException { - name = normalizeIdentifier(name); - boolean committed = false; - Query query = null; - try { - query = createGetResourcePlanQuery(); - MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, ns); - if (mResourcePlan == null) { - throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns); - } - WMValidateResourcePlanResponse result = getResourcePlanErrors(mResourcePlan); - committed = commitTransaction(); - return result; - } finally { - rollbackAndCleanup(committed, query); - } - } - - @Override - public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - name = normalizeIdentifier(name); - boolean commited = false; - Query query = null; - try { - query = createGetResourcePlanQuery(); - MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); - pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? - if (resourcePlan == null) { - throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); - } - if (resourcePlan.getStatus() == Status.ACTIVE) { - throw new MetaException("Cannot drop an active resource plan"); - } - // First, drop all the dependencies. - resourcePlan.setDefaultPool(null); - pm.deletePersistentAll(resourcePlan.getTriggers()); - pm.deletePersistentAll(resourcePlan.getMappings()); - pm.deletePersistentAll(resourcePlan.getPools()); - pm.deletePersistent(resourcePlan); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public void createWMTrigger(WMTrigger trigger) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException { - boolean commited = false; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - trigger.getResourcePlanName(), trigger.getNs(), true); - MWMTrigger mTrigger = new MWMTrigger(resourcePlan, - normalizeIdentifier(trigger.getTriggerName()), trigger.getTriggerExpression(), - trigger.getActionExpression(), null, - trigger.isSetIsInUnmanaged() && trigger.isIsInUnmanaged()); - pm.makePersistent(mTrigger); - commited = commitTransaction(); - } catch (Exception e) { - checkForConstraintException(e, "Trigger already exists, use alter: "); - throw e; - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - @Override - public void alterWMTrigger(WMTrigger trigger) - throws NoSuchObjectException, InvalidOperationException, MetaException { - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - trigger.getResourcePlanName(), trigger.getNs(), true); - MWMTrigger mTrigger = getTrigger(resourcePlan, trigger.getTriggerName()); - // Update the object. - if (trigger.isSetTriggerExpression()) { - mTrigger.setTriggerExpression(trigger.getTriggerExpression()); - } - if (trigger.isSetActionExpression()) { - mTrigger.setActionExpression(trigger.getActionExpression()); - } - if (trigger.isSetIsInUnmanaged()) { - mTrigger.setIsInUnmanaged(trigger.isIsInUnmanaged()); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - private MWMTrigger getTrigger(MWMResourcePlan resourcePlan, String triggerName) - throws NoSuchObjectException { - triggerName = normalizeIdentifier(triggerName); - boolean commited = false; - Query query = null; - try { - openTransaction(); - // Get the MWMTrigger object from DN - query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); - query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); - query.setUnique(true); - MWMTrigger mTrigger = (MWMTrigger) query.execute(resourcePlan, triggerName); - if (mTrigger == null) { - throw new NoSuchObjectException("Cannot find trigger with name: " + triggerName); - } - pm.retrieve(mTrigger); - commited = commitTransaction(); - return mTrigger; - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException { - resourcePlanName = normalizeIdentifier(resourcePlanName); - triggerName = normalizeIdentifier(triggerName); - - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); - query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); - query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); - if (query.deletePersistentAll(resourcePlan, triggerName) != 1) { - throw new NoSuchObjectException("Cannot delete trigger: " + triggerName); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public List getTriggersForResourcePlan(String resourcePlanName, String ns) - throws NoSuchObjectException, MetaException { - List triggers = new ArrayList(); - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan; - try { - resourcePlan = getMWMResourcePlan(resourcePlanName, ns, false); - } catch (InvalidOperationException e) { - // Should not happen, edit check is false. - throw new RuntimeException(e); - } - query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp"); - query.declareParameters("MWMResourcePlan rp"); - List mTriggers = (List) query.execute(resourcePlan); - pm.retrieveAll(mTriggers); - commited = commitTransaction(); - if (mTriggers != null) { - for (MWMTrigger trigger : mTriggers) { - triggers.add(fromMWMTrigger(trigger, resourcePlanName)); - } - } - } finally { - rollbackAndCleanup(commited, query); - } - return triggers; - } - - private WMTrigger fromMWMTrigger(MWMTrigger mTrigger, String resourcePlanName) { - WMTrigger trigger = new WMTrigger(); - trigger.setResourcePlanName(resourcePlanName); - trigger.setTriggerName(mTrigger.getName()); - trigger.setTriggerExpression(mTrigger.getTriggerExpression()); - trigger.setActionExpression(mTrigger.getActionExpression()); - trigger.setIsInUnmanaged(mTrigger.getIsInUnmanaged()); - trigger.setNs(mTrigger.getResourcePlan().getNs()); - return trigger; - } - - @Override - public void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, - InvalidOperationException, MetaException { - boolean commited = false; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - pool.getResourcePlanName(), pool.getNs(), true); - - if (!poolParentExists(resourcePlan, pool.getPoolPath())) { - throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); - } - String policy = pool.getSchedulingPolicy(); - if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) { - throw new InvalidOperationException("Invalid scheduling policy " + policy); - } - MWMPool mPool = new MWMPool(resourcePlan, pool.getPoolPath(), pool.getAllocFraction(), - pool.getQueryParallelism(), policy); - pm.makePersistent(mPool); - commited = commitTransaction(); - } catch (Exception e) { - checkForConstraintException(e, "Pool already exists: "); - throw e; - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - @Override - public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, - NoSuchObjectException, InvalidOperationException, MetaException { - boolean commited = false; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - pool.getResourcePlanName(), pool.getNs(), true); - MWMPool mPool = getPool(resourcePlan, poolPath); - pm.retrieve(mPool); - if (pool.isSetAllocFraction()) { - mPool.setAllocFraction(pool.getAllocFraction()); - } - if (pool.isSetQueryParallelism()) { - mPool.setQueryParallelism(pool.getQueryParallelism()); - } - if (pool.isSetIsSetSchedulingPolicy() && pool.isIsSetSchedulingPolicy()) { - if (pool.isSetSchedulingPolicy()) { - String policy = pool.getSchedulingPolicy(); - if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) { - throw new InvalidOperationException("Invalid scheduling policy " + policy); - } - mPool.setSchedulingPolicy(pool.getSchedulingPolicy()); - } else { - mPool.setSchedulingPolicy(null); - } - } - if (pool.isSetPoolPath() && !pool.getPoolPath().equals(mPool.getPath())) { - moveDescendents(resourcePlan, mPool.getPath(), pool.getPoolPath()); - mPool.setPath(pool.getPoolPath()); - } - - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - private MWMPool getPool(MWMResourcePlan resourcePlan, String poolPath) - throws NoSuchObjectException { - poolPath = normalizeIdentifier(poolPath); - boolean commited = false; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path == poolPath"); - query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); - query.setUnique(true); - MWMPool mPool = (MWMPool) query.execute(resourcePlan, poolPath); - commited = commitTransaction(); - if (mPool == null) { - throw new NoSuchObjectException("Cannot find pool: " + poolPath); - } - pm.retrieve(mPool); - return mPool; - } finally { - rollbackAndCleanup(commited, query); - } - } - - private void moveDescendents(MWMResourcePlan resourcePlan, String path, String newPoolPath) - throws NoSuchObjectException { - if (!poolParentExists(resourcePlan, newPoolPath)) { - throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); - } - boolean commited = false; - Query query = null; - openTransaction(); - try { - query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); - query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); - List descPools = (List) query.execute(resourcePlan, path + "."); - pm.retrieveAll(descPools); - for (MWMPool pool : descPools) { - pool.setPath(newPoolPath + pool.getPath().substring(path.length())); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - private String getParentPath(String poolPath, String defValue) { - int idx = poolPath.lastIndexOf('.'); - if (idx == -1) { - return defValue; - } - return poolPath.substring(0, idx); - } - - private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) { - String parent = getParentPath(poolPath, null); - if (parent == null) { - return true; - } - try { - getPool(resourcePlan, parent); - return true; - } catch (NoSuchObjectException e) { - return false; - } - } - - @Override - public void dropWMPool(String resourcePlanName, String poolPath, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException { - poolPath = normalizeIdentifier(poolPath); - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); - if (resourcePlan.getDefaultPool() != null && - resourcePlan.getDefaultPool().getPath().equals(poolPath)) { - throw new InvalidOperationException("Cannot drop default pool of a resource plan"); - } - if (poolHasChildren(resourcePlan, poolPath)) { - throw new InvalidOperationException("Cannot drop a pool that has child pools"); - } - query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); - query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); - if (query.deletePersistentAll(resourcePlan, poolPath) != 1) { - throw new NoSuchObjectException("Cannot delete pool: " + poolPath); - } - commited = commitTransaction(); - } catch(Exception e) { - if (getConstraintException(e) != null) { - throw new InvalidOperationException("Please remove all mappings for this pool."); - } - throw e; - } finally { - rollbackAndCleanup(commited, query); - } - } - - private boolean poolHasChildren(MWMResourcePlan resourcePlan, String poolPath) { - boolean commited = false; - Query query = null; - try { - openTransaction(); - query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); - query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); - query.setResult("count(this)"); - query.setUnique(true); - Long count = (Long) query.execute(resourcePlan, poolPath + "."); - commited = commitTransaction(); - return count != null && count > 0; - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public void createOrUpdateWMMapping(WMMapping mapping, boolean update) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException { - EntityType entityType = EntityType.valueOf(mapping.getEntityType().trim().toUpperCase()); - String entityName = normalizeIdentifier(mapping.getEntityName()); - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - mapping.getResourcePlanName(), mapping.getNs(), true); - MWMPool pool = null; - if (mapping.isSetPoolPath()) { - pool = getPool(resourcePlan, mapping.getPoolPath()); - } - if (!update) { - MWMMapping mMapping = new MWMMapping(resourcePlan, entityType, entityName, pool, - mapping.getOrdering()); - pm.makePersistent(mMapping); - } else { - query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type " + - "&& entityName == name"); - query.declareParameters( - "MWMResourcePlan rp, java.lang.String type, java.lang.String name"); - query.setUnique(true); - MWMMapping mMapping = (MWMMapping) query.execute( - resourcePlan, entityType.toString(), entityName); - mMapping.setPool(pool); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public void dropWMMapping(WMMapping mapping) - throws NoSuchObjectException, InvalidOperationException, MetaException { - String entityType = mapping.getEntityType().trim().toUpperCase(); - String entityName = normalizeIdentifier(mapping.getEntityName()); - boolean commited = false; - Query query = null; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan( - mapping.getResourcePlanName(), mapping.getNs(), true); - query = pm.newQuery(MWMMapping.class, - "resourcePlan == rp && entityType == type && entityName == name"); - query.declareParameters("MWMResourcePlan rp, java.lang.String type, java.lang.String name"); - if (query.deletePersistentAll(resourcePlan, entityType, entityName) != 1) { - throw new NoSuchObjectException("Cannot delete mapping."); - } - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, query); - } - } - - @Override - public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, - InvalidOperationException, MetaException { - boolean commited = false; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); - MWMPool pool = getPool(resourcePlan, poolPath); - MWMTrigger trigger = getTrigger(resourcePlan, triggerName); - pool.getTriggers().add(trigger); - trigger.getPools().add(pool); - pm.makePersistent(pool); - pm.makePersistent(trigger); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - @Override - public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, - String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - boolean commited = false; - try { - openTransaction(); - MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); - MWMPool pool = getPool(resourcePlan, poolPath); - MWMTrigger trigger = getTrigger(resourcePlan, triggerName); - pool.getTriggers().remove(trigger); - trigger.getPools().remove(pool); - pm.makePersistent(pool); - pm.makePersistent(trigger); - commited = commitTransaction(); - } finally { - rollbackAndCleanup(commited, (Query)null); - } - } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws MetaException { - LOG.debug("runtimeStat: {}", stat); - MRuntimeStat mStat = MRuntimeStat.fromThrift(stat); - boolean committed = false; - openTransaction(); - try { - pm.makePersistent(mStat); - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, null); - } - } - - @Override - public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { - if (maxRetainSecs < 0) { - LOG.warn("runtime stats retention is disabled"); - return 0; - } - boolean committed = false; - Query q = null; - try { - openTransaction(); - int maxCreateTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs; - q = pm.newQuery(MRuntimeStat.class); - q.setFilter("createTime <= maxCreateTime"); - q.declareParameters("int maxCreateTime"); - long deleted = q.deletePersistentAll(maxCreateTime); - committed = commitTransaction(); - return (int) deleted; - } finally { - rollbackAndCleanup(committed, q); - } - } - - @Override - public List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { - boolean committed = false; - try { - openTransaction(); - List stats = getMRuntimeStats(maxEntries, maxCreateTime); - committed = commitTransaction(); - return stats; - } finally { - rollbackAndCleanup(committed, null); - } - } - - private List getMRuntimeStats(int maxEntries, int maxCreateTime) { - try (QueryWrapper query = new QueryWrapper(pm.newQuery(MRuntimeStat.class))) { - query.setOrdering("createTime descending"); - if (maxCreateTime > 0) { - query.setFilter("createTime < " + maxCreateTime); - } - if (maxEntries < 0) { - maxEntries = Integer.MAX_VALUE; - } - List ret = new ArrayList<>(); - List res = (List) query.execute(); - int totalEntries = 0; - for (MRuntimeStat mRuntimeStat : res) { - pm.retrieve(mRuntimeStat); - totalEntries += mRuntimeStat.getWeight(); - ret.add(MRuntimeStat.toThrift(mRuntimeStat)); - if (totalEntries >= maxEntries) { - break; - } - } - return ret; - } - } - @Override public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) throws MetaException { ensureScheduledQueriesEnabled(); @@ -4993,11 +3800,6 @@ public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperat } } - @Override - public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) { - - } - private void recoverInvalidScheduledQueryState(int timeoutSecs) { int maxLastUpdateTime = (int) (System.currentTimeMillis() / 1000) - timeoutSecs; try (QueryWrapper q = new QueryWrapper(pm.newQuery(MScheduledQuery.class))) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 9cc76a57d19b..8cfa9cfa2307 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -121,6 +121,7 @@ import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; +import org.apache.hadoop.hive.metastore.metastore.iface.WLMStore; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MTable; @@ -2101,60 +2102,96 @@ default List addCheckConstraints(List cc */ String getMetastoreDbUuid() throws MetaException; - void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) - throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException; + default void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) + throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException { + unwrap(WLMStore.class).createResourcePlan(resourcePlan, copyFrom, defaultPoolSize); + } - WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException; + default WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException { + return unwrap(WLMStore.class).getResourcePlan(name, string); + } - List getAllResourcePlans(String string) throws MetaException; + default List getAllResourcePlans(String string) throws MetaException { + return unwrap(WLMStore.class).getAllResourcePlans(string); + } - WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, + default WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException; + MetaException { + return unwrap(WLMStore.class).alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + } - WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException; + default WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + return unwrap(WLMStore.class).getActiveResourcePlan(ns); + } - WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) - throws NoSuchObjectException, InvalidObjectException, MetaException; + default WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) + throws NoSuchObjectException, InvalidObjectException, MetaException { + return unwrap(WLMStore.class).validateResourcePlan(name, ns); + } - void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException; + default void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + unwrap(WLMStore.class).dropResourcePlan(name, ns); + } - void createWMTrigger(WMTrigger trigger) + default void createWMTrigger(WMTrigger trigger) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException; + MetaException { + unwrap(WLMStore.class).createWMTrigger(trigger); + } - void alterWMTrigger(WMTrigger trigger) - throws NoSuchObjectException, InvalidOperationException, MetaException; + default void alterWMTrigger(WMTrigger trigger) + throws NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).alterWMTrigger(trigger); + } - void dropWMTrigger(String resourcePlanName, String triggerName, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException; + default void dropWMTrigger(String resourcePlanName, String triggerName, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).dropWMTrigger(resourcePlanName, triggerName, ns); + } - List getTriggersForResourcePlan(String resourcePlanName, String ns) - throws NoSuchObjectException, MetaException; + default List getTriggersForResourcePlan(String resourcePlanName, String ns) + throws NoSuchObjectException, MetaException { + return unwrap(WLMStore.class).getTriggersForResourcePlan(resourcePlanName, ns); + } - void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, - InvalidOperationException, MetaException; + default void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, + InvalidOperationException, MetaException { + unwrap(WLMStore.class).createPool(pool); + } - void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, - NoSuchObjectException, InvalidOperationException, MetaException; + default void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, + NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).alterPool(pool, poolPath); + } - void dropWMPool(String resourcePlanName, String poolPath, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException; + default void dropWMPool(String resourcePlanName, String poolPath, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).dropWMPool(resourcePlanName, poolPath, ns); + } - void createOrUpdateWMMapping(WMMapping mapping, boolean update) + default void createOrUpdateWMMapping(WMMapping mapping, boolean update) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException; + MetaException { + unwrap(WLMStore.class).createOrUpdateWMMapping(mapping, update); + } - void dropWMMapping(WMMapping mapping) - throws NoSuchObjectException, InvalidOperationException, MetaException; + default void dropWMMapping(WMMapping mapping) + throws NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).dropWMMapping(mapping); + } - void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) + default void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, - MetaException; + MetaException { + unwrap(WLMStore.class).createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); + } - void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException; + default void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException { + unwrap(WLMStore.class).dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); + } /** * Create a new ISchema. @@ -2289,13 +2326,19 @@ List getSchemaVersionsByColumns(String colName, String colNamespa void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException; /** Adds a RuntimeStat for persistence. */ - void addRuntimeStat(RuntimeStat stat) throws MetaException; + default void addRuntimeStat(RuntimeStat stat) throws MetaException { + unwrap(WLMStore.class).addRuntimeStat(stat); + } /** Reads runtime statistic entries. */ - List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException; + default List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { + return unwrap(WLMStore.class).getRuntimeStats(maxEntries, maxCreateTime); + } /** Removes outdated statistics. */ - int deleteRuntimeStats(int maxRetainSecs) throws MetaException; + default int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + return unwrap(WLMStore.class).deleteRuntimeStats(maxRetainSecs); + } default List getTableNamesWithStats() throws MetaException, NoSuchObjectException { return unwrap(ColStatsStore.class).getTableNamesWithStats(); @@ -2391,7 +2434,10 @@ default Map> updatePartitionColumnStatisticsInBatch( int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException; - void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList); + default void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) + throws MetaException, NoSuchObjectException { + unwrap(ColStatsStore.class).deleteAllPartitionColumnStatistics(tn, writeIdList); + } void createOrUpdateStoredProcedure(StoredProcedure proc) throws NoSuchObjectException, MetaException; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 7622efd155ca..134cb69c3141 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2542,11 +2542,6 @@ public List addCheckConstraints(List ccs rawStore.createISchema(schema); } - @Override public List getPartitionColStatsForDatabase(String catName, String dbName) - throws MetaException, NoSuchObjectException { - return rawStore.getPartitionColStatsForDatabase(catName, dbName); - } - @Override public void alterISchema(ISchemaName schemaName, ISchema newSchema) throws NoSuchObjectException, MetaException { rawStore.alterISchema(schemaName, newSchema); @@ -2611,99 +2606,10 @@ public RawStore getRawStore() { return rawStore.getMetastoreDbUuid(); } - @Override public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) - throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException { - rawStore.createResourcePlan(resourcePlan, copyFrom, defaultPoolSize); - } - - @Override public WMFullResourcePlan getResourcePlan(String name, String ns) - throws NoSuchObjectException, MetaException { - return rawStore.getResourcePlan(name, ns); - } - - @Override public List getAllResourcePlans(String ns) throws MetaException { - return rawStore.getAllResourcePlans(ns); - } - - @Override public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, - boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - return rawStore.alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); - } - - @Override public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { - return rawStore.getActiveResourcePlan(ns); - } - - @Override public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) - throws NoSuchObjectException, InvalidObjectException, MetaException { - return rawStore.validateResourcePlan(name, ns); - } - - @Override public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - rawStore.dropResourcePlan(name, ns); - } - - @Override public void createWMTrigger(WMTrigger trigger) - throws AlreadyExistsException, MetaException, NoSuchObjectException, InvalidOperationException { - rawStore.createWMTrigger(trigger); - } - - @Override public void alterWMTrigger(WMTrigger trigger) - throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.alterWMTrigger(trigger); - } - - @Override public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTrigger(resourcePlanName, triggerName, ns); - } - - @Override public List getTriggersForResourcePlan(String resourcePlanName, String ns) - throws NoSuchObjectException, MetaException { - return rawStore.getTriggersForResourcePlan(resourcePlanName, ns); - } - - @Override public void createPool(WMPool pool) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.createPool(pool); - } - - @Override public void alterPool(WMNullablePool pool, String poolPath) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.alterPool(pool, poolPath); - } - - @Override public void dropWMPool(String resourcePlanName, String poolPath, String ns) - throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMPool(resourcePlanName, poolPath, ns); - } - - @Override public void createOrUpdateWMMapping(WMMapping mapping, boolean update) - throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.createOrUpdateWMMapping(mapping, update); - } - - @Override public void dropWMMapping(WMMapping mapping) - throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMMapping(mapping); - } - - @Override public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, - String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); - } - - @Override public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, - String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); - } - public long getCacheUpdateCount() { return sharedCache.getUpdateCount(); } - static boolean isNotInBlackList(String catName, String dbName, String tblName) { String str = TableName.getQualified(catName, dbName, tblName); for (Pattern pattern : blacklistPatterns) { @@ -2777,35 +2683,6 @@ static boolean isBlacklistWhitelistEmpty(Configuration conf) { setCachePrewarmedState(false); } - @Override public void addRuntimeStat(RuntimeStat stat) throws MetaException { - rawStore.addRuntimeStat(stat); - } - - @Override public List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { - return rawStore.getRuntimeStats(maxEntries, maxCreateTime); - } - - @Override public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { - return rawStore.deleteRuntimeStats(maxRetainSecs); - } - - @Override public List getTableNamesWithStats() throws MetaException, NoSuchObjectException { - return rawStore.getTableNamesWithStats(); - } - - @Override public List getAllTableNamesForStats() throws MetaException, NoSuchObjectException { - return rawStore.getAllTableNamesForStats(); - } - - @Override public Map> getPartitionColsWithStats(String catName, String dbName, String tableName) - throws MetaException, NoSuchObjectException { - return rawStore.getPartitionColsWithStats(catName, dbName, tableName); - } - - @Override public List isPartOfMaterializedView(String catName, String dbName, String tblName) { - return rawStore.isPartOfMaterializedView(catName, dbName, tblName); - } - @Override public ScheduledQueryPollResponse scheduledQueryPoll(ScheduledQueryPollRequest request) throws MetaException { return rawStore.scheduledQueryPoll(request); @@ -2853,11 +2730,6 @@ public int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperat return rawStore.markScheduledExecutionsTimedOut(timeoutSecs); } - @Override - public void deleteAllPartitionColumnStatistics(TableName tn, String w) { - rawStore.deleteAllPartitionColumnStatistics(tn, w); - } - @Override public void createOrUpdateStoredProcedure(StoredProcedure proc) throws NoSuchObjectException, MetaException { rawStore.createOrUpdateStoredProcedure(proc); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java index 52b7aa23c6b3..26626ab78352 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ColStatsStore.java @@ -226,6 +226,6 @@ List getPartitionColStatsForData Map> getPartitionColsWithStats(TableName tableName) throws MetaException, NoSuchObjectException; - public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) + void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) throws MetaException, NoSuchObjectException; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java index 57716f17ce68..ca9e7a3c3c89 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java @@ -17,6 +17,91 @@ */ package org.apache.hadoop.hive.metastore.metastore.iface; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.RuntimeStat; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMMapping; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMTrigger; +import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; +import org.apache.hadoop.hive.metastore.metastore.impl.WLMStoreImpl; + // work load management +@MetaDescriptor(alias = "wlm", defaultImpl = WLMStoreImpl.class) public interface WLMStore { + void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) + throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException; + + WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException; + + List getAllResourcePlans(String string) throws MetaException; + + WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, + boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException; + + WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException; + + WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) + throws NoSuchObjectException, InvalidObjectException, MetaException; + + void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException; + + void createWMTrigger(WMTrigger trigger) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException; + + void alterWMTrigger(WMTrigger trigger) + throws NoSuchObjectException, InvalidOperationException, MetaException; + + void dropWMTrigger(String resourcePlanName, String triggerName, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException; + + List getTriggersForResourcePlan(String resourcePlanName, String ns) + throws NoSuchObjectException, MetaException; + + void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, + InvalidOperationException, MetaException; + + void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, + NoSuchObjectException, InvalidOperationException, MetaException; + + void dropWMPool(String resourcePlanName, String poolPath, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException; + + void createOrUpdateWMMapping(WMMapping mapping, boolean update) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException; + + void dropWMMapping(WMMapping mapping) + throws NoSuchObjectException, InvalidOperationException, MetaException; + + void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException; + + void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException; + + /** Adds a RuntimeStat for persistence. */ + void addRuntimeStat(RuntimeStat stat) throws MetaException; + + /** Reads runtime statistic entries. */ + List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException; + + /** Removes outdated statistics. */ + int deleteRuntimeStats(int maxRetainSecs) throws MetaException; + } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 6d07b8886e2e..c6c0a80b4377 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -1217,6 +1217,4 @@ public void validateTableCols(Table table, List colNames) throws MetaExc } } } - - } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java new file mode 100644 index 000000000000..8c8682c44e0a --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java @@ -0,0 +1,1047 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.metastore.impl; + +import com.google.common.collect.Sets; + +import javax.jdo.Query; +import java.sql.SQLIntegrityConstraintViolationException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.RawStore; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.RuntimeStat; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMMapping; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMPoolTrigger; +import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; +import org.apache.hadoop.hive.metastore.api.WMTrigger; +import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.iface.WLMStore; +import org.apache.hadoop.hive.metastore.model.MRuntimeStat; +import org.apache.hadoop.hive.metastore.model.MWMMapping; +import org.apache.hadoop.hive.metastore.model.MWMPool; +import org.apache.hadoop.hive.metastore.model.MWMResourcePlan; +import org.apache.hadoop.hive.metastore.model.MWMTrigger; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + +public class WLMStoreImpl extends RawStoreAware implements WLMStore { + private static final Logger LOG = LoggerFactory.getLogger(WLMStoreImpl.class); + private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern( + "yyyy_MM_dd_HH_mm_ss"); + + private void checkForConstraintException(Exception e, String msg) throws AlreadyExistsException { + if (getConstraintException(e) != null) { + LOG.error(msg, e); + throw new AlreadyExistsException(msg); + } + } + + private Throwable getConstraintException(Throwable t) { + while (t != null) { + if (t instanceof SQLIntegrityConstraintViolationException) { + return t; + } + t = t.getCause(); + } + return null; + } + + @Override + public void createResourcePlan( + WMResourcePlan resourcePlan, String copyFromName, int defaultPoolSize) + throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException { + String rpName = normalizeIdentifier(resourcePlan.getName()); + if (rpName.isEmpty()) { + throw new InvalidObjectException("Resource name cannot be empty."); + } + MWMResourcePlan rp = null; + if (copyFromName == null) { + Integer queryParallelism = null; + if (resourcePlan.isSetQueryParallelism()) { + queryParallelism = resourcePlan.getQueryParallelism(); + if (queryParallelism <= 0) { + throw new InvalidObjectException("Query parallelism should be positive."); + } + } + rp = new MWMResourcePlan(rpName, queryParallelism, MWMResourcePlan.Status.DISABLED); + } else { + rp = new MWMResourcePlan(rpName, null, MWMResourcePlan.Status.DISABLED); + } + rp.setNs(resourcePlan.getNs()); + try { + pm.makePersistent(rp); + if (copyFromName != null) { + String ns = getNsOrDefault(resourcePlan.getNs()); + MWMResourcePlan copyFrom = getMWMResourcePlan(copyFromName, ns, false); + if (copyFrom == null) { + throw new NoSuchObjectException(copyFromName + " in " + ns); + } + copyRpContents(rp, copyFrom); + } else { + // TODO: ideally, this should be moved outside to HiveMetaStore to be shared between + // all the RawStore-s. Right now there's no method to create a pool. + if (defaultPoolSize > 0) { + MWMPool defaultPool = new MWMPool(rp, "default", 1.0, defaultPoolSize, null); + pm.makePersistent(defaultPool); + rp.setPools(Sets.newHashSet(defaultPool)); + rp.setDefaultPool(defaultPool); + } + } + } catch (InvalidOperationException e) { + throw new RuntimeException(e); + } catch (Exception e) { + checkForConstraintException(e, "Resource plan already exists: "); + throw e; + } + } + + private void copyRpContents(MWMResourcePlan dest, MWMResourcePlan src) { + dest.setQueryParallelism(src.getQueryParallelism()); + dest.setNs(src.getNs()); + Map pools = new HashMap<>(); + Map> triggersToPools = new HashMap<>(); + for (MWMPool copyPool : src.getPools()) { + MWMPool pool = new MWMPool(dest, copyPool.getPath(), copyPool.getAllocFraction(), + copyPool.getQueryParallelism(), copyPool.getSchedulingPolicy()); + pm.makePersistent(pool); + pools.put(copyPool.getPath(), pool); + if (copyPool.getTriggers() != null) { + for (MWMTrigger trigger : copyPool.getTriggers()) { + Set p2t = triggersToPools.get(trigger.getName()); + if (p2t == null) { + p2t = new HashSet<>(); + triggersToPools.put(trigger.getName(), p2t); + } + p2t.add(pool); + pool.setTriggers(new HashSet<>()); + } + } + } + dest.setPools(new HashSet<>(pools.values())); + if (src.getDefaultPool() != null) { + dest.setDefaultPool(pools.get(src.getDefaultPool().getPath())); + } + Set mappings = new HashSet<>(); + for (MWMMapping copyMapping : src.getMappings()) { + MWMPool pool = null; + if (copyMapping.getPool() != null) { + pool = pools.get(copyMapping.getPool().getPath()); + } + MWMMapping mapping = new MWMMapping(dest, copyMapping.getEntityType(), + copyMapping.getEntityName(), pool, copyMapping.getOrdering()); + pm.makePersistent(mapping); + mappings.add(mapping); + } + dest.setMappings(mappings); + Set triggers = new HashSet<>(); + for (MWMTrigger copyTrigger : src.getTriggers()) { + Set p2t = triggersToPools.get(copyTrigger.getName()); + if (p2t == null) { + p2t = new HashSet<>(); + } + MWMTrigger trigger = new MWMTrigger(dest, copyTrigger.getName(), + copyTrigger.getTriggerExpression(), copyTrigger.getActionExpression(), p2t, + copyTrigger.getIsInUnmanaged()); + pm.makePersistent(trigger); + for (MWMPool pool : p2t) { + pool.getTriggers().add(trigger); + } + triggers.add(trigger); + } + dest.setTriggers(triggers); + } + + private WMResourcePlan fromMResourcePlan(MWMResourcePlan mplan) { + if (mplan == null) { + return null; + } + WMResourcePlan rp = new WMResourcePlan(); + rp.setName(mplan.getName()); + rp.setNs(mplan.getNs()); + rp.setStatus(WMResourcePlanStatus.valueOf(mplan.getStatus().name())); + if (mplan.getQueryParallelism() != null) { + rp.setQueryParallelism(mplan.getQueryParallelism()); + } + if (mplan.getDefaultPool() != null) { + rp.setDefaultPoolPath(mplan.getDefaultPool().getPath()); + } + return rp; + } + + private WMFullResourcePlan fullFromMResourcePlan(MWMResourcePlan mplan) { + if (mplan == null) { + return null; + } + WMFullResourcePlan rp = new WMFullResourcePlan(); + rp.setPlan(fromMResourcePlan(mplan)); + for (MWMPool mPool : mplan.getPools()) { + rp.addToPools(fromMPool(mPool, mplan.getName())); + for (MWMTrigger mTrigger : mPool.getTriggers()) { + rp.addToPoolTriggers(new WMPoolTrigger(mPool.getPath(), mTrigger.getName())); + } + } + for (MWMTrigger mTrigger : mplan.getTriggers()) { + rp.addToTriggers(fromMWMTrigger(mTrigger, mplan.getName())); + } + for (MWMMapping mMapping : mplan.getMappings()) { + rp.addToMappings(fromMMapping(mMapping, mplan.getName())); + } + return rp; + } + + private WMPool fromMPool(MWMPool mPool, String rpName) { + WMPool result = new WMPool(rpName, mPool.getPath()); + assert mPool.getAllocFraction() != null; + result.setAllocFraction(mPool.getAllocFraction()); + assert mPool.getQueryParallelism() != null; + result.setQueryParallelism(mPool.getQueryParallelism()); + result.setSchedulingPolicy(mPool.getSchedulingPolicy()); + result.setNs(mPool.getResourcePlan().getNs()); + return result; + } + + private WMMapping fromMMapping(MWMMapping mMapping, String rpName) { + WMMapping result = new WMMapping(rpName, + mMapping.getEntityType().toString(), mMapping.getEntityName()); + if (mMapping.getPool() != null) { + result.setPoolPath(mMapping.getPool().getPath()); + } + if (mMapping.getOrdering() != null) { + result.setOrdering(mMapping.getOrdering()); + } + result.setNs(mMapping.getResourcePlan().getNs()); + return result; + } + + private String getNsOrDefault(String ns) { + // This is only needed for old clients not setting NS in requests. + // Not clear how to handle this... this is properly a HS2 config but metastore needs its default + // value for backward compat, and we don't want it configurable separately because it's also + // used in upgrade scripts, were it cannot be configured. + return normalizeIdentifier(ns == null ? "default" : ns); + } + + @Override + public WMFullResourcePlan getResourcePlan(String name, String ns) throws NoSuchObjectException { + try { + return fullFromMResourcePlan(getMWMResourcePlan(name, ns, false)); + } catch (InvalidOperationException e) { + // Should not happen, edit check is false. + throw new RuntimeException(e); + } + } + + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck) + throws NoSuchObjectException, InvalidOperationException { + return getMWMResourcePlan(name, ns, editCheck, true); + } + + private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editCheck, boolean mustExist) + throws NoSuchObjectException, InvalidOperationException { + MWMResourcePlan resourcePlan; + + name = normalizeIdentifier(name); + Query query = createGetResourcePlanQuery(); + ns = getNsOrDefault(ns); + resourcePlan = (MWMResourcePlan) query.execute(name, ns); + pm.retrieve(resourcePlan); + if (mustExist && resourcePlan == null) { + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); + } + if (editCheck && resourcePlan != null + && resourcePlan.getStatus() != MWMResourcePlan.Status.DISABLED) { + throw new InvalidOperationException("Resource plan must be disabled to edit it."); + } + return resourcePlan; + } + + private Query createGetResourcePlanQuery() { + Query query = pm.newQuery(MWMResourcePlan.class, "name == rpname && ns == nsname"); + query.declareParameters("java.lang.String rpname, java.lang.String nsname"); + query.setUnique(true); + return query; + } + + @Override + public List getAllResourcePlans(String ns) throws MetaException { + List resourcePlans = new ArrayList(); + Query query = pm.newQuery(MWMResourcePlan.class, "ns == nsname"); + query.declareParameters("java.lang.String nsname"); + List mplans = (List) query.execute(getNsOrDefault(ns)); + pm.retrieveAll(mplans); + if (mplans != null) { + for (MWMResourcePlan mplan : mplans) { + resourcePlans.add(fromMResourcePlan(mplan)); + } + } + return resourcePlans; + } + + @Override + public WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan changes, + boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { + name = name == null ? null : normalizeIdentifier(name); + if (isReplace && name == null) { + throw new InvalidOperationException("Cannot replace without specifying the source plan"); + } + // This method only returns the result when activating a resource plan. + // We could also add a boolean flag to be specified by the caller to see + // when the result might be needed. + WMFullResourcePlan result = null; + try { + if (isReplace) { + result = handleAlterReplace(name, ns, changes); + } else { + result = handleSimpleAlter(name, ns, changes, canActivateDisabled, canDeactivate); + } + return result; + } catch (Exception e) { + checkForConstraintException(e, "Resource plan name should be unique: "); + throw e; + } + } + + private WMFullResourcePlan handleSimpleAlter(String name, String ns, WMNullableResourcePlan changes, + boolean canActivateDisabled, boolean canDeactivate) + throws InvalidOperationException, NoSuchObjectException, MetaException { + MWMResourcePlan plan = name == null ? getActiveMWMResourcePlan(ns) + : getMWMResourcePlan(name, ns, !changes.isSetStatus()); + boolean hasNsChange = changes.isSetNs() && !changes.getNs().equals(getNsOrDefault(plan.getNs())); + if (hasNsChange) { + throw new InvalidOperationException("Cannot change ns; from " + getNsOrDefault(plan.getNs()) + + " to " + changes.getNs()); + } + boolean hasNameChange = changes.isSetName() && !changes.getName().equals(name); + // Verify that field changes are consistent with what Hive does. Note: we could handle this. + if (changes.isSetIsSetQueryParallelism() + || changes.isSetIsSetDefaultPoolPath() || hasNameChange) { + if (changes.isSetStatus()) { + throw new InvalidOperationException("Cannot change values during status switch."); + } else if (plan.getStatus() != MWMResourcePlan.Status.DISABLED) { + throw new InvalidOperationException("Resource plan must be disabled to edit it."); + } + } + + // Handle rename and other changes. + if (changes.isSetName()) { + String newName = normalizeIdentifier(changes.getName()); + if (newName.isEmpty()) { + throw new InvalidOperationException("Cannot rename to empty value."); + } + if (!newName.equals(plan.getName())) { + plan.setName(newName); + } + } + if (changes.isSetIsSetQueryParallelism() && changes.isIsSetQueryParallelism()) { + if (changes.isSetQueryParallelism()) { + if (changes.getQueryParallelism() <= 0) { + throw new InvalidOperationException("queryParallelism should be positive."); + } + plan.setQueryParallelism(changes.getQueryParallelism()); + } else { + plan.setQueryParallelism(null); + } + } + if (changes.isSetIsSetDefaultPoolPath() && changes.isIsSetDefaultPoolPath()) { + if (changes.isSetDefaultPoolPath()) { + MWMPool pool = getPool(plan, changes.getDefaultPoolPath()); + plan.setDefaultPool(pool); + } else { + plan.setDefaultPool(null); + } + } + + // Handle the status change. + if (changes.isSetStatus()) { + return switchStatus(name, plan, + changes.getStatus().name(), canActivateDisabled, canDeactivate); + } + return null; + } + + private WMFullResourcePlan handleAlterReplace(String name, String ns, WMNullableResourcePlan changes) + throws InvalidOperationException, NoSuchObjectException, MetaException { + // Verify that field changes are consistent with what Hive does. Note: we could handle this. + if (changes.isSetQueryParallelism() || changes.isSetDefaultPoolPath()) { + throw new InvalidOperationException("Cannot change values during replace."); + } + boolean isReplacingSpecific = changes.isSetName(); + boolean isReplacingActive = (changes.isSetStatus() + && changes.getStatus() == WMResourcePlanStatus.ACTIVE); + if (isReplacingActive == isReplacingSpecific) { + throw new InvalidOperationException("Must specify a name, or the active plan; received " + + changes.getName() + ", " + (changes.isSetStatus() ? changes.getStatus() : null)); + } + if (name == null) { + throw new InvalidOperationException("Invalid replace - no name specified"); + } + ns = getNsOrDefault(ns); + MWMResourcePlan replacedPlan = isReplacingSpecific + ? getMWMResourcePlan(changes.getName(), ns, false) : getActiveMWMResourcePlan(ns); + MWMResourcePlan plan = getMWMResourcePlan(name, ns, false); + + if (replacedPlan.getName().equals(plan.getName())) { + throw new InvalidOperationException("A plan cannot replace itself"); + } + String oldNs = getNsOrDefault(replacedPlan.getNs()), newNs = getNsOrDefault(plan.getNs()); + if (!oldNs.equals(newNs)) { + throw new InvalidOperationException("Cannot change the namespace; replacing " + + oldNs + " with " + newNs); + } + + // We will inherit the name and status from the plan we are replacing. + String newName = replacedPlan.getName(); + int i = 0; + String copyName = generateOldPlanName(newName, i); + while (true) { + MWMResourcePlan dup = getMWMResourcePlan(copyName, ns, false, false); + if (dup == null) { + break; + } + // Note: this can still conflict with parallel transactions. We do not currently handle + // parallel changes from two admins (by design :(). + copyName = generateOldPlanName(newName, ++i); + } + replacedPlan.setName(copyName); + plan.setName(newName); + plan.setStatus(replacedPlan.getStatus()); + replacedPlan.setStatus(MWMResourcePlan.Status.DISABLED); + // TODO: add a configurable option to skip the history and just drop it? + return plan.getStatus() == MWMResourcePlan.Status.ACTIVE ? fullFromMResourcePlan(plan) : null; + } + + private String generateOldPlanName(String newName, int i) { + if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) { + // Do not use datetime in tests to avoid result changes. + return newName + "_old_" + i; + } else { + return newName + "_old_" + + LocalDateTime.now().format(YMDHMS_FORMAT) + (i == 0 ? "" : ("_" + i)); + } + } + + @Override + public WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { + // Note: fullFromMResroucePlan needs to be called inside the txn, otherwise we could have + // deduplicated this with getActiveMWMResourcePlan. + WMFullResourcePlan result = null; + Query query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns)); + if (mResourcePlan != null) { + result = fullFromMResourcePlan(mResourcePlan); + } + return result; + } + + private MWMResourcePlan getActiveMWMResourcePlan(String ns) { + MWMResourcePlan result = null; + Query query = createActivePlanQuery(); + result = (MWMResourcePlan) query.execute( + MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns)); + pm.retrieve(result); + return result; + } + + private Query createActivePlanQuery() { + Query query = pm.newQuery(MWMResourcePlan.class, "status == activeStatus && ns == nsname"); + query.declareParameters("java.lang.String activeStatus, java.lang.String nsname"); + query.setUnique(true); + return query; + } + + private WMFullResourcePlan switchStatus(String name, MWMResourcePlan mResourcePlan, String status, + boolean canActivateDisabled, boolean canDeactivate) throws InvalidOperationException { + MWMResourcePlan.Status currentStatus = mResourcePlan.getStatus(); + MWMResourcePlan.Status newStatus = null; + try { + newStatus = MWMResourcePlan.Status.valueOf(status); + } catch (IllegalArgumentException e) { + throw new InvalidOperationException("Invalid status: " + status); + } + + if (newStatus == currentStatus) { + return null; + } + + boolean doActivate = false, doValidate = false; + switch (currentStatus) { + case ACTIVE: // No status change for active resource plan, first activate another plan. + if (!canDeactivate) { + throw new InvalidOperationException( + "Resource plan " + name + + " is active; activate another plan first, or disable workload management."); + } + break; + case DISABLED: + assert newStatus == MWMResourcePlan.Status.ACTIVE || newStatus == MWMResourcePlan.Status.ENABLED; + doValidate = true; + doActivate = (newStatus == MWMResourcePlan.Status.ACTIVE); + if (doActivate && !canActivateDisabled) { + throw new InvalidOperationException("Resource plan " + name + + " is disabled and should be enabled before activation (or in the same command)"); + } + break; + case ENABLED: + if (newStatus == MWMResourcePlan.Status.DISABLED) { + mResourcePlan.setStatus(newStatus); + return null; // A simple case. + } + assert newStatus == MWMResourcePlan.Status.ACTIVE; + doActivate = true; + break; + default: throw new AssertionError("Unexpected status " + currentStatus); + } + if (doValidate) { + // Note: this may use additional inputs from the caller, e.g. maximum query + // parallelism in the cluster based on physical constraints. + WMValidateResourcePlanResponse response = getResourcePlanErrors(mResourcePlan); + if (!response.getErrors().isEmpty()) { + throw new InvalidOperationException( + "ResourcePlan: " + name + " is invalid: " + response.getErrors()); + } + } + if (doActivate) { + // Deactivate currently active resource plan. + deactivateActiveResourcePlan(mResourcePlan.getNs()); + mResourcePlan.setStatus(newStatus); + return fullFromMResourcePlan(mResourcePlan); + } else { + mResourcePlan.setStatus(newStatus); + } + return null; + } + + private void deactivateActiveResourcePlan(String ns) { + Query query = createActivePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute( + MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns)); + // We may not have an active resource plan in the start. + if (mResourcePlan != null) { + mResourcePlan.setStatus(MWMResourcePlan.Status.ENABLED); + } + } + + private static class PoolData { + double totalChildrenAllocFraction = 0; + boolean found = false; + boolean hasChildren = false; + } + + private PoolData getPoolData(Map poolInfo, String poolPath) { + PoolData poolData = poolInfo.get(poolPath); + if (poolData == null) { + poolData = new PoolData(); + poolInfo.put(poolPath, poolData); + } + return poolData; + } + + private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mResourcePlan) { + WMValidateResourcePlanResponse response = new WMValidateResourcePlanResponse(); + response.setErrors(new ArrayList()); + response.setWarnings(new ArrayList()); + Integer rpParallelism = mResourcePlan.getQueryParallelism(); + if (rpParallelism != null && rpParallelism < 1) { + response.addToErrors("Query parallelism should for resource plan be positive. Got: " + + rpParallelism); + } + int totalQueryParallelism = 0; + Map poolInfo = new HashMap<>(); + for (MWMPool pool : mResourcePlan.getPools()) { + PoolData currentPoolData = getPoolData(poolInfo, pool.getPath()); + currentPoolData.found = true; + String parent = getParentPath(pool.getPath(), ""); + PoolData parentPoolData = getPoolData(poolInfo, parent); + parentPoolData.hasChildren = true; + parentPoolData.totalChildrenAllocFraction += pool.getAllocFraction(); + if (pool.getQueryParallelism() != null && pool.getQueryParallelism() < 1) { + response.addToErrors("Invalid query parallelism for pool: " + pool.getPath()); + } else { + totalQueryParallelism += pool.getQueryParallelism(); + } + if (!MetaStoreUtils.isValidSchedulingPolicy(pool.getSchedulingPolicy())) { + response.addToErrors("Invalid scheduling policy " + pool.getSchedulingPolicy() + + " for pool: " + pool.getPath()); + } + } + if (rpParallelism != null) { + if (rpParallelism < totalQueryParallelism) { + response.addToErrors("Sum of all pools' query parallelism: " + totalQueryParallelism + + " exceeds resource plan query parallelism: " + rpParallelism); + } else if (rpParallelism != totalQueryParallelism) { + response.addToWarnings("Sum of all pools' query parallelism: " + totalQueryParallelism + + " is less than resource plan query parallelism: " + rpParallelism); + } + } + for (Map.Entry entry : poolInfo.entrySet()) { + final PoolData poolData = entry.getValue(); + final boolean isRoot = entry.getKey().isEmpty(); + // Special case for root parent + if (isRoot) { + poolData.found = true; + if (!poolData.hasChildren) { + response.addToErrors("Root has no children"); + // TODO: change fractions to use decimal? somewhat brittle + } else if (Math.abs(1.0 - poolData.totalChildrenAllocFraction) > 0.00001) { + response.addToErrors("Sum of root children pools' alloc fraction should be 1.0 got: " + + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); + } + } + if (!poolData.found) { + response.addToErrors("Pool does not exists but has children: " + entry.getKey()); + } + if (poolData.hasChildren) { + + if (!isRoot && (poolData.totalChildrenAllocFraction - 1.0) > 0.00001) { + response.addToErrors("Sum of children pools' alloc fraction should be less than 1 got: " + + poolData.totalChildrenAllocFraction + " for pool: " + entry.getKey()); + } + } + } + // trigger and action expressions are not validated here, since counters are not + // available and grammar check is there in the language itself. + return response; + } + + @Override + public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) + throws NoSuchObjectException, InvalidObjectException, MetaException { + name = normalizeIdentifier(name); + Query query = createGetResourcePlanQuery(); + MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, ns); + if (mResourcePlan == null) { + throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns); + } + WMValidateResourcePlanResponse result = getResourcePlanErrors(mResourcePlan); + return result; + } + + @Override + public void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { + name = normalizeIdentifier(name); + Query query = createGetResourcePlanQuery(); + MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); + pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? + if (resourcePlan == null) { + throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); + } + if (resourcePlan.getStatus() == MWMResourcePlan.Status.ACTIVE) { + throw new MetaException("Cannot drop an active resource plan"); + } + // First, drop all the dependencies. + resourcePlan.setDefaultPool(null); + pm.deletePersistentAll(resourcePlan.getTriggers()); + pm.deletePersistentAll(resourcePlan.getMappings()); + pm.deletePersistentAll(resourcePlan.getPools()); + pm.deletePersistent(resourcePlan); + } + + @Override + public void createWMTrigger(WMTrigger trigger) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException { + try { + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); + MWMTrigger mTrigger = new MWMTrigger(resourcePlan, + normalizeIdentifier(trigger.getTriggerName()), trigger.getTriggerExpression(), + trigger.getActionExpression(), null, + trigger.isSetIsInUnmanaged() && trigger.isIsInUnmanaged()); + pm.makePersistent(mTrigger); + } catch (Exception e) { + checkForConstraintException(e, "Trigger already exists, use alter: "); + throw e; + } + } + + @Override + public void alterWMTrigger(WMTrigger trigger) + throws NoSuchObjectException, InvalidOperationException, MetaException { + MWMResourcePlan resourcePlan = getMWMResourcePlan( + trigger.getResourcePlanName(), trigger.getNs(), true); + MWMTrigger mTrigger = getTrigger(resourcePlan, trigger.getTriggerName()); + // Update the object. + if (trigger.isSetTriggerExpression()) { + mTrigger.setTriggerExpression(trigger.getTriggerExpression()); + } + if (trigger.isSetActionExpression()) { + mTrigger.setActionExpression(trigger.getActionExpression()); + } + if (trigger.isSetIsInUnmanaged()) { + mTrigger.setIsInUnmanaged(trigger.isIsInUnmanaged()); + } + } + + private MWMTrigger getTrigger(MWMResourcePlan resourcePlan, String triggerName) + throws NoSuchObjectException { + triggerName = normalizeIdentifier(triggerName); + Query // Get the MWMTrigger object from DN + query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); + query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); + query.setUnique(true); + MWMTrigger mTrigger = (MWMTrigger) query.execute(resourcePlan, triggerName); + if (mTrigger == null) { + throw new NoSuchObjectException("Cannot find trigger with name: " + triggerName); + } + pm.retrieve(mTrigger); + return mTrigger; + } + + @Override + public void dropWMTrigger(String resourcePlanName, String triggerName, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException { + resourcePlanName = normalizeIdentifier(resourcePlanName); + triggerName = normalizeIdentifier(triggerName); + + Query query = null; + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); + query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp && name == triggerName"); + query.declareParameters("MWMResourcePlan rp, java.lang.String triggerName"); + if (query.deletePersistentAll(resourcePlan, triggerName) != 1) { + throw new NoSuchObjectException("Cannot delete trigger: " + triggerName); + } + } + + @Override + public List getTriggersForResourcePlan(String resourcePlanName, String ns) + throws NoSuchObjectException, MetaException { + List triggers = new ArrayList(); + Query query = null; + MWMResourcePlan resourcePlan; + try { + resourcePlan = getMWMResourcePlan(resourcePlanName, ns, false); + } catch (InvalidOperationException e) { + // Should not happen, edit check is false. + throw new RuntimeException(e); + } + query = pm.newQuery(MWMTrigger.class, "resourcePlan == rp"); + query.declareParameters("MWMResourcePlan rp"); + List mTriggers = (List) query.execute(resourcePlan); + pm.retrieveAll(mTriggers); + if (mTriggers != null) { + for (MWMTrigger trigger : mTriggers) { + triggers.add(fromMWMTrigger(trigger, resourcePlanName)); + } + } + return triggers; + } + + private WMTrigger fromMWMTrigger(MWMTrigger mTrigger, String resourcePlanName) { + WMTrigger trigger = new WMTrigger(); + trigger.setResourcePlanName(resourcePlanName); + trigger.setTriggerName(mTrigger.getName()); + trigger.setTriggerExpression(mTrigger.getTriggerExpression()); + trigger.setActionExpression(mTrigger.getActionExpression()); + trigger.setIsInUnmanaged(mTrigger.getIsInUnmanaged()); + trigger.setNs(mTrigger.getResourcePlan().getNs()); + return trigger; + } + + @Override + public void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, + InvalidOperationException, MetaException { + try { + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); + + if (!poolParentExists(resourcePlan, pool.getPoolPath())) { + throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); + } + String policy = pool.getSchedulingPolicy(); + if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) { + throw new InvalidOperationException("Invalid scheduling policy " + policy); + } + MWMPool mPool = new MWMPool(resourcePlan, pool.getPoolPath(), pool.getAllocFraction(), + pool.getQueryParallelism(), policy); + pm.makePersistent(mPool); + } catch (Exception e) { + checkForConstraintException(e, "Pool already exists: "); + throw e; + } + } + + @Override + public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, + NoSuchObjectException, InvalidOperationException, MetaException { + MWMResourcePlan resourcePlan = getMWMResourcePlan( + pool.getResourcePlanName(), pool.getNs(), true); + MWMPool mPool = getPool(resourcePlan, poolPath); + pm.retrieve(mPool); + if (pool.isSetAllocFraction()) { + mPool.setAllocFraction(pool.getAllocFraction()); + } + if (pool.isSetQueryParallelism()) { + mPool.setQueryParallelism(pool.getQueryParallelism()); + } + if (pool.isSetIsSetSchedulingPolicy() && pool.isIsSetSchedulingPolicy()) { + if (pool.isSetSchedulingPolicy()) { + String policy = pool.getSchedulingPolicy(); + if (!MetaStoreUtils.isValidSchedulingPolicy(policy)) { + throw new InvalidOperationException("Invalid scheduling policy " + policy); + } + mPool.setSchedulingPolicy(pool.getSchedulingPolicy()); + } else { + mPool.setSchedulingPolicy(null); + } + } + if (pool.isSetPoolPath() && !pool.getPoolPath().equals(mPool.getPath())) { + moveDescendents(resourcePlan, mPool.getPath(), pool.getPoolPath()); + mPool.setPath(pool.getPoolPath()); + } + } + + private MWMPool getPool(MWMResourcePlan resourcePlan, String poolPath) + throws NoSuchObjectException { + poolPath = normalizeIdentifier(poolPath); + Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path == poolPath"); + query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); + query.setUnique(true); + MWMPool mPool = (MWMPool) query.execute(resourcePlan, poolPath); + if (mPool == null) { + throw new NoSuchObjectException("Cannot find pool: " + poolPath); + } + pm.retrieve(mPool); + return mPool; + } + + private void moveDescendents(MWMResourcePlan resourcePlan, String path, String newPoolPath) + throws NoSuchObjectException { + if (!poolParentExists(resourcePlan, newPoolPath)) { + throw new NoSuchObjectException("Pool path is invalid, the parent does not exist"); + } + Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); + query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); + List descPools = (List) query.execute(resourcePlan, path + "."); + pm.retrieveAll(descPools); + for (MWMPool pool : descPools) { + pool.setPath(newPoolPath + pool.getPath().substring(path.length())); + } + } + + private String getParentPath(String poolPath, String defValue) { + int idx = poolPath.lastIndexOf('.'); + if (idx == -1) { + return defValue; + } + return poolPath.substring(0, idx); + } + + private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) { + String parent = getParentPath(poolPath, null); + if (parent == null) { + return true; + } + try { + getPool(resourcePlan, parent); + return true; + } catch (NoSuchObjectException e) { + return false; + } + } + + @Override + public void dropWMPool(String resourcePlanName, String poolPath, String ns) + throws NoSuchObjectException, InvalidOperationException, MetaException { + poolPath = normalizeIdentifier(poolPath); + boolean commited = false; + Query query = null; + try { + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); + if (resourcePlan.getDefaultPool() != null && + resourcePlan.getDefaultPool().getPath().equals(poolPath)) { + throw new InvalidOperationException("Cannot drop default pool of a resource plan"); + } + if (poolHasChildren(resourcePlan, poolPath)) { + throw new InvalidOperationException("Cannot drop a pool that has child pools"); + } + query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); + query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); + if (query.deletePersistentAll(resourcePlan, poolPath) != 1) { + throw new NoSuchObjectException("Cannot delete pool: " + poolPath); + } + } catch(Exception e) { + if (getConstraintException(e) != null) { + throw new InvalidOperationException("Please remove all mappings for this pool."); + } + throw e; + } + } + + private boolean poolHasChildren(MWMResourcePlan resourcePlan, String poolPath) { + Query query = pm.newQuery(MWMPool.class, "resourcePlan == rp && path.startsWith(poolPath)"); + query.declareParameters("MWMResourcePlan rp, java.lang.String poolPath"); + query.setResult("count(this)"); + query.setUnique(true); + Long count = (Long) query.execute(resourcePlan, poolPath + "."); + return count != null && count > 0; + } + + @Override + public void createOrUpdateWMMapping(WMMapping mapping, boolean update) + throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, + MetaException { + MWMMapping.EntityType entityType = MWMMapping.EntityType.valueOf(mapping.getEntityType().trim().toUpperCase()); + String entityName = normalizeIdentifier(mapping.getEntityName()); + Query query = null; + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); + MWMPool pool = null; + if (mapping.isSetPoolPath()) { + pool = getPool(resourcePlan, mapping.getPoolPath()); + } + if (!update) { + MWMMapping mMapping = new MWMMapping(resourcePlan, entityType, entityName, pool, + mapping.getOrdering()); + pm.makePersistent(mMapping); + } else { + query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type " + + "&& entityName == name"); + query.declareParameters( + "MWMResourcePlan rp, java.lang.String type, java.lang.String name"); + query.setUnique(true); + MWMMapping mMapping = (MWMMapping) query.execute( + resourcePlan, entityType.toString(), entityName); + mMapping.setPool(pool); + } + } + + @Override + public void dropWMMapping(WMMapping mapping) + throws NoSuchObjectException, InvalidOperationException, MetaException { + String entityType = mapping.getEntityType().trim().toUpperCase(); + String entityName = normalizeIdentifier(mapping.getEntityName()); + Query query = null; + MWMResourcePlan resourcePlan = getMWMResourcePlan( + mapping.getResourcePlanName(), mapping.getNs(), true); + query = pm.newQuery(MWMMapping.class, + "resourcePlan == rp && entityType == type && entityName == name"); + query.declareParameters("MWMResourcePlan rp, java.lang.String type, java.lang.String name"); + if (query.deletePersistentAll(resourcePlan, entityType, entityName) != 1) { + throw new NoSuchObjectException("Cannot delete mapping."); + } + } + + @Override + public void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, + String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, + InvalidOperationException, MetaException { + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); + MWMPool pool = getPool(resourcePlan, poolPath); + MWMTrigger trigger = getTrigger(resourcePlan, triggerName); + pool.getTriggers().add(trigger); + trigger.getPools().add(pool); + pm.makePersistent(pool); + pm.makePersistent(trigger); + } + + @Override + public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, + String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { + MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); + MWMPool pool = getPool(resourcePlan, poolPath); + MWMTrigger trigger = getTrigger(resourcePlan, triggerName); + pool.getTriggers().remove(trigger); + trigger.getPools().remove(pool); + pm.makePersistent(pool); + pm.makePersistent(trigger); + } + + @Override + public void addRuntimeStat(RuntimeStat stat) throws MetaException { + LOG.debug("runtimeStat: {}", stat); + MRuntimeStat mStat = MRuntimeStat.fromThrift(stat); + pm.makePersistent(mStat); + } + + @Override + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + if (maxRetainSecs < 0) { + LOG.warn("runtime stats retention is disabled"); + return 0; + } + int maxCreateTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs; + Query q = pm.newQuery(MRuntimeStat.class); + q.setFilter("createTime <= maxCreateTime"); + q.declareParameters("int maxCreateTime"); + long deleted = q.deletePersistentAll(maxCreateTime); + return (int) deleted; + } + + @Override + public List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { + return getMRuntimeStats(maxEntries, maxCreateTime); + } + + private List getMRuntimeStats(int maxEntries, int maxCreateTime) { + try (QueryWrapper query = new QueryWrapper(pm.newQuery(MRuntimeStat.class))) { + query.setOrdering("createTime descending"); + if (maxCreateTime > 0) { + query.setFilter("createTime < " + maxCreateTime); + } + if (maxEntries < 0) { + maxEntries = Integer.MAX_VALUE; + } + List ret = new ArrayList<>(); + List res = (List) query.execute(); + int totalEntries = 0; + for (MRuntimeStat mRuntimeStat : res) { + pm.retrieve(mRuntimeStat); + totalEntries += mRuntimeStat.getWeight(); + ret.add(MRuntimeStat.toThrift(mRuntimeStat)); + if (totalEntries >= maxEntries) { + break; + } + } + return ret; + } + } + + private Configuration conf; + @Override + public void setBaseStore(RawStore store) { + super.setBaseStore(store); + this.conf = store.getConf(); + } +} From 41fdf66db841b0ffaf510f463d115ba8dc70eada Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 13:40:05 +0800 Subject: [PATCH 12/22] ut --- .../org/apache/hadoop/hive/metastore/TestHiveMetaStore.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 3153cd1a3942..12a31afd1a2d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -3611,8 +3611,7 @@ public void testValidateTableCols() throws Throwable { List expectedCols = Lists.newArrayList(); expectedCols.add("name"); - ObjectStore objStore = new ObjectStore(); - ColStatsStoreImpl impl = (ColStatsStoreImpl) objStore.unwrap(ColStatsStore.class); + ColStatsStoreImpl impl = new ColStatsStoreImpl(); try { impl.validateTableCols(tbl, expectedCols); } catch (MetaException ex) { From a2f77a97e84ec5f1bead1baff05bb98639c5d5e6 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 16:53:41 +0800 Subject: [PATCH 13/22] reformat --- .../metastore/impl/ColStatsStoreImpl.java | 27 +++++++++++-------- .../metastore/impl/ConstraintStoreImpl.java | 25 ++++++++++------- .../metastore/impl/NotificationStoreImpl.java | 3 ++- .../metastore/impl/PrivilegeStoreImpl.java | 6 +++-- .../metastore/impl/TableStoreImpl.java | 27 ++++++++++++------- 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index c6c0a80b4377..9c75c8dbf6f7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -276,8 +276,8 @@ public Map updateTableColumnStatistics(ColumnStatistics colStats int maxRetries = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_MAX_RETRIES); long sleepInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.METASTORE_S4U_NOWAIT_RETRY_SLEEP_INTERVAL, TimeUnit.MILLISECONDS); - GetHelper transactionHelper = createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), - statsDesc.getTableName())); + GetHelper transactionHelper = + createSubTransactionHelper(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName())); Map result = new RetryingExecutor<>(maxRetries, () -> { AtomicReference exceptionRef = new AtomicReference<>(); String savePoint = "uts_" + ThreadLocalRandom.current().nextInt(10000) + "_" + System.nanoTime(); @@ -1003,7 +1003,8 @@ public List run(List input) throws Exception { } else { filter = "t1.contains(partition.partitionName) && partition.table.database.name == t2 && partition.table.tableName == t3 && " + "partition.table.database.catalogName == t4" + (engine != null ? " && engine == t5" : ""); - parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + (engine != null ? ", java.lang.String t5" : ""); + parameters = "java.util.Collection t1, java.lang.String t2, java.lang.String t3, java.lang.String t4" + + (engine != null ? ", java.lang.String t5" : ""); } query.setFilter(filter); query.declareParameters(parameters); @@ -1036,8 +1037,8 @@ public List run(List input) throws Exception { Batchable.runBatched(batchSize, partNames, new Batchable() { @Override public List run(List input) throws MetaException { - Pair> queryWithParams = getPartQueryWithParams(pm, catalog, database, tableName, - input); + Pair> queryWithParams = + getPartQueryWithParams(pm, catalog, database, tableName, input); try (QueryWrapper qw = new QueryWrapper(queryWithParams.getLeft())) { qw.setResultClass(MPartition.class); qw.setClass(MPartition.class); @@ -1086,8 +1087,8 @@ protected Boolean getJdoResult() }.run(true); } - private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, - List colNames, String engine) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, List colNames, String engine) + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { String catName = normalizeIdentifier(tblName.getCat()); String dbName = normalizeIdentifier(tblName.getDb()); String tableName = normalizeIdentifier(tblName.getTable()); @@ -1098,11 +1099,15 @@ private boolean deleteTableColumnStatisticsViaJdo(TableName tblName, String filter; String parameters; if (colNames != null && !colNames.isEmpty()) { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + (engine != null ? " && engine == t5" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + (engine != null ? ", java.lang.String t5" : ""); + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3 && t4.contains(colName)" + + (engine != null ? " && engine == t5" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3, java.util.Collection t4" + + (engine != null ? ", java.lang.String t5" : ""); } else { - filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + (engine != null ? " && engine == t4" : ""); - parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + (engine != null ? ", java.lang.String t4" : ""); + filter = "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3" + + (engine != null ? " && engine == t4" : ""); + parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3" + + (engine != null ? ", java.lang.String t4" : ""); } query.setFilter(filter); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java index d45f9c76fd53..8b4416643df1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -431,7 +431,8 @@ public List addPrimaryKeys(List pks) throws Invali return addPrimaryKeys(pks, true); } - private List addPrimaryKeys(List pks, boolean retrieveCD) throws InvalidObjectException, + private List addPrimaryKeys(List pks, boolean retrieveCD) + throws InvalidObjectException, MetaException { List mpks = new ArrayList<>(); String constraintName = null; @@ -452,7 +453,8 @@ private List addPrimaryKeys(List pks, boolean retr } MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + int parentIntegerIndex = + getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); if (parentIntegerIndex == -1) { if (parentTable.getPartitionKeys() != null) { parentCD = null; @@ -536,7 +538,8 @@ private List addUniqueConstraints(List } MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + int parentIntegerIndex = + getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); if (parentIntegerIndex == -1) { if (parentTable.getPartitionKeys() != null) { parentCD = null; @@ -637,8 +640,8 @@ private List addCheckConstraints(List cc return ccs; } - private MConstraint addConstraint(String catName, String tableDB, String tableName, String columnName, String ccName, - boolean isEnable, boolean isRely, boolean isValidate, int constraintType, + private MConstraint addConstraint(String catName, String tableDB, String tableName, String columnName, + String ccName, boolean isEnable, boolean isRely, boolean isValidate, int constraintType, String constraintValue, boolean retrieveCD) throws InvalidObjectException, MetaException { String constraintName = null; @@ -652,7 +655,8 @@ private MConstraint addConstraint(String catName, String tableDB, String tableNa } MColumnDescriptor parentCD = retrieveCD ? nParentTable.mcd : parentTable.getSd().getCD(); - int parentIntegerIndex = getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); + int parentIntegerIndex = + getColumnIndexFromTableColumns(parentCD == null ? null : parentCD.getCols(), columnName); if (parentIntegerIndex == -1) { if (parentTable.getPartitionKeys() != null) { parentCD = null; @@ -704,7 +708,8 @@ private List addDefaultConstraints(List addNotNullConstraints(List getForeignKeys(ForeignKeysRequest request) throws Met private List getForeignKeysInternal( final String catName, final String parent_db_name_input, final String parent_tbl_name_input, - final String foreign_db_name_input, final String foreign_tbl_name_input) throws MetaException, NoSuchObjectException { + final String foreign_db_name_input, final String foreign_tbl_name_input) + throws MetaException, NoSuchObjectException { final String parent_db_name = (parent_db_name_input != null) ? normalizeIdentifier(parent_db_name_input) : null; final String parent_tbl_name = (parent_tbl_name_input != null) ? normalizeIdentifier(parent_tbl_name_input) : null; final String foreign_db_name = (foreign_db_name_input != null) ? normalizeIdentifier(foreign_db_name_input) : null; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java index db6c606f1826..8c542fc661cb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java @@ -264,7 +264,8 @@ private void cleanOlderEvents(int olderThan, Class table, String tableName) { TimeUnit.NANOSECONDS.toMillis(finish - start)); } - private int doCleanNotificationEvents(final int ageSec, final Optional batchSize, Class tableClass, String tableName) { + private int doCleanNotificationEvents(final int ageSec, final Optional batchSize, + Class tableClass, String tableName) { int eventsCount = 0; Query query = pm.newQuery(tableClass, "eventTime <= tooOld"); String key = null; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java index ea142b5ec980..513b72e8b4e4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java @@ -336,7 +336,8 @@ private List listMSecurityPrincipalMembershipRole(final String roleNam LOG.debug("Executing listMSecurityPrincipalMembershipRole"); Query query = pm.newQuery(MRoleMap.class, "principalName == t1 && principalType == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); - final List mRoleMemebership = (List) query.execute(roleName, principalType.toString()); + final List mRoleMemebership = + (List) query.execute(roleName, principalType.toString()); LOG.debug("Retrieving all objects for listMSecurityPrincipalMembershipRole"); pm.retrieveAll(mRoleMemebership); @@ -1258,7 +1259,8 @@ public boolean refreshPrivileges(HiveObjectRef objToRefresh, String authorizer, } break; case TABLE: - grants = listTableGrantsAll(new TableName(catName, objToRefresh.getDbName(), objToRefresh.getObjectName()), authorizer); + grants = listTableGrantsAll(new TableName(catName, objToRefresh.getDbName(), + objToRefresh.getObjectName()), authorizer); break; case COLUMN: Preconditions.checkArgument(objToRefresh.getColumnName()==null, "columnName must be null"); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index 6543efb8ace7..e23060554813 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -661,7 +661,8 @@ private MCreationMetadata getCreationMetadata(String catName, String dbName, Str } @Override - public boolean addPartitions(TableName tableName, List parts) throws InvalidObjectException, MetaException { + public boolean addPartitions(TableName tableName, List parts) + throws InvalidObjectException, MetaException { String catName = normalizeIdentifier(tableName.getCat()); String dbName = normalizeIdentifier(tableName.getDb()); String tblName = normalizeIdentifier(tableName.getTable()); @@ -845,7 +846,8 @@ public Partition getPartition(TableName tabName, List part_vals, String * @param part_vals The values defining the partition * @return The MPartition object in the backend database */ - private MPartition getMPartition(String catName, String dbName, String tableName, List part_vals, MTable mtbl) + private MPartition getMPartition(String catName, String dbName, String tableName, + List part_vals, MTable mtbl) throws MetaException { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); @@ -1011,7 +1013,8 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { // If we find it, we will change the reference for the CD. // If we do not find it, i.e., the column will be deleted, we do not change it // and we let the logic in removeUnusedColumnDescriptor take care of it - try (QueryWrapper query = new QueryWrapper(pm.newQuery(MConstraint.class, "parentColumn == inCD || childColumn == inCD"))) { + try (QueryWrapper query = new QueryWrapper(pm.newQuery(MConstraint.class, + "parentColumn == inCD || childColumn == inCD"))) { query.declareParameters("MColumnDescriptor inCD"); List mConstraintsList = (List) query.execute(oldSd.getCD()); pm.retrieveAll(mConstraintsList); @@ -1212,7 +1215,8 @@ public List
getTableObjectsByName(String catName, String db, List } else { if (projectionFields.size() > 1) { // Execute the query to fetch the partial results. - List results = (List) query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()])); + List results = (List) + query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()])); // Declare the tables array to return the list of tables mtables = new ArrayList<>(results.size()); // Iterate through each row of the result and create the MTable object. @@ -1227,7 +1231,8 @@ public List
getTableObjectsByName(String catName, String db, List } } else if (projectionFields.size() == 1) { // Execute the query to fetch the partial results. - List results = (List) query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()])); + List results = (List) + query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()])); // Iterate through each row of the result and create the MTable object. mtables = new ArrayList<>(results.size()); for (Object row : results) { @@ -1588,7 +1593,8 @@ private List getPartNamesPrunedByExpr(Table table, boolean isJdoQuery) t } @Override protected List getSqlResult() throws MetaException { - MetaStoreDirectSql.SqlFilterForPushdown filter = new MetaStoreDirectSql.SqlFilterForPushdown(getTable(), false); + MetaStoreDirectSql.SqlFilterForPushdown filter = + new MetaStoreDirectSql.SqlFilterForPushdown(getTable(), false); List partNames = null; Table table = getTable(); if (exprTree != null) { @@ -1643,7 +1649,8 @@ public boolean getPartitionsByExpr(TableName tableName, List result, assert result != null; byte[] expr = args.getExpr(); final ExpressionTree exprTree = expr.length != 0 ? PartFilterExprUtil.makeExpressionTree( - expressionProxy, expr, getDefaultPartitionName(args.getDefaultPartName()), baseStore.getConf()) : ExpressionTree.EMPTY_TREE; + expressionProxy, expr, + getDefaultPartitionName(args.getDefaultPartName()), baseStore.getConf()) : ExpressionTree.EMPTY_TREE; final AtomicBoolean hasUnknownPartitions = new AtomicBoolean(false); String catName = normalizeIdentifier(tableName.getCat()); @@ -2059,7 +2066,8 @@ protected List getJdoResult() throws MetaException, NoSuchObjectExcep } @Override - public List getPartitionSpecsByFilterAndProjection(Table table, GetProjectionsSpec partitionsProjectSpec, + public List getPartitionSpecsByFilterAndProjection(Table table, + GetProjectionsSpec partitionsProjectSpec, GetPartitionsFilterSpec filterSpec) throws MetaException, NoSuchObjectException { List fieldList = null; String inputIncludePattern = null; @@ -2971,7 +2979,8 @@ private Table convertToTable(MTable mtbl, Configuration conf) throws MetaExcepti tableType = TableType.MANAGED_TABLE.toString(); } } else { - if (tableType.equals(TableType.VIRTUAL_VIEW.toString()) || tableType.equals(TableType.MATERIALIZED_VIEW.toString())) { + if (tableType.equals(TableType.VIRTUAL_VIEW.toString()) || + tableType.equals(TableType.MATERIALIZED_VIEW.toString())) { viewOriginalText = mtbl.getViewOriginalText(); viewExpandedText = mtbl.getViewExpandedText(); } From ee13974b36e9fcecdfa389fb13c98446b61917cc Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 16:59:31 +0800 Subject: [PATCH 14/22] review-1 --- .../metastore/impl/ColStatsStoreImpl.java | 6 +++--- .../metastore/impl/TableStoreImpl.java | 2 +- .../metastore/metastore/impl/WLMStoreImpl.java | 18 ++++++++++++------ .../hive/metastore/TestHiveMetaStore.java | 1 - .../hive/metastore/VerifyingObjectStore.java | 5 ++--- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index 9c75c8dbf6f7..f07cfe4488ec 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -199,8 +199,8 @@ protected List getJdoResult() throws MetaException { } Query query = pm.newQuery(MTable.class, whereStr); query.declareParameters(paramStr); - Collection tbls = (Collection) query.executeWithArray( - query, MetaStoreDirectSql.STATS_TABLE_TYPES); + Collection tbls = + (Collection) query.executeWithArray(MetaStoreDirectSql.STATS_TABLE_TYPES); pm.retrieveAll(tbls); for (MTable tbl : tbls) { result.add(new TableName( @@ -916,7 +916,7 @@ public void deleteAllPartitionColumnStatistics(TableName tn, String writeIdList) query.setFilter(filter); query.declareParameters(parameters); - Long number = query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), + query.deletePersistentAll(normalizeIdentifier(dbName), normalizeIdentifier(tableName), normalizeIdentifier(catName)); new GetHelper(this, tn) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index e23060554813..352c9078a44c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -1061,7 +1061,7 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { @Override public void createTable(Table tbl) throws InvalidObjectException, MetaException { - MTable mtbl = convertToMTable(tbl);; + MTable mtbl = convertToMTable(tbl); if (TxnUtils.isTransactionalTable(tbl)) { mtbl.setWriteId(tbl.getWriteId()); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java index 8c8682c44e0a..22b8aece8eb1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java @@ -130,7 +130,7 @@ public void createResourcePlan( } catch (InvalidOperationException e) { throw new RuntimeException(e); } catch (Exception e) { - checkForConstraintException(e, "Resource plan already exists: "); + checkForConstraintException(e, "Resource plan already exists: " + rpName); throw e; } } @@ -284,7 +284,9 @@ private MWMResourcePlan getMWMResourcePlan(String name, String ns, boolean editC Query query = createGetResourcePlanQuery(); ns = getNsOrDefault(ns); resourcePlan = (MWMResourcePlan) query.execute(name, ns); - pm.retrieve(resourcePlan); + if (resourcePlan != null) { + pm.retrieve(resourcePlan); + } if (mustExist && resourcePlan == null) { throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } @@ -480,7 +482,9 @@ private MWMResourcePlan getActiveMWMResourcePlan(String ns) { Query query = createActivePlanQuery(); result = (MWMResourcePlan) query.execute( MWMResourcePlan.Status.ACTIVE.toString(), getNsOrDefault(ns)); - pm.retrieve(result); + if (result != null) { + pm.retrieve(result); + } return result; } @@ -649,7 +653,7 @@ private WMValidateResourcePlanResponse getResourcePlanErrors(MWMResourcePlan mRe public WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { name = normalizeIdentifier(name); - Query query = createGetResourcePlanQuery(); + Query query = createGetResourcePlanQuery(); MWMResourcePlan mResourcePlan = (MWMResourcePlan) query.execute(name, ns); if (mResourcePlan == null) { throw new NoSuchObjectException("Cannot find resourcePlan: " + name + " in " + ns); @@ -663,10 +667,10 @@ public void dropResourcePlan(String name, String ns) throws NoSuchObjectExceptio name = normalizeIdentifier(name); Query query = createGetResourcePlanQuery(); MWMResourcePlan resourcePlan = (MWMResourcePlan) query.execute(name, getNsOrDefault(ns)); - pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? if (resourcePlan == null) { throw new NoSuchObjectException("There is no resource plan named: " + name + " in " + ns); } + pm.retrieve(resourcePlan); // TODO: why do some codepaths call retrieve and some don't? if (resourcePlan.getStatus() == MWMResourcePlan.Status.ACTIVE) { throw new MetaException("Cannot drop an active resource plan"); } @@ -885,7 +889,6 @@ private boolean poolParentExists(MWMResourcePlan resourcePlan, String poolPath) public void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { poolPath = normalizeIdentifier(poolPath); - boolean commited = false; Query query = null; try { MWMResourcePlan resourcePlan = getMWMResourcePlan(resourcePlanName, ns, true); @@ -943,6 +946,9 @@ public void createOrUpdateWMMapping(WMMapping mapping, boolean update) query.setUnique(true); MWMMapping mMapping = (MWMMapping) query.execute( resourcePlan, entityType.toString(), entityName); + if (mMapping == null) { + throw new NoSuchObjectException("Cannot find mapping for " + entityType + ":" + entityName); + } mMapping.setPool(pool); } } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 12a31afd1a2d..08f479542fce 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -61,7 +61,6 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.dataconnector.jdbc.AbstractJDBCConnectorProvider; import org.apache.hadoop.hive.metastore.handler.AddPartitionsHandler; -import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.metastore.impl.ColStatsStoreImpl; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java index fd01623d9558..8bd1306fc5a3 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; -import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.utils.DirectSqlConfigurator; import org.slf4j.Logger; @@ -149,9 +148,9 @@ public List alterPartitions(String catName, String dbName, String tbl newParts.forEach(newPart -> newPart.setWriteId(writeId)); } TableName tableName = new TableName(catName, dbName, tblName); - Table table = unwrap(TableStore.class).getTable(tableName, queryWriteIdList, writeId); + Table table = unwrap(TableStore.class).getTable(tableName, queryWriteIdList, -1); if (table == null) { - throw new NoSuchObjectException("Table " + table + " doesn't exist"); + throw new NoSuchObjectException("Table " + tableName + " doesn't exist"); } List partCols = table.getPartitionKeys(); List partNames = new ArrayList<>(); From f468f6206d3fee6c1f6da707bdc04272778fe7cb Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 17:46:06 +0800 Subject: [PATCH 15/22] token store --- .../hadoop/hive/metastore/ObjectStore.java | 191 ------------------ .../hadoop/hive/metastore/RawStore.java | 35 +++- .../hive/metastore/cache/CachedStore.java | 32 --- .../metastore/iface/ConstraintStore.java | 1 - .../metastore/metastore/iface/TokenStore.java | 46 +++++ .../metastore/metastore/iface/WLMStore.java | 1 - .../metastore/impl/TokenStoreImpl.java | 147 ++++++++++++++ 7 files changed, 219 insertions(+), 234 deletions(-) create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TokenStore.java create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index a272f12a79a7..ec4e4ad32cc3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -126,11 +126,9 @@ import org.apache.hadoop.hive.metastore.model.MDataConnector; import org.apache.hadoop.hive.metastore.model.MDCPrivilege; import org.apache.hadoop.hive.metastore.model.MDatabase; -import org.apache.hadoop.hive.metastore.model.MDelegationToken; import org.apache.hadoop.hive.metastore.model.MFieldSchema; import org.apache.hadoop.hive.metastore.model.MFunction; import org.apache.hadoop.hive.metastore.model.MISchema; -import org.apache.hadoop.hive.metastore.model.MMasterKey; import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties; import org.apache.hadoop.hive.metastore.model.MPackage; import org.apache.hadoop.hive.metastore.model.MPartitionEvent; @@ -1990,195 +1988,6 @@ public long cleanupEvents() { return delCnt; } - private MDelegationToken getTokenFrom(String tokenId) { - try (QueryWrapper query = new QueryWrapper(pm.newQuery(MDelegationToken.class, "tokenIdentifier == tokenId"))) { - query.declareParameters("java.lang.String tokenId"); - query.setUnique(true); - MDelegationToken delegationToken = (MDelegationToken) query.execute(tokenId); - return delegationToken; - } - } - - @Override - public boolean addToken(String tokenId, String delegationToken) { - - LOG.debug("Begin executing addToken"); - boolean committed = false; - MDelegationToken token; - try{ - openTransaction(); - token = getTokenFrom(tokenId); - if (token == null) { - // add Token, only if it already doesn't exist - pm.makePersistent(new MDelegationToken(tokenId, delegationToken)); - } - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, null); - } - LOG.debug("Done executing addToken with status : {}", committed); - return committed && (token == null); - } - - @Override - public boolean removeToken(String tokenId) { - - LOG.debug("Begin executing removeToken"); - boolean committed = false; - MDelegationToken token; - try{ - openTransaction(); - token = getTokenFrom(tokenId); - if (null != token) { - pm.deletePersistent(token); - } - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, null); - } - LOG.debug("Done executing removeToken with status : {}", committed); - return committed && (token != null); - } - - @Override - public String getToken(String tokenId) { - - LOG.debug("Begin executing getToken"); - boolean committed = false; - MDelegationToken token; - try{ - openTransaction(); - token = getTokenFrom(tokenId); - if (null != token) { - pm.retrieve(token); - } - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, null); - } - LOG.debug("Done executing getToken with status : {}", committed); - return (null == token) ? null : token.getTokenStr(); - } - - @Override - public List getAllTokenIdentifiers() { - LOG.debug("Begin executing getAllTokenIdentifiers"); - boolean committed = false; - Query query = null; - List tokenIdents = new ArrayList<>(); - - try { - openTransaction(); - query = pm.newQuery(MDelegationToken.class); - List tokens = (List) query.execute(); - pm.retrieveAll(tokens); - committed = commitTransaction(); - - for (MDelegationToken token : tokens) { - tokenIdents.add(token.getTokenIdentifier()); - } - return tokenIdents; - } finally { - LOG.debug("Done executing getAllTokenIdentifers with status : {}", committed); - rollbackAndCleanup(committed, query); - } - } - - @Override - public int addMasterKey(String key) throws MetaException{ - LOG.debug("Begin executing addMasterKey"); - boolean committed = false; - MMasterKey masterKey = new MMasterKey(key); - try{ - openTransaction(); - pm.makePersistent(masterKey); - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, null); - } - LOG.debug("Done executing addMasterKey with status : {}", committed); - if (committed) { - return ((IntIdentity)pm.getObjectId(masterKey)).getKey(); - } else { - throw new MetaException("Failed to add master key."); - } - } - - @Override - public void updateMasterKey(Integer id, String key) throws NoSuchObjectException, MetaException { - LOG.debug("Begin executing updateMasterKey"); - boolean committed = false; - Query query = null; - MMasterKey masterKey; - try { - openTransaction(); - query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey) query.execute(id); - if (null != masterKey) { - masterKey.setMasterKey(key); - } - committed = commitTransaction(); - } finally { - rollbackAndCleanup(committed, query); - } - LOG.debug("Done executing updateMasterKey with status : {}", committed); - if (null == masterKey) { - throw new NoSuchObjectException("No key found with keyId: " + id); - } - if (!committed) { - throw new MetaException("Though key is found, failed to update it. " + id); - } - } - - @Override - public boolean removeMasterKey(Integer id) { - LOG.debug("Begin executing removeMasterKey"); - boolean success = false; - Query query = null; - MMasterKey masterKey; - try { - openTransaction(); - query = pm.newQuery(MMasterKey.class, "keyId == id"); - query.declareParameters("java.lang.Integer id"); - query.setUnique(true); - masterKey = (MMasterKey) query.execute(id); - if (null != masterKey) { - pm.deletePersistent(masterKey); - } - success = commitTransaction(); - } finally { - rollbackAndCleanup(success, query); - } - LOG.debug("Done executing removeMasterKey with status : {}", success); - return (null != masterKey) && success; - } - - @Override - public String[] getMasterKeys() { - LOG.debug("Begin executing getMasterKeys"); - boolean committed = false; - Query query = null; - List keys; - try { - openTransaction(); - query = pm.newQuery(MMasterKey.class); - keys = (List) query.execute(); - pm.retrieveAll(keys); - committed = commitTransaction(); - - String[] masterKeys = new String[keys.size()]; - for (int i = 0; i < keys.size(); i++) { - masterKeys[i] = keys.get(i).getMasterKey(); - } - return masterKeys; - } finally { - LOG.debug("Done executing getMasterKeys with status : {}", committed); - rollbackAndCleanup(committed, query); - } - } - // compare hive version and metastore version @Override public void verifySchema() throws MetaException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 8cfa9cfa2307..b843e2f8030e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -121,6 +121,7 @@ import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; +import org.apache.hadoop.hive.metastore.metastore.iface.TokenStore; import org.apache.hadoop.hive.metastore.metastore.iface.WLMStore; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MPartition; @@ -1478,22 +1479,38 @@ default boolean deleteTableColumnStatistics(String catName, String dbName, Strin long cleanupEvents(); - boolean addToken(String tokenIdentifier, String delegationToken); + default boolean addToken(String tokenIdentifier, String delegationToken) { + return unwrap(TokenStore.class).addToken(tokenIdentifier, delegationToken); + } - boolean removeToken(String tokenIdentifier); + default boolean removeToken(String tokenIdentifier) { + return unwrap(TokenStore.class).removeToken(tokenIdentifier); + } - String getToken(String tokenIdentifier); + default String getToken(String tokenIdentifier) { + return unwrap(TokenStore.class).getToken(tokenIdentifier); + } - List getAllTokenIdentifiers(); + default List getAllTokenIdentifiers() { + return unwrap(TokenStore.class).getAllTokenIdentifiers(); + } - int addMasterKey(String key) throws MetaException; + default int addMasterKey(String key) throws MetaException { + return unwrap(TokenStore.class).addMasterKey(key); + } - void updateMasterKey(Integer seqNo, String key) - throws NoSuchObjectException, MetaException; + default void updateMasterKey(Integer seqNo, String key) + throws NoSuchObjectException, MetaException { + unwrap(TokenStore.class).updateMasterKey(seqNo, key); + } - boolean removeMasterKey(Integer keySeq); + default boolean removeMasterKey(Integer keySeq) { + return unwrap(TokenStore.class).removeMasterKey(keySeq); + } - String[] getMasterKeys(); + default String[] getMasterKeys() { + return unwrap(TokenStore.class).getMasterKeys(); + } void verifySchema() throws MetaException; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 134cb69c3141..9a9c6356ad31 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2239,38 +2239,6 @@ long getPartsFound() { return rawStore.cleanupEvents(); } - @Override public boolean addToken(String tokenIdentifier, String delegationToken) { - return rawStore.addToken(tokenIdentifier, delegationToken); - } - - @Override public boolean removeToken(String tokenIdentifier) { - return rawStore.removeToken(tokenIdentifier); - } - - @Override public String getToken(String tokenIdentifier) { - return rawStore.getToken(tokenIdentifier); - } - - @Override public List getAllTokenIdentifiers() { - return rawStore.getAllTokenIdentifiers(); - } - - @Override public int addMasterKey(String key) throws MetaException { - return rawStore.addMasterKey(key); - } - - @Override public void updateMasterKey(Integer seqNo, String key) throws NoSuchObjectException, MetaException { - rawStore.updateMasterKey(seqNo, key); - } - - @Override public boolean removeMasterKey(Integer keySeq) { - return rawStore.removeMasterKey(keySeq); - } - - @Override public String[] getMasterKeys() { - return rawStore.getMasterKeys(); - } - @Override public void verifySchema() throws MetaException { rawStore.verifySchema(); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java index 018139235d43..4a7cb274e56b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/ConstraintStore.java @@ -193,5 +193,4 @@ List addDefaultConstraints(List dv) * @throws MetaException error accessing the RDBMS */ List addCheckConstraints(List cc) throws InvalidObjectException, MetaException; - } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TokenStore.java new file mode 100644 index 000000000000..92d646fa6f60 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/TokenStore.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.metastore.iface; + +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; +import org.apache.hadoop.hive.metastore.metastore.impl.TokenStoreImpl; + +@MetaDescriptor(alias = "token", defaultImpl = TokenStoreImpl.class) +public interface TokenStore { + boolean addToken(String tokenIdentifier, String delegationToken); + + boolean removeToken(String tokenIdentifier); + + String getToken(String tokenIdentifier); + + List getAllTokenIdentifiers(); + + int addMasterKey(String key) throws MetaException; + + void updateMasterKey(Integer seqNo, String key) + throws NoSuchObjectException, MetaException; + + boolean removeMasterKey(Integer keySeq); + + String[] getMasterKeys(); +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java index ca9e7a3c3c89..e0eb9f0913ab 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java @@ -103,5 +103,4 @@ void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, Str /** Removes outdated statistics. */ int deleteRuntimeStats(int maxRetainSecs) throws MetaException; - } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java new file mode 100644 index 000000000000..ba2fa99a25c2 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.metastore.impl; + +import javax.jdo.Query; +import javax.jdo.identity.IntIdentity; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.metastore.QueryWrapper; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.iface.TokenStore; +import org.apache.hadoop.hive.metastore.model.MDelegationToken; +import org.apache.hadoop.hive.metastore.model.MMasterKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TokenStoreImpl extends RawStoreAware implements TokenStore { + private static final Logger LOG = LoggerFactory.getLogger(TokenStoreImpl.class); + + private MDelegationToken getTokenFrom(String tokenId) { + try (QueryWrapper query = + new QueryWrapper(pm.newQuery(MDelegationToken.class, "tokenIdentifier == tokenId"))) { + query.declareParameters("java.lang.String tokenId"); + query.setUnique(true); + MDelegationToken delegationToken = (MDelegationToken) query.execute(tokenId); + return delegationToken; + } + } + + @Override + public boolean addToken(String tokenId, String delegationToken) { + LOG.debug("Begin executing addToken"); + MDelegationToken token = getTokenFrom(tokenId); + if (token == null) { + // add Token, only if it already doesn't exist + pm.makePersistent(new MDelegationToken(tokenId, delegationToken)); + } + LOG.debug("Done executing addToken with status"); + return token == null; + } + + @Override + public boolean removeToken(String tokenId) { + LOG.debug("Begin executing removeToken"); + MDelegationToken token = getTokenFrom(tokenId); + if (null != token) { + pm.deletePersistent(token); + } + return token != null; + } + + @Override + public String getToken(String tokenId) { + LOG.debug("Begin executing getToken"); + MDelegationToken token = getTokenFrom(tokenId); + if (null != token) { + pm.retrieve(token); + } + return (null == token) ? null : token.getTokenStr(); + } + + @Override + public List getAllTokenIdentifiers() { + LOG.debug("Begin executing getAllTokenIdentifiers"); + List tokenIdents = new ArrayList<>(); + Query query = pm.newQuery(MDelegationToken.class); + List tokens = (List) query.execute(); + pm.retrieveAll(tokens); + + for (MDelegationToken token : tokens) { + tokenIdents.add(token.getTokenIdentifier()); + } + return tokenIdents; + } + + @Override + public int addMasterKey(String key) throws MetaException { + LOG.debug("Begin executing addMasterKey"); + MMasterKey masterKey = new MMasterKey(key); + pm.makePersistent(masterKey); + return ((IntIdentity)pm.getObjectId(masterKey)).getKey(); + } + + @Override + public void updateMasterKey(Integer id, String key) throws NoSuchObjectException, MetaException { + LOG.debug("Begin executing updateMasterKey"); + MMasterKey masterKey; + Query query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + masterKey.setMasterKey(key); + } + if (null == masterKey) { + throw new NoSuchObjectException("No key found with keyId: " + id); + } + } + + @Override + public boolean removeMasterKey(Integer id) { + LOG.debug("Begin executing removeMasterKey"); + MMasterKey masterKey; + Query query = pm.newQuery(MMasterKey.class, "keyId == id"); + query.declareParameters("java.lang.Integer id"); + query.setUnique(true); + masterKey = (MMasterKey) query.execute(id); + if (null != masterKey) { + pm.deletePersistent(masterKey); + } + return (null != masterKey); + } + + @Override + public String[] getMasterKeys() { + LOG.debug("Begin executing getMasterKeys"); + List keys; + Query query = pm.newQuery(MMasterKey.class); + keys = (List) query.execute(); + pm.retrieveAll(keys); + + String[] masterKeys = new String[keys.size()]; + for (int i = 0; i < keys.size(); i++) { + masterKeys[i] = keys.get(i).getMasterKey(); + } + return masterKeys; + } +} From 9abda2bd1f4de1becd2f6257e25ea3fe213ed775 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 20:57:55 +0800 Subject: [PATCH 16/22] fix --- .../hadoop/hive/metastore/ObjectStore.java | 2 +- .../hadoop/hive/metastore/RawStore.java | 44 +++++++++---------- .../iface/{WLMStore.java => WMStore.java} | 8 ++-- .../{WLMStoreImpl.java => WMStoreImpl.java} | 6 +-- 4 files changed, 30 insertions(+), 30 deletions(-) rename standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/{WLMStore.java => WMStore.java} (96%) rename standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/{WLMStoreImpl.java => WMStoreImpl.java} (99%) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ec4e4ad32cc3..962211a1fde4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -214,7 +214,7 @@ private enum TXN_STATUS { private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; private Counter directSqlErrors; private PropertyStore propertyStore; - private Map, Object> cachedImpls = new HashMap<>(); + private final Map, Object> cachedImpls = new HashMap<>(); public ObjectStore() { } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index b843e2f8030e..4aa06b30c556 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -122,7 +122,7 @@ import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; import org.apache.hadoop.hive.metastore.metastore.iface.TokenStore; -import org.apache.hadoop.hive.metastore.metastore.iface.WLMStore; +import org.apache.hadoop.hive.metastore.metastore.iface.WMStore; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MTable; @@ -2121,93 +2121,93 @@ default List addCheckConstraints(List cc default void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException { - unwrap(WLMStore.class).createResourcePlan(resourcePlan, copyFrom, defaultPoolSize); + unwrap(WMStore.class).createResourcePlan(resourcePlan, copyFrom, defaultPoolSize); } default WMFullResourcePlan getResourcePlan(String name, String string) throws NoSuchObjectException, MetaException { - return unwrap(WLMStore.class).getResourcePlan(name, string); + return unwrap(WMStore.class).getResourcePlan(name, string); } default List getAllResourcePlans(String string) throws MetaException { - return unwrap(WLMStore.class).getAllResourcePlans(string); + return unwrap(WMStore.class).getAllResourcePlans(string); } default WMFullResourcePlan alterResourcePlan(String name, String ns, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - return unwrap(WLMStore.class).alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); + return unwrap(WMStore.class).alterResourcePlan(name, ns, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } default WMFullResourcePlan getActiveResourcePlan(String ns) throws MetaException { - return unwrap(WLMStore.class).getActiveResourcePlan(ns); + return unwrap(WMStore.class).getActiveResourcePlan(ns); } default WMValidateResourcePlanResponse validateResourcePlan(String name, String ns) throws NoSuchObjectException, InvalidObjectException, MetaException { - return unwrap(WLMStore.class).validateResourcePlan(name, ns); + return unwrap(WMStore.class).validateResourcePlan(name, ns); } default void dropResourcePlan(String name, String ns) throws NoSuchObjectException, MetaException { - unwrap(WLMStore.class).dropResourcePlan(name, ns); + unwrap(WMStore.class).dropResourcePlan(name, ns); } default void createWMTrigger(WMTrigger trigger) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).createWMTrigger(trigger); + unwrap(WMStore.class).createWMTrigger(trigger); } default void alterWMTrigger(WMTrigger trigger) throws NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).alterWMTrigger(trigger); + unwrap(WMStore.class).alterWMTrigger(trigger); } default void dropWMTrigger(String resourcePlanName, String triggerName, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).dropWMTrigger(resourcePlanName, triggerName, ns); + unwrap(WMStore.class).dropWMTrigger(resourcePlanName, triggerName, ns); } default List getTriggersForResourcePlan(String resourcePlanName, String ns) throws NoSuchObjectException, MetaException { - return unwrap(WLMStore.class).getTriggersForResourcePlan(resourcePlanName, ns); + return unwrap(WMStore.class).getTriggersForResourcePlan(resourcePlanName, ns); } default void createPool(WMPool pool) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).createPool(pool); + unwrap(WMStore.class).createPool(pool); } default void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).alterPool(pool, poolPath); + unwrap(WMStore.class).alterPool(pool, poolPath); } default void dropWMPool(String resourcePlanName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).dropWMPool(resourcePlanName, poolPath, ns); + unwrap(WMStore.class).dropWMPool(resourcePlanName, poolPath, ns); } default void createOrUpdateWMMapping(WMMapping mapping, boolean update) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).createOrUpdateWMMapping(mapping, update); + unwrap(WMStore.class).createOrUpdateWMMapping(mapping, update); } default void dropWMMapping(WMMapping mapping) throws NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).dropWMMapping(mapping); + unwrap(WMStore.class).dropWMMapping(mapping); } default void createWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); + unwrap(WMStore.class).createWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } default void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath, String ns) throws NoSuchObjectException, InvalidOperationException, MetaException { - unwrap(WLMStore.class).dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); + unwrap(WMStore.class).dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath, ns); } /** @@ -2344,17 +2344,17 @@ List getSchemaVersionsByColumns(String colName, String colNamespa /** Adds a RuntimeStat for persistence. */ default void addRuntimeStat(RuntimeStat stat) throws MetaException { - unwrap(WLMStore.class).addRuntimeStat(stat); + unwrap(WMStore.class).addRuntimeStat(stat); } /** Reads runtime statistic entries. */ default List getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { - return unwrap(WLMStore.class).getRuntimeStats(maxEntries, maxCreateTime); + return unwrap(WMStore.class).getRuntimeStats(maxEntries, maxCreateTime); } /** Removes outdated statistics. */ default int deleteRuntimeStats(int maxRetainSecs) throws MetaException { - return unwrap(WLMStore.class).deleteRuntimeStats(maxRetainSecs); + return unwrap(WMStore.class).deleteRuntimeStats(maxRetainSecs); } default List getTableNamesWithStats() throws MetaException, NoSuchObjectException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WMStore.java similarity index 96% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WMStore.java index e0eb9f0913ab..320b111306ee 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WLMStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/iface/WMStore.java @@ -35,11 +35,11 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; -import org.apache.hadoop.hive.metastore.metastore.impl.WLMStoreImpl; +import org.apache.hadoop.hive.metastore.metastore.impl.WMStoreImpl; -// work load management -@MetaDescriptor(alias = "wlm", defaultImpl = WLMStoreImpl.class) -public interface WLMStore { +// work management +@MetaDescriptor(alias = "wm", defaultImpl = WMStoreImpl.class) +public interface WMStore { void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) throws AlreadyExistsException, MetaException, InvalidObjectException, NoSuchObjectException; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java similarity index 99% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java index 22b8aece8eb1..c2f8dc6ce6f2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WLMStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; -import org.apache.hadoop.hive.metastore.metastore.iface.WLMStore; +import org.apache.hadoop.hive.metastore.metastore.iface.WMStore; import org.apache.hadoop.hive.metastore.model.MRuntimeStat; import org.apache.hadoop.hive.metastore.model.MWMMapping; import org.apache.hadoop.hive.metastore.model.MWMPool; @@ -64,8 +64,8 @@ import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -public class WLMStoreImpl extends RawStoreAware implements WLMStore { - private static final Logger LOG = LoggerFactory.getLogger(WLMStoreImpl.class); +public class WMStoreImpl extends RawStoreAware implements WMStore { + private static final Logger LOG = LoggerFactory.getLogger(WMStoreImpl.class); private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern( "yyyy_MM_dd_HH_mm_ss"); From fadb4bdf44cc1aef2903fd042ce75bde70ede650 Mon Sep 17 00:00:00 2001 From: zdeng Date: Fri, 19 Jun 2026 21:41:17 +0800 Subject: [PATCH 17/22] fix ut --- .../results/clientpositive/llap/resourceplan.q.out | 11 +++++++++-- .../metastore/metastore/impl/ColStatsStoreImpl.java | 13 +++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 8a29af8520ee..f7c4189f20b4 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -535,7 +535,12 @@ FAILED: SemanticException Invalid create arguments (tok_create_rp plan_3 (tok_qu PREHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_2 PREHOOK: type: ALTER RESOURCEPLAN PREHOOK: Output: dummyHostnameForTest -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. AlreadyExistsException(message:Resource plan name should be unique: ) +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:JDODataStoreException: Update of object with id "1[OID]org.apache.hadoop.hive.metastore.model.MWMResourcePlan" using statement "UPDATE WM_RESOURCEPLAN SET "NAME"=? WHERE RP_ID=?" failed : org.apache.derby.shared.common.error.DerbySQLIntegrityConstraintViolationException: The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'UNIQUE_WM_RESOURCEPLAN' defined on 'WM_RESOURCEPLAN'. +#### A masked pattern was here #### +Caused by: ERROR 23505: The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'UNIQUE_WM_RESOURCEPLAN' defined on 'WM_RESOURCEPLAN'. +#### A masked pattern was here #### + +Root cause: ERROR 23505: The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by 'UNIQUE_WM_RESOURCEPLAN' defined on 'WM_RESOURCEPLAN'.) PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_resourceplans @@ -1991,7 +1996,9 @@ plan_2 default USER user2 def 1 PREHOOK: query: DROP POOL plan_2.def.c1 PREHOOK: type: DROP POOL PREHOOK: Output: dummyHostnameForTest -FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. InvalidOperationException(message:Please remove all mappings for this pool.) +FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. MetaException(message:JDODataStoreException: Batched statement "DELETE FROM WM_POOL WHERE POOL_ID=?" failed. +DELETE on table 'WM_POOL' caused a violation of foreign key constraint 'WM_MAPPING_FK2' for key (6). The statement has been rolled back. +Root cause: ERROR 23503: DELETE on table 'WM_POOL' caused a violation of foreign key constraint 'WM_MAPPING_FK2' for key (6). The statement has been rolled back.) PREHOOK: query: EXPLAIN DROP USER MAPPING "user2" in plan_2 PREHOOK: type: DROP MAPPING PREHOOK: Output: dummyHostnameForTest diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index f07cfe4488ec..c0489ebb5651 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -105,7 +105,7 @@ public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { protected int batchSize = NO_BATCHING; private boolean areTxnStatsSupported = false; private Configuration conf; - private DirectSqlAggrStats directSqlAggrStats; + private String schema; private SQLGenerator sqlGenerator; @Override @@ -119,8 +119,7 @@ public void setBaseStore(RawStore store) { this.conf = store.getConf(); this.sqlGenerator = new SQLGenerator(dbType, conf); String schema = PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema"); - schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null); - this.directSqlAggrStats = new DirectSqlAggrStats(pm, conf, schema); + this.schema = org.apache.commons.lang3.StringUtils.defaultIfBlank(schema, null); } @Override @@ -589,7 +588,8 @@ protected ColumnStatistics getSqlResult() throws MetaException { String catName = normalizeIdentifier(tableName.getCat()); String dbName = normalizeIdentifier(tableName.getDb()); String tblName = normalizeIdentifier(tableName.getTable()); - return directSqlAggrStats.getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll); + return new DirectSqlAggrStats(pm, conf, schema) + .getTableStats(catName, dbName, tblName, colNames, engine, enableBitVector, enableKll); } @Override @@ -705,7 +705,7 @@ protected List getSqlResult() throws MetaException { String catName = normalizeIdentifier(tableName.getCat()); String dbName = normalizeIdentifier(tableName.getDb()); String tblName = normalizeIdentifier(tableName.getTable()); - return directSqlAggrStats.getPartitionStats( + return new DirectSqlAggrStats(pm, conf, schema).getPartitionStats( catName, dbName, tblName, partNames, colNames, engine, enableBitVector, enableKll); } @Override @@ -831,7 +831,8 @@ public List getPartitionColStats return new GetHelper>(this, null) { @Override protected List getSqlResult() throws MetaException { - return directSqlAggrStats.getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll); + return new DirectSqlAggrStats(pm, conf, schema) + .getColStatsForAllTablePartitions(catName, dbName, enableBitVector, enableKll); } @Override From a9bf47731807fbebbc60a3b74b2165601afbbc26 Mon Sep 17 00:00:00 2001 From: zdeng Date: Sat, 20 Jun 2026 06:43:40 +0800 Subject: [PATCH 18/22] review-3 --- .../main/java/org/apache/hadoop/hive/metastore/RawStore.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 4aa06b30c556..ca9211f27d81 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1051,7 +1051,7 @@ default PrincipalPrivilegeSet getTablePrivilegeSet (String catName, String dbNam * @throws InvalidObjectException no such partition * @throws MetaException error accessing the RDBMS */ - default PrincipalPrivilegeSet getPartitionPrivilegeSet (String catName, String dbName, String tableName, + default PrincipalPrivilegeSet getPartitionPrivilegeSet(String catName, String dbName, String tableName, String partition, String userName, List groupNames) throws InvalidObjectException, MetaException { return unwrap(PrivilegeStore.class).getPartitionPrivilegeSet(new TableName(catName, dbName, tableName), partition, userName, groupNames); } @@ -1305,8 +1305,9 @@ default Map updatePartitionColumnStatistics(ColumnStatistics sta List partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { ColumnStatisticsDesc statsDesc = statsObj.getStatsDesc(); + String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : MetaStoreUtils.getDefaultCatalog(getConf()); Table table = unwrap(TableStore.class) - .getTable(new TableName(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()), null, -1); + .getTable(new TableName(catName, statsDesc.getDbName(), statsDesc.getTableName()), null, -1); MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()); return updatePartitionColumnStatistics(table, mTable, statsObj, partVals, validWriteIds, writeId); } From 12f9d8d47969de7046d310f93c54b354050761ce Mon Sep 17 00:00:00 2001 From: zdeng Date: Sat, 20 Jun 2026 07:21:31 +0800 Subject: [PATCH 19/22] remove GetHelper from ObjectStore --- .../hadoop/hive/metastore/ObjectStore.java | 328 +++--------------- .../hive/metastore/TestObjectStore.java | 47 ++- 2 files changed, 71 insertions(+), 304 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 962211a1fde4..ab88fc2302d1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -62,6 +62,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; @@ -110,16 +111,14 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SerdeType; import org.apache.hadoop.hive.metastore.api.StoredProcedure; -import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableParamsUpdate; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.metastore.GetHelper; +import org.apache.hadoop.hive.metastore.metastore.GetListHelper; import org.apache.hadoop.hive.metastore.metastore.iface.PrivilegeStore; -import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; -import org.apache.hadoop.hive.metastore.metrics.Metrics; -import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MCatalog; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; @@ -150,15 +149,10 @@ import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; -import org.datanucleus.ExecutionContext; -import org.datanucleus.api.jdo.JDOPersistenceManager; -import org.datanucleus.api.jdo.JDOTransaction; import org.datanucleus.store.rdbms.exceptions.MissingTableException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.codahale.metrics.Counter; -import com.codahale.metrics.MetricRegistry; import com.cronutils.model.CronType; import com.cronutils.model.definition.CronDefinition; import com.cronutils.model.definition.CronDefinitionBuilder; @@ -212,7 +206,6 @@ private enum TXN_STATUS { private volatile int openTrasactionCalls = 0; private Transaction currentTransaction = null; private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; - private Counter directSqlErrors; private PropertyStore propertyStore; private final Map, Object> cachedImpls = new HashMap<>(); @@ -249,13 +242,6 @@ public void setConf(Configuration conf) { initialize(); - // Note, if metrics have not been initialized this will return null, which means we aren't - // using metrics. Thus we should always check whether this is non-null before using. - MetricRegistry registry = Metrics.getRegistry(); - if (registry != null) { - directSqlErrors = Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS); - } - this.batchSize = MetastoreConf.getIntVar(conf, ConfVars.RAWSTORE_PARTITION_BATCH_SIZE); if (!isInitialized) { @@ -455,15 +441,30 @@ public T unwrap(Class iface) { return TransactionHandler.getProxy(iface, new TransactionHandler<>(this, simpl, openQueries)); } + @VisibleForTesting + public RawStoreAware createRawStoreAware() { + return new RawStoreAware() { + @Override + public RawStore getBaseStore() { + return ObjectStore.this; + } + + @Override + public PersistenceManager getPersistentManager() { + return pm; + } + }; + } + @Override public void updateTableParams(List updates) throws MetaException, NoSuchObjectException { if (updates == null || updates.isEmpty()) { return; } - new GetListHelper(null, null, null, true, false) { + new GetListHelper(createRawStoreAware(), null) { @Override - protected List getSqlResult(GetHelper> ctx) throws MetaException { + protected List getSqlResult() throws MetaException { boolean success = false; try { openTransaction(); @@ -476,7 +477,12 @@ protected List getSqlResult(GetHelper> ctx) throws MetaExceptio } @Override - protected List getJdoResult(GetHelper> ctx) { + protected boolean canUseJdoQuery() { + return false; + } + + @Override + protected List getJdoResult() { throw new UnsupportedOperationException("UnsupportedOperationException"); } }.run(false); @@ -551,20 +557,6 @@ public void rollbackTransaction() { } } - private void setTransactionSavePoint(String savePoint) { - if (savePoint != null) { - ExecutionContext ec = ((JDOPersistenceManager) pm).getExecutionContext(); - ec.getStoreManager().getConnectionManager().getConnection(ec); - ((JDOTransaction) currentTransaction).setSavepoint(savePoint); - } - } - - private void rollbackTransactionToSavePoint(String savePoint) { - if (savePoint != null) { - ((JDOTransaction) currentTransaction).rollbackToSavepoint(savePoint); - } - } - @Override public void createCatalog(Catalog cat) throws MetaException { LOG.debug("Creating catalog {}", cat); @@ -776,15 +768,21 @@ public Database getDatabase(String catalogName, String name) throws NoSuchObject public Database getDatabaseInternal(String catalogName, String name) throws MetaException, NoSuchObjectException { - return new GetDbHelper(catalogName, name, true, true) { + return new GetHelper(createRawStoreAware(), + new DatabaseName(catalogName == null? getDefaultCatalog(conf) : catalogName, name)) { @Override - protected Database getSqlResult(GetHelper ctx) throws MetaException { - return directSql.getDatabase(catalogName, dbName); + protected String describeResult() { + return "db details for db ".concat(argument.getDb()); } @Override - protected Database getJdoResult(GetHelper ctx) throws MetaException, NoSuchObjectException { - return getJDODatabase(catalogName, dbName); + protected Database getSqlResult() throws MetaException { + return directSql.getDatabase(argument.getCat(), argument.getDb()); + } + + @Override + protected Database getJdoResult() throws MetaException, NoSuchObjectException { + return getJDODatabase(argument.getCat(), argument.getDb()); } }.run(false); } @@ -1402,250 +1400,6 @@ public static MColumnDescriptor createNewMColumnDescriptor(List co return new MColumnDescriptor(cols); } - /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */ - @VisibleForTesting - public abstract class GetHelper { - private final boolean isInTxn, doTrace, allowJdo; - private boolean doUseDirectSql; - private long start; - private Table table; - protected final List partitionFields; - protected final String catName, dbName, tblName; - private boolean success = false; - protected T results = null; - - public GetHelper(String catalogName, String dbName, String tblName, - boolean allowSql, boolean allowJdo) throws MetaException { - this(catalogName, dbName, tblName, null, allowSql, allowJdo); - } - - public GetHelper(String catalogName, String dbName, String tblName, - List fields, boolean allowSql, boolean allowJdo) throws MetaException { - assert allowSql || allowJdo; - this.allowJdo = allowJdo; - this.catName = (catalogName != null) ? normalizeIdentifier(catalogName) : null; - this.dbName = (dbName != null) ? normalizeIdentifier(dbName) : null; - this.partitionFields = fields; - if (tblName != null) { - this.tblName = normalizeIdentifier(tblName); - } else { - // tblName can be null in cases of Helper being used at a higher - // abstraction level, such as with datbases - this.tblName = null; - this.table = null; - } - this.doTrace = LOG.isDebugEnabled(); - this.isInTxn = isActiveTransaction(); - - boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL); - if (isConfigEnabled && directSql == null) { - directSql = new MetaStoreDirectSql(pm, getConf(), ""); - } - - if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) { - throw new MetaException("SQL is not operational"); // test path; SQL is enabled and broken. - } - this.doUseDirectSql = allowSql && isConfigEnabled && directSql.isCompatibleDatastore(); - } - - protected boolean canUseDirectSql(GetHelper ctx) throws MetaException { - return true; // By default, assume we can user directSQL - that's kind of the point. - } - protected abstract String describeResult(); - protected abstract T getSqlResult(GetHelper ctx) throws MetaException; - protected abstract T getJdoResult( - GetHelper ctx) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException; - - public T run(boolean initTable) throws MetaException, NoSuchObjectException { - try { - start(initTable); - String savePoint = isInTxn && allowJdo ? "rollback_" + System.nanoTime() : null; - if (doUseDirectSql) { - try { - directSql.prepareTxn(); - setTransactionSavePoint(savePoint); - this.results = getSqlResult(this); - LOG.debug("Using direct SQL optimization."); - } catch (Exception ex) { - handleDirectSqlError(ex, savePoint); - } - } - // Note that this will be invoked in 2 cases: - // 1) DirectSQL was disabled to start with; - // 2) DirectSQL threw and was disabled in handleDirectSqlError. - if (!doUseDirectSql) { - this.results = getJdoResult(this); - LOG.debug("Not using direct SQL optimization."); - } - return commit(); - } catch (NoSuchObjectException | MetaException ex) { - throw ex; - } catch (Exception ex) { - LOG.error("", ex); - throw new MetaException(ex.getMessage()); - } finally { - close(); - } - } - - private void start(boolean initTable) throws MetaException, NoSuchObjectException { - start = doTrace ? System.nanoTime() : 0; - openTransaction(); - if (initTable && (tblName != null)) { - table = ensureGetTable(catName, dbName, tblName); - } - doUseDirectSql = doUseDirectSql && canUseDirectSql(this); - } - - private void handleDirectSqlError(Exception ex, String savePoint) throws MetaException, NoSuchObjectException { - String message = null; - try { - message = generateShorterMessage(ex); - } catch (Throwable t) { - message = ex.toString() + "; error building a better message: " + t.getMessage(); - } - LOG.warn(message); // Don't log the exception, people just get confused. - LOG.debug("Full DirectSQL callstack for debugging (not an error)", ex); - - if (!allowJdo || !DatabaseProduct.isRecoverableException(ex)) { - throw ExceptionHandler.newMetaException(ex); - } - - if (!isInTxn) { - JDOException rollbackEx = null; - try { - rollbackTransaction(); - } catch (JDOException jex) { - rollbackEx = jex; - } - if (rollbackEx != null) { - // Datanucleus propagates some pointless exceptions and rolls back in the finally. - if (currentTransaction != null && currentTransaction.isActive()) { - throw rollbackEx; // Throw if the tx wasn't rolled back. - } - LOG.info("Ignoring exception, rollback succeeded: " + rollbackEx.getMessage()); - } - - start = doTrace ? System.nanoTime() : 0; - openTransaction(); - if (table != null) { - table = ensureGetTable(catName, dbName, tblName); - } - } else { - rollbackTransactionToSavePoint(savePoint); - start = doTrace ? System.nanoTime() : 0; - } - - if (directSqlErrors != null) { - directSqlErrors.inc(); - } - - doUseDirectSql = false; - } - - private String generateShorterMessage(Exception ex) { - StringBuilder message = new StringBuilder( - "Falling back to ORM path due to direct SQL failure (this is not an error): "); - Throwable t = ex; - StackTraceElement[] prevStack = null; - while (t != null) { - message.append(t.getMessage()); - StackTraceElement[] stack = t.getStackTrace(); - int uniqueFrames = stack.length - 1; - if (prevStack != null) { - int n = prevStack.length - 1; - while (uniqueFrames >= 0 && n >= 0 && stack[uniqueFrames].equals(prevStack[n])) { - uniqueFrames--; n--; - } - } - for (int i = 0; i <= uniqueFrames; ++i) { - StackTraceElement ste = stack[i]; - message.append(" at ").append(ste); - if (ste.getMethodName().contains("getSqlResult") - && (ste.getFileName() == null || ste.getFileName().contains("ObjectStore"))) { - break; - } - } - prevStack = stack; - t = t.getCause(); - if (t != null) { - message.append(";\n Caused by: "); - } - } - return message.toString(); - } - - private T commit() { - success = commitTransaction(); - if (doTrace) { - double time = ((System.nanoTime() - start) / 1000000.0); - String result = describeResult(); - String retrieveType = doUseDirectSql ? "SQL" : "ORM"; - - LOG.debug("{} retrieved using {} in {}ms", result, retrieveType, time); - } - return results; - } - - private void close() { - if (!success) { - rollbackTransaction(); - } - } - - public Table getTable() { - return table; - } - } - - private abstract class GetListHelper extends GetHelper> { - public GetListHelper(String catName, String dbName, String tblName, boolean allowSql, - boolean allowJdo) throws MetaException { - super(catName, dbName, tblName, null, allowSql, allowJdo); - } - - public GetListHelper(String catName, String dbName, String tblName, List fields, - boolean allowSql, boolean allowJdo) throws MetaException { - super(catName, dbName, tblName, fields, allowSql, allowJdo); - } - - @Override - protected String describeResult() { - return results.size() + " entries"; - } - } - - @VisibleForTesting - public abstract class GetDbHelper extends GetHelper { - /** - * GetHelper for returning db info using directSql/JDO. - * @param dbName The Database Name - * @param allowSql Whether or not we allow DirectSQL to perform this query. - * @param allowJdo Whether or not we allow ORM to perform this query. - */ - public GetDbHelper(String catalogName, String dbName,boolean allowSql, boolean allowJdo) - throws MetaException { - super(catalogName, dbName,null,allowSql,allowJdo); - } - - @Override - protected String describeResult() { - return "db details for db ".concat(dbName); - } - } - - private Table ensureGetTable(String catName, String dbName, String tblName) - throws NoSuchObjectException, MetaException { - TableName tableName = new TableName(catName, dbName, tblName); - Table table = - unwrap(TableStore.class).getTable(tableName, null, -1); - if (table == null) { - throw new NoSuchObjectException( - "Specified catalog.database.table does not exist : " + tableName); - } - return table; - } - /** * Verifies that the stats JSON string is unchanged for alter table (txn stats). * @return Error message with the details of the change, or null if the value has not changed. @@ -2335,13 +2089,13 @@ public List getAllFunctions(String catName) throws MetaException { protected List getFunctionsInternal(String catalogName) throws MetaException, NoSuchObjectException { - return new GetListHelper(catalogName, "", "", true, true) { + return new GetListHelper(createRawStoreAware(), catalogName) { @Override - protected List getSqlResult(GetHelper> ctx) throws MetaException { + protected List getSqlResult() throws MetaException { return directSql.getFunctions(catalogName); } @Override - protected List getJdoResult(GetHelper> ctx) throws MetaException { + protected List getJdoResult() throws MetaException { try { return getAllFunctionsViaJDO(catalogName); } catch (Exception e) { @@ -2352,7 +2106,7 @@ protected List getJdoResult(GetHelper> ctx) throws Meta }.run(false); } - private List getAllFunctionsViaJDO (String catName) { + private List getAllFunctionsViaJDO(String catName) { boolean commited = false; Query query = null; try { diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 6328c980bb29..64152ce0f681 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; @@ -77,6 +78,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hadoop.hive.metastore.metastore.GetHelper; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.model.MNotificationLog; @@ -1379,14 +1381,20 @@ public void testDirectSqlErrorMetrics() throws Exception { Counter directSqlErrors = Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS); - objectStore.new GetDbHelper(DEFAULT_CATALOG_NAME, "foo", true, true) { + new GetHelper(objectStore.createRawStoreAware(), + new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @Override - protected Database getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + protected Database getSqlResult() throws MetaException { return null; } - @Override - protected Database getJdoResult(ObjectStore.GetHelper ctx) throws MetaException, + @Override + protected String describeResult() { + return ""; + } + + @Override + protected Database getJdoResult() throws MetaException, NoSuchObjectException { return null; } @@ -1394,14 +1402,20 @@ protected Database getJdoResult(ObjectStore.GetHelper ctx) throws Meta Assert.assertEquals(0, directSqlErrors.getCount()); - objectStore.new GetDbHelper(DEFAULT_CATALOG_NAME, "foo", true, true) { + new GetHelper (objectStore.createRawStoreAware(), + new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @Override - protected Database getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + protected Database getSqlResult() throws MetaException { throw new RuntimeException(); } @Override - protected Database getJdoResult(ObjectStore.GetHelper ctx) throws MetaException, + protected String describeResult() { + return ""; + } + + @Override + protected Database getJdoResult() throws MetaException, NoSuchObjectException { return null; } @@ -1962,18 +1976,18 @@ public void testSavePoint() throws Exception { Assert.assertEquals(3, objectStore.getPartitionCount()); objectStore.openTransaction(); - objectStore.new GetHelper(DEFAULT_CATALOG_NAME, DB1, TABLE1, true, true) { + new GetHelper(objectStore.createRawStoreAware(), + new TableName(DEFAULT_CATALOG_NAME, DB1, TABLE1)) { @Override protected String describeResult() { return "test savepoint"; } @Override - protected Object getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + protected Object getSqlResult() throws MetaException { // drop the partitions with SQL alone try (AutoCloseable c = deadline(); AutoCloseable d = new DirectSqlConfigurator(conf, true)) { - objectStore.unwrap(TableStore.class) - .dropPartitions(new TableName(ctx.catName, ctx.dbName, ctx.tblName), partNames); + objectStore.unwrap(TableStore.class).dropPartitions(argument, partNames); assertEquals(0, objectStore.getPartitionCount()); } catch (Exception e) { throw new MetaException(e.getMessage()); @@ -1982,12 +1996,11 @@ protected Object getSqlResult(ObjectStore.GetHelper ctx) throws MetaExce } @Override - protected Object getJdoResult(ObjectStore.GetHelper ctx) throws MetaException { + protected Object getJdoResult() throws MetaException { // drop the partitions with JDO alone try (AutoCloseable c = deadline(); AutoCloseable d = new DirectSqlConfigurator(conf, false)) { assertEquals(3, objectStore.getPartitionCount()); - objectStore.unwrap(TableStore.class) - .dropPartitions(new TableName(ctx.catName, ctx.dbName, ctx.tblName), partNames); + objectStore.unwrap(TableStore.class).dropPartitions(argument, partNames); } catch (Exception e) { throw new MetaException(e.getMessage()); } @@ -2008,14 +2021,14 @@ public void testNoJdoForUnrecoverableException() throws Exception { AtomicBoolean runDirectSql = new AtomicBoolean(false); AtomicBoolean runJdo = new AtomicBoolean(false); try { - objectStore.new GetHelper(DEFAULT_CATALOG_NAME, DB1, TABLE1, true, true) { + new GetHelper(objectStore.createRawStoreAware(), null) { @Override protected String describeResult() { return "test not run jdo for unrecoverable exception"; } @Override - protected Object getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { + protected Object getSqlResult() throws MetaException { runDirectSql.set(true); MetaException me = new MetaException("Throwing unrecoverable exception to test not run jdo."); me.initCause(unrecoverableException); @@ -2023,7 +2036,7 @@ protected Object getSqlResult(ObjectStore.GetHelper ctx) throws MetaException { } @Override - protected Object getJdoResult(ObjectStore.GetHelper ctx) throws MetaException, NoSuchObjectException { + protected Object getJdoResult() throws MetaException, NoSuchObjectException { runJdo.set(true); SQLIntegrityConstraintViolationException ex = new SQLIntegrityConstraintViolationException("Unrecoverable ex"); MetaException me = new MetaException("Throwing unrecoverable exception to test not run jdo."); From 2420d8bbd31ca6f9a664555177b22812ccf6485c Mon Sep 17 00:00:00 2001 From: zdeng Date: Sat, 20 Jun 2026 07:32:07 +0800 Subject: [PATCH 20/22] RawStoreAware -> RawStoreBundle --- .../apache/hadoop/hive/metastore/ObjectStore.java | 14 +++++++------- .../hadoop/hive/metastore/metastore/GetHelper.java | 4 ++-- .../hive/metastore/metastore/GetListHelper.java | 4 ++-- .../{RawStoreAware.java => RawStoreBundle.java} | 2 +- .../metastore/impl/ColStatsStoreImpl.java | 6 +++--- .../metastore/impl/ConstraintStoreImpl.java | 4 ++-- .../metastore/impl/NotificationStoreImpl.java | 4 ++-- .../metastore/impl/PrivilegeStoreImpl.java | 5 ++--- .../metastore/metastore/impl/TableStoreImpl.java | 6 ++---- .../metastore/metastore/impl/TokenStoreImpl.java | 4 ++-- .../hive/metastore/metastore/impl/WMStoreImpl.java | 4 ++-- .../hadoop/hive/metastore/TestObjectStore.java | 8 ++++---- 12 files changed, 31 insertions(+), 34 deletions(-) rename standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/{RawStoreAware.java => RawStoreBundle.java} (97%) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index ab88fc2302d1..b5cae384be04 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -143,7 +143,7 @@ import org.apache.hadoop.hive.metastore.properties.CachingPropertyStore; import org.apache.hadoop.hive.metastore.properties.PropertyStore; import org.apache.hadoop.hive.metastore.metastore.PersistenceManagerProxy; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.MetaDescriptor; import org.apache.hadoop.hive.metastore.metastore.TransactionHandler; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; @@ -434,7 +434,7 @@ public T unwrap(Class iface) { cachedImpls.put(iface, simpl); } List openQueries = new LinkedList<>(); - if (simpl instanceof RawStoreAware rsa) { + if (simpl instanceof RawStoreBundle rsa) { rsa.setBaseStore(this); rsa.setPersistentManager(PersistenceManagerProxy.getProxy(pm, openQueries)); } @@ -442,8 +442,8 @@ public T unwrap(Class iface) { } @VisibleForTesting - public RawStoreAware createRawStoreAware() { - return new RawStoreAware() { + public RawStoreBundle createRawStoreBundle() { + return new RawStoreBundle() { @Override public RawStore getBaseStore() { return ObjectStore.this; @@ -462,7 +462,7 @@ public void updateTableParams(List updates) throws MetaExcept return; } - new GetListHelper(createRawStoreAware(), null) { + new GetListHelper(createRawStoreBundle(), null) { @Override protected List getSqlResult() throws MetaException { boolean success = false; @@ -768,7 +768,7 @@ public Database getDatabase(String catalogName, String name) throws NoSuchObject public Database getDatabaseInternal(String catalogName, String name) throws MetaException, NoSuchObjectException { - return new GetHelper(createRawStoreAware(), + return new GetHelper(createRawStoreBundle(), new DatabaseName(catalogName == null? getDefaultCatalog(conf) : catalogName, name)) { @Override protected String describeResult() { @@ -2089,7 +2089,7 @@ public List getAllFunctions(String catName) throws MetaException { protected List getFunctionsInternal(String catalogName) throws MetaException, NoSuchObjectException { - return new GetListHelper(createRawStoreAware(), catalogName) { + return new GetListHelper(createRawStoreBundle(), catalogName) { @Override protected List getSqlResult() throws MetaException { return directSql.getFunctions(catalogName); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java index 4c911e919dbc..303eba1aa08c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java @@ -61,11 +61,11 @@ public abstract class GetHelper { private boolean success = false; protected T results = null; - public GetHelper(RawStoreAware rsa, A args) throws MetaException { + public GetHelper(RawStoreBundle rsa, A args) throws MetaException { this(rsa, args, null); } - public GetHelper(RawStoreAware rsa, + public GetHelper(RawStoreBundle rsa, A args, List fields) throws MetaException { this.baseStore = rsa.getBaseStore(); this.partitionFields = fields; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java index fdd9e769cbed..5b485fffc28a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java @@ -23,11 +23,11 @@ import org.apache.hadoop.hive.metastore.api.MetaException; public abstract class GetListHelper extends GetHelper> { - public GetListHelper(RawStoreAware rsa, A args) throws MetaException { + public GetListHelper(RawStoreBundle rsa, A args) throws MetaException { super(rsa, args, null); } - public GetListHelper(RawStoreAware rsa, + public GetListHelper(RawStoreBundle rsa, A args, List fields) throws MetaException { super(rsa, args, fields); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreAware.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreBundle.java similarity index 97% rename from standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreAware.java rename to standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreBundle.java index 1d16764811cb..add2f07dfd23 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreAware.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/RawStoreBundle.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.metastore.RawStore; -public abstract class RawStoreAware { +public abstract class RawStoreBundle { protected RawStore baseStore; protected PersistenceManager pm; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java index c0489ebb5651..9e3874453f0e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ColStatsStoreImpl.java @@ -76,7 +76,7 @@ import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.metastore.GetHelper; import org.apache.hadoop.hive.metastore.metastore.GetListHelper; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.iface.ColStatsStore; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.model.MPartition; @@ -98,7 +98,7 @@ import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifiers; -public class ColStatsStoreImpl extends RawStoreAware implements ColStatsStore { +public class ColStatsStoreImpl extends RawStoreBundle implements ColStatsStore { private static final Logger LOG = LoggerFactory.getLogger(ColStatsStoreImpl.class); private DatabaseProduct dbType; @@ -1177,7 +1177,7 @@ private void executePlainSQL(String sql, } private abstract class GetStatHelper extends GetHelper { - public GetStatHelper(TableName tableName, RawStoreAware baseStore) throws MetaException { + public GetStatHelper(TableName tableName, RawStoreBundle baseStore) throws MetaException { super(baseStore, tableName); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java index 8b4416643df1..f7dcee00dfaa 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/ConstraintStoreImpl.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.metastore.GetListHelper; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.iface.ConstraintStore; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; @@ -66,7 +66,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -public class ConstraintStoreImpl extends RawStoreAware implements ConstraintStore { +public class ConstraintStoreImpl extends RawStoreBundle implements ConstraintStore { private Configuration conf; @Override diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java index 8c542fc661cb..b7159dfda158 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/NotificationStoreImpl.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.model.MNotificationLog; import org.apache.hadoop.hive.metastore.model.MNotificationNextId; import org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog; @@ -62,7 +62,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -public class NotificationStoreImpl extends RawStoreAware implements NotificationStore { +public class NotificationStoreImpl extends RawStoreBundle implements NotificationStore { private static final Logger LOG = LoggerFactory.getLogger(NotificationStoreImpl.class); private Configuration conf; private SQLGenerator sqlGenerator; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java index 513b72e8b4e4..17f58867e5ba 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/PrivilegeStoreImpl.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.iface.TableStore; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDCPrivilege; @@ -67,7 +67,6 @@ import org.apache.hadoop.hive.metastore.model.MTable; import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; -import org.apache.hadoop.hive.metastore.metastore.GetHelper; import org.apache.hadoop.hive.metastore.metastore.GetListHelper; import org.apache.hadoop.hive.metastore.metastore.iface.PrivilegeStore; import org.slf4j.Logger; @@ -77,7 +76,7 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -public class PrivilegeStoreImpl extends RawStoreAware implements PrivilegeStore { +public class PrivilegeStoreImpl extends RawStoreBundle implements PrivilegeStore { private static final Logger LOG = LoggerFactory.getLogger(PrivilegeStoreImpl.class); private Configuration conf; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index 352c9078a44c..c90232ff6ff1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -47,8 +47,6 @@ import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.Batchable; import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.Deadline; @@ -91,7 +89,7 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.client.builder.GetPartitionsArgs; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.model.FetchGroups; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; @@ -142,7 +140,7 @@ import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; @SuppressWarnings("unchecked") -public class TableStoreImpl extends RawStoreAware implements TableStore { +public class TableStoreImpl extends RawStoreBundle implements TableStore { private final static Logger LOG = LoggerFactory.getLogger(TableStoreImpl.class); private DatabaseProduct dbType; protected int batchSize = NO_BATCHING; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java index ba2fa99a25c2..6de4cbf0bd25 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TokenStoreImpl.java @@ -26,14 +26,14 @@ import org.apache.hadoop.hive.metastore.QueryWrapper; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.iface.TokenStore; import org.apache.hadoop.hive.metastore.model.MDelegationToken; import org.apache.hadoop.hive.metastore.model.MMasterKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TokenStoreImpl extends RawStoreAware implements TokenStore { +public class TokenStoreImpl extends RawStoreBundle implements TokenStore { private static final Logger LOG = LoggerFactory.getLogger(TokenStoreImpl.class); private MDelegationToken getTokenFrom(String tokenId) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java index c2f8dc6ce6f2..82b05d0ad392 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/WMStoreImpl.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.metastore.RawStoreAware; +import org.apache.hadoop.hive.metastore.metastore.RawStoreBundle; import org.apache.hadoop.hive.metastore.metastore.iface.WMStore; import org.apache.hadoop.hive.metastore.model.MRuntimeStat; import org.apache.hadoop.hive.metastore.model.MWMMapping; @@ -64,7 +64,7 @@ import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; -public class WMStoreImpl extends RawStoreAware implements WMStore { +public class WMStoreImpl extends RawStoreBundle implements WMStore { private static final Logger LOG = LoggerFactory.getLogger(WMStoreImpl.class); private static final DateTimeFormatter YMDHMS_FORMAT = DateTimeFormatter.ofPattern( "yyyy_MM_dd_HH_mm_ss"); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 64152ce0f681..f0a4451b0987 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -1381,7 +1381,7 @@ public void testDirectSqlErrorMetrics() throws Exception { Counter directSqlErrors = Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS); - new GetHelper(objectStore.createRawStoreAware(), + new GetHelper(objectStore.createRawStoreBundle(), new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @Override protected Database getSqlResult() throws MetaException { @@ -1402,7 +1402,7 @@ protected Database getJdoResult() throws MetaException, Assert.assertEquals(0, directSqlErrors.getCount()); - new GetHelper (objectStore.createRawStoreAware(), + new GetHelper (objectStore.createRawStoreBundle(), new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @Override protected Database getSqlResult() throws MetaException { @@ -1976,7 +1976,7 @@ public void testSavePoint() throws Exception { Assert.assertEquals(3, objectStore.getPartitionCount()); objectStore.openTransaction(); - new GetHelper(objectStore.createRawStoreAware(), + new GetHelper(objectStore.createRawStoreBundle(), new TableName(DEFAULT_CATALOG_NAME, DB1, TABLE1)) { @Override protected String describeResult() { @@ -2021,7 +2021,7 @@ public void testNoJdoForUnrecoverableException() throws Exception { AtomicBoolean runDirectSql = new AtomicBoolean(false); AtomicBoolean runJdo = new AtomicBoolean(false); try { - new GetHelper(objectStore.createRawStoreAware(), null) { + new GetHelper(objectStore.createRawStoreBundle(), null) { @Override protected String describeResult() { return "test not run jdo for unrecoverable exception"; From 87fe349ba507d65b100c4cf11b18a3a974706e9e Mon Sep 17 00:00:00 2001 From: zdeng Date: Sat, 20 Jun 2026 07:58:00 +0800 Subject: [PATCH 21/22] fix ut --- .../org/apache/hadoop/hive/metastore/ObjectStore.java | 5 +++-- .../hadoop/hive/metastore/metastore/GetHelper.java | 10 ++++++++-- .../hive/metastore/metastore/impl/TableStoreImpl.java | 1 - .../apache/hadoop/hive/metastore/TestObjectStore.java | 4 ++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b5cae384be04..cadb13d6a552 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -443,15 +443,16 @@ public T unwrap(Class iface) { @VisibleForTesting public RawStoreBundle createRawStoreBundle() { + final ObjectStore objectStore = this; return new RawStoreBundle() { @Override public RawStore getBaseStore() { - return ObjectStore.this; + return objectStore; } @Override public PersistenceManager getPersistentManager() { - return pm; + return objectStore.pm; } }; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java index 303eba1aa08c..51d15d4c3b33 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java @@ -47,8 +47,8 @@ @VisibleForTesting public abstract class GetHelper { private static final Logger LOG = LoggerFactory.getLogger(GetHelper.class); - private static final Counter directSqlErrors = Metrics.getRegistry() != null ? - Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS) : new Counter(); + private static Counter directSqlErrors = setDirectSqlErrors(Metrics.getRegistry() != null ? + Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS) : new Counter()); private final boolean isInTxn, doTrace, allowJdo; private boolean doUseDirectSql; private long start; @@ -272,4 +272,10 @@ public List getPartitionFields() { public static long getDirectSqlErrors() { return directSqlErrors.getCount(); } + + @VisibleForTesting + public static Counter setDirectSqlErrors(Counter counter) { + directSqlErrors = counter; + return counter; + } } \ No newline at end of file diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java index c90232ff6ff1..28a34be97b5e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/impl/TableStoreImpl.java @@ -326,7 +326,6 @@ private void preDropStorageDescriptor(MStorageDescriptor msd) { removeUnusedColumnDescriptor(mcd); } - @Override public List dropAllPartitionsAndGetLocations(TableName table, String baseLocationToNotShow, AtomicReference message) diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index f0a4451b0987..e2f87ee4fdb4 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -1379,7 +1379,7 @@ public void testDirectSqlErrorMetrics() throws Exception { // recall setup so that we get an object store with the metrics initalized setUp(); Counter directSqlErrors = - Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS); + GetHelper.setDirectSqlErrors(Metrics.getOrCreateCounter(MetricsConstants.DIRECTSQL_ERRORS)); new GetHelper(objectStore.createRawStoreBundle(), new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @@ -1402,7 +1402,7 @@ protected Database getJdoResult() throws MetaException, Assert.assertEquals(0, directSqlErrors.getCount()); - new GetHelper (objectStore.createRawStoreBundle(), + new GetHelper(objectStore.createRawStoreBundle(), new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { @Override protected Database getSqlResult() throws MetaException { From a6cd0d8e957f8a42b0a668fad32b5386d8b6406e Mon Sep 17 00:00:00 2001 From: zdeng Date: Sat, 20 Jun 2026 17:41:10 +0800 Subject: [PATCH 22/22] fix --- .../java/org/apache/hadoop/hive/metastore/ObjectStore.java | 6 ++++-- .../hadoop/hive/metastore/metastore/GetListHelper.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index cadb13d6a552..cdabbcebc3bf 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -778,12 +778,14 @@ protected String describeResult() { @Override protected Database getSqlResult() throws MetaException { - return directSql.getDatabase(argument.getCat(), argument.getDb()); + return directSql.getDatabase(normalizeIdentifier(argument.getCat()), + normalizeIdentifier(argument.getDb())); } @Override protected Database getJdoResult() throws MetaException, NoSuchObjectException { - return getJDODatabase(argument.getCat(), argument.getDb()); + return getJDODatabase(normalizeIdentifier(argument.getCat()), + normalizeIdentifier(argument.getDb())); } }.run(false); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java index 5b485fffc28a..c9a8267c41be 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetListHelper.java @@ -34,6 +34,6 @@ public GetListHelper(RawStoreBundle rsa, @Override protected String describeResult() { - return results.size() + " entries"; + return (results != null ? results.size() : 0) + " entries"; } }