apache · zhaorongsheng · Sep 22, 2026
@@ -53,6 +53,14 @@ public interface HMSCachedClient {
 
     List<Partition> listPartitions(String dbName, String tblName);
 
+    /**
+     * Lists partitions matching the HMS filter grammar. Callers use this to avoid enumerating every
+     * partition before applying a selective equality or IN predicate.
+     */
+    default List<Partition> listPartitionsByFilter(String dbName, String tblName, String filter) {
+        throw new UnsupportedOperationException("listPartitionsByFilter is not supported");
+    }
+
     List<String> listPartitionNames(String dbName, String tblName, long maxListPartitionNum);
 
     Partition getPartition(String dbName, String tblName, List<String> partitionValues);

@@ -17,13 +17,15 @@
 
 package org.apache.doris.datasource.hive;
 
+import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.analysis.TableScanParams;
 import org.apache.doris.analysis.TableSnapshot;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.ListPartitionItem;
 import org.apache.doris.catalog.MTMV;
 import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.catalog.PartitionKey;
 import org.apache.doris.catalog.PartitionType;
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.ScalarType;
@@ -62,6 +64,7 @@
 import org.apache.doris.mtmv.MTMVSnapshotIf;
 import org.apache.doris.nereids.exceptions.NotSupportedException;
 import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
+import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
 import org.apache.doris.qe.GlobalVariable;
@@ -97,6 +100,7 @@
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.utils.FileUtils;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.internal.schema.InternalSchema;
@@ -467,6 +471,42 @@ public Optional<SortedPartitionRanges<String>> getSortedPartitionRanges(CatalogR
         return hivePartitionValues.getSortedPartitionRanges();
     }
 
+    @Override
+    public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot> snapshot) {
+        if (getDlaType() == DLAType.HIVE && !getPartitionColumns(snapshot).isEmpty()) {
+            return SelectedPartitions.DEFERRED_PARTITION_PRUNING;
+        }
+        return super.initSelectedPartitions(snapshot);
+    }
+
+    /**
+     * Materializes only the partitions admitted by a safe HMS partition filter. An empty result means
+     * the predicate or metastore does not support the filter grammar and local pruning remains intact.
+     */
+    public Optional<Map<String, PartitionItem>> getNameToPartitionItemsByFilter(
+            Optional<MvccSnapshot> snapshot, Expression predicate) {
+        if (getDlaType() != DLAType.HIVE) {
+            return Optional.empty();
+        }
+        List<Column> partitionColumns = getPartitionColumns(snapshot);
+        if (partitionColumns.isEmpty()) {
+            return Optional.empty();
+        }
+        String filter = HivePartitionFilterBuilder.build(predicate, partitionColumns);
+        if (filter == null) {
+            return Optional.empty();
+        }
+        try {
+            List<Partition> partitions = ((HMSExternalCatalog) catalog).getClient()
+                    .listPartitionsByFilter(getRemoteDbName(), getRemoteName(), filter);
+            return Optional.of(toNameToPartitionItems(partitions, partitionColumns, snapshot));
+        } catch (RuntimeException e) {
+            LOG.warn("Failed to prune Hive partitions through HMS filter for table {}.{}",
+                    getDbName(), getName());
+            return Optional.empty();
+        }
+    }
+
     public SelectedPartitions initHudiSelectedPartitions(Optional<TableSnapshot> tableSnapshot) {
         if (getDlaType() != DLAType.HUDI) {
             return SelectedPartitions.NOT_PRUNED;
@@ -518,6 +558,35 @@ public Map<String, PartitionItem> getNameToPartitionItems() {
         return nameToPartitionItem;
     }
 
+    private Map<String, PartitionItem> toNameToPartitionItems(List<Partition> partitions,
+            List<Column> partitionColumns, Optional<MvccSnapshot> snapshot) {
+        List<String> partitionColumnNames = partitionColumns.stream()
+                .map(Column::getName)
+                .collect(Collectors.toList());
+        List<Type> partitionColumnTypes = getPartitionColumnTypes(snapshot);
+        Map<String, PartitionItem> result = Maps.newHashMapWithExpectedSize(partitions.size());
+        for (Partition partition : partitions) {
+            String partitionName = FileUtils.makePartName(partitionColumnNames, partition.getValues());
+            result.put(partitionName, toListPartitionItem(partitionName, partitionColumnTypes));
+        }
+        return result;
+    }
+
+    private ListPartitionItem toListPartitionItem(String partitionName, List<Type> types) {
+        List<String> partitionValues = HiveUtil.toPartitionValues(partitionName);
+        List<PartitionValue> values = Lists.newArrayListWithExpectedSize(types.size());
+        for (String partitionValue : partitionValues) {
+            values.add(new PartitionValue(partitionValue,
+                    HiveExternalMetaCache.HIVE_DEFAULT_PARTITION.equals(partitionValue)));
+        }
+        try {
+            PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(values, types, true);
+            return new ListPartitionItem(Lists.newArrayList(partitionKey));
+        } catch (AnalysisException e) {
+            throw new HMSClientException("failed to convert filtered Hive partition %s", e, partitionName);
+        }
+    }
+
     public boolean isHiveTransactionalTable() {
         return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable);
     }

@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.hive;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.Lists;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/** Converts the safe subset of Nereids partition predicates to the HMS filter grammar. */
+public final class HivePartitionFilterBuilder {
+
+    private HivePartitionFilterBuilder() {
+    }
+
+    /**
+     * Builds an HMS filter from direct equality and IN predicates. Returns null when any predicate
+     * or column type is outside the supported grammar, so callers can retain local pruning.
+     */
+    public static String build(Expression predicate, List<Column> partitionColumns) {
+        Map<String, Column> columnsByName = partitionColumns.stream()
+                .collect(Collectors.toMap(column -> column.getName().toLowerCase(Locale.ROOT),
+                        Function.identity()));
+        Map<String, List<String>> valuesByName = new HashMap<>();
+        for (Expression conjunct : ExpressionUtils.extractConjunction(predicate)) {
+            if (!collectValues(conjunct, columnsByName, valuesByName)) {
+                return null;
+            }
+        }
+        return buildFilter(partitionColumns, valuesByName);
+    }
+
+    private static boolean collectValues(Expression expression, Map<String, Column> columnsByName,
+            Map<String, List<String>> valuesByName) {
+        if (expression instanceof EqualTo) {
+            EqualTo equalTo = (EqualTo) expression;
+            return collectEqualValue(equalTo.left(), equalTo.right(), columnsByName, valuesByName)
+                    || collectEqualValue(equalTo.right(), equalTo.left(), columnsByName, valuesByName);
+        }
+        if (expression instanceof InPredicate) {
+            InPredicate inPredicate = (InPredicate) expression;
+            String columnName = slotName(inPredicate.getCompareExpr());
+            if (columnName == null || !columnsByName.containsKey(columnName)) {
+                return false;
+            }
+            List<String> values = Lists.newArrayList();
+            for (Expression option : inPredicate.getOptions()) {
+                String value = literalValue(option);
+                if (value == null
+                        || !literalIsSupported((Literal) option, columnsByName.get(columnName))) {
+                    return false;
+                }
+                values.add(value);
+            }
+            valuesByName.computeIfAbsent(columnName, ignored -> Lists.newArrayList()).addAll(values);
+            return true;
+        }
+        return false;
+    }
+
+    private static boolean collectEqualValue(Expression slotExpression, Expression literalExpression,
+            Map<String, Column> columnsByName, Map<String, List<String>> valuesByName) {
+        String columnName = slotName(slotExpression);
+        if (columnName == null || !columnsByName.containsKey(columnName)) {
+            return false;
+        }
+        String value = literalValue(literalExpression);
+        if (value == null
+                || !literalIsSupported((Literal) literalExpression, columnsByName.get(columnName))) {
+            return false;
+        }
+        valuesByName.computeIfAbsent(columnName, ignored -> Lists.newArrayList()).add(value);
+        return true;
+    }
+
+    private static String slotName(Expression expression) {
+        if (expression instanceof SlotReference) {
+            return ((SlotReference) expression).getName().toLowerCase(Locale.ROOT);
+        }
+        return null;
+    }
+
+    private static String literalValue(Expression expression) {
+        if (!(expression instanceof Literal)) {
+            return null;
+        }
+        Object value = ((Literal) expression).getValue();
+        return value == null ? null : value.toString();
+    }
+
+    private static boolean literalIsSupported(Literal literal, Column column) {
+        DataType literalType = literal.getDataType();
+        String value = literal.getValue().toString();
+        if (column.getType().isIntegerType()) {
+            return literalType.isIntegerType() && isIntegralLiteral(literal.getValue().toString());
+        }
+        return column.getType().isStringType() && literalType.isStringType()
+                && value.indexOf('\\') < 0 && value.indexOf('\'') < 0;
+    }
+
+    private static String buildFilter(List<Column> partitionColumns, Map<String, List<String>> valuesByName) {
+        List<String> filters = Lists.newArrayList();
+        for (Column partitionColumn : partitionColumns) {
+            List<String> values = valuesByName.get(partitionColumn.getName().toLowerCase(Locale.ROOT));
+            if (values == null || values.isEmpty()) {
+                continue;
+            }
+            if (!isHmsFilterIdentifier(partitionColumn.getName())) {
+                return null;
+            }
+            List<String> valueFilters = values.stream()
+                    .map(value -> partitionColumn.getName() + " = " + toHmsLiteral(value, partitionColumn))
+                    .collect(Collectors.toList());
+            filters.add("(" + String.join(" OR ", valueFilters) + ")");
+        }
+        return filters.isEmpty() ? null : String.join(" AND ", filters);
+    }
+
+    private static String toHmsLiteral(String value, Column column) {
+        return column.getType().isIntegerType() ? value : "'" + value + "'";
+    }
+
+    private static boolean isHmsFilterIdentifier(String value) {
+        if (value.isEmpty() || !isHmsFilterLetterOrDigit(value.charAt(0))) {
+            return false;
+        }
+        String lowerCaseValue = value.toLowerCase(Locale.ROOT);
+        if (lowerCaseValue.equals("not") || lowerCaseValue.equals("and") || lowerCaseValue.equals("or")
+                || lowerCaseValue.equals("like") || lowerCaseValue.equals("date")
+                || lowerCaseValue.equals("const") || lowerCaseValue.equals("struct")
+                || isAllDigits(value)) {
+            return false;
+        }
+        for (int index = 1; index < value.length(); index++) {
+            char character = value.charAt(index);
+            if (!isHmsFilterLetterOrDigit(character) && character != '_') {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private static boolean isAllDigits(String value) {
+        return value.chars().allMatch(character -> character >= '0' && character <= '9');
+    }
+
+    private static boolean isHmsFilterLetterOrDigit(char value) {
+        return value >= 'a' && value <= 'z'
+                || value >= 'A' && value <= 'Z'
+                || value >= '0' && value <= '9';
+    }
+
+    private static boolean isIntegralLiteral(String value) {
+        int start = value.startsWith("-") ? 1 : 0;
+        if (start == value.length()) {
+            return false;
+        }
+        return value.chars().skip(start).allMatch(character -> character >= '0' && character <= '9');
+    }
+}
@@ -332,6 +332,28 @@ public List<Partition> listPartitions(String dbName, String tblName) {
         }
     }
 
+    @Override
+    public List<Partition> listPartitionsByFilter(String dbName, String tblName, String filter) {
+        short maxPartitions = (short) (DEFAULT_PARTITION_BATCH_SIZE + 1);
+        try (ThriftHMSClient client = getClient()) {
+            try {
+                List<Partition> partitions = ugiDoAs(() -> client.client.listPartitionsByFilter(
+                        dbName, tblName, filter, maxPartitions));
+                if (partitions.size() > DEFAULT_PARTITION_BATCH_SIZE) {
+                    throw new HMSClientException(
+                            "HMS partition filter matched more than %d partitions in table '%s.%s'.",
+                            DEFAULT_PARTITION_BATCH_SIZE, dbName, tblName);
+                }
+                return partitions;
+            } catch (Exception e) {
+                client.setThrowable(e);
+                throw e;
+            }
+        } catch (Exception e) {
+            throw new HMSClientException("failed to filter partitions in table '%s.%s'.", e, dbName, tblName);
+        }
+    }
+
     @Override
     public List<String> listPartitionNames(String dbName, String tblName, long maxListPartitionNum) {
         // list all parts when the limit is greater than the short maximum

@@ -43,6 +43,7 @@
 import org.apache.doris.catalog.Function.NullableMode;
 import org.apache.doris.catalog.OdbcTable;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionItem;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.Type;
 import org.apache.doris.common.Config;
@@ -127,6 +128,7 @@
 import org.apache.doris.nereids.trees.plans.PreAggStatus;
 import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
 import org.apache.doris.nereids.trees.plans.algebra.Relation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
 import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
 import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
@@ -747,7 +749,8 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
                     scanNode = new HiveScanNode(context.nextPlanNodeId(), tupleDescriptor, false, sv, directoryLister,
                             context.getScanContext());
                     HiveScanNode hiveScanNode = (HiveScanNode) scanNode;
-                    hiveScanNode.setSelectedPartitions(fileScan.getSelectedPartitions());
+                    hiveScanNode.setSelectedPartitions(
+                            materializeDeferredHivePartitions((HMSExternalTable) table, fileScan));
                     if (fileScan.getTableSample().isPresent()) {
                         hiveScanNode.setTableSample(new TableSample(fileScan.getTableSample().get().isPercent,
                                 fileScan.getTableSample().get().sampleValue, fileScan.getTableSample().get().seek));
@@ -796,6 +799,17 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla
         return getPlanFragmentForPhysicalFileScan(fileScan, context, scanNode, table, tupleDescriptor);
     }
 
+    private LogicalFileScan.SelectedPartitions materializeDeferredHivePartitions(
+            HMSExternalTable table, PhysicalFileScan fileScan) {
+        LogicalFileScan.SelectedPartitions selectedPartitions = fileScan.getSelectedPartitions();
+        if (!selectedPartitions.isDeferredPartitionPruning()) {
+            return selectedPartitions;
+        }
+        Map<String, PartitionItem> partitionItems =
+                table.getNameToPartitionItems(fileScan.getRelationSnapshot());
+        return new LogicalFileScan.SelectedPartitions(partitionItems.size(), partitionItems, false);
+    }
+
     @Override
     public PlanFragment visitPhysicalEmptyRelation(PhysicalEmptyRelation emptyRelation, PlanTranslatorContext context) {
         List<Slot> output = emptyRelation.getOutput();