From 8ca11c2bd8d45772f73491e49e3386dab655f8ec Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Fri, 18 Sep 2026 07:18:14 +0000 Subject: [PATCH 1/8] HIVE-30037: Resolve ORDER BY ordinals when CBO declines the statement When hive.cbo.enable is true, processPositionAlias skips ORDER BY position substitution because CBO is expected to resolve ordinals in genSortByKey. If CBO then declines the AST, the legacy planner compiled the unsubstituted ordinal as a constant and dropped the sort. Substitute ORDER BY ordinals on the CBO-decline path so the requested order is preserved. Co-authored-by: Sankalp Thakur --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 4 + .../hive/ql/parse/SemanticAnalyzer.java | 15 ++- .../hive/ql/parse/TestSemanticAnalyzer.java | 112 ++++++++++++++++++ 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index a8140689a717..74b7cbb23ab9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -706,6 +706,10 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept msg = "Plan not optimized by CBO."; } this.ctx.setCboInfo(msg); + // hive.cbo.enable is still true, so analyzeInternal skipped ORDER BY ordinal + // substitution. Resolve them before the legacy planner compiles the ordinal + // as a constant (and then drops the sort entirely). See HIVE-30037. + processPositionAlias(ast, true); sinkOp = super.genOPTree(ast, plannerCtx); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index cadb097c4d4d..d70d6d134a77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -14239,6 +14239,14 @@ private void validateCreateView() // Process the position alias in GROUPBY and ORDERBY void processPositionAlias(ASTNode ast) throws SemanticException { + // When CBO is enabled, ORDER BY ordinals are resolved in CalcitePlanner.genSortByKey. + // If CBO later declines the statement, CalcitePlanner.genOPTree calls this again with + // processOrderByPositionAlias=true so the legacy planner does not compile the ordinal + // as a constant sort key (HIVE-30037). + processPositionAlias(ast, !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)); + } + + void processPositionAlias(ASTNode ast, boolean processOrderByPositionAlias) throws SemanticException { boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS); boolean isGbyByPos = isBothByPos || HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_POSITION_ALIAS); @@ -14301,10 +14309,9 @@ void processPositionAlias(ASTNode ast) throws SemanticException { } } - // replace each of the position alias in ORDERBY with the actual column name, - // if cbo is enabled, orderby position will be processed in genPlan - if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) - && orderbyNode != null) { + // replace each of the position alias in ORDERBY with the actual column name. + // When CBO actually plans the statement, ordinals are resolved in genSortByKey. + if (processOrderByPositionAlias && orderbyNode != null) { isAllCol = false; for (int child_pos = 0; child_pos < selectNode.getChildCount(); ++child_pos) { ASTNode node = (ASTNode) selectNode.getChild(child_pos).getChild(0); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index 59a8c105a5fa..0c72481982fb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -677,6 +678,117 @@ public void testCboAmbiguityChecksAreReturnPathInvariant() throws Exception { } } + // ==== HIVE-30037: ORDER BY ordinals when CBO declines the statement ==== + + @Test + public void testOrderByOrdinalResolvedWhenCboDeclinesTablesample() throws Exception { + assertOrderByOrdinalResolvedOnCboDecline( + "select key from table1 tablesample (2 rows) order by 1 desc"); + } + + @Test + public void testOrderByOrdinalResolvedWhenCboDeclinesSortByLimitSubquery() throws Exception { + assertOrderByOrdinalResolvedOnCboDecline( + "select key from (select key from table1 sort by key limit 5) s order by 1 desc"); + } + + @Test + public void testOrderByOrdinalStillSortedWhenCboHandlesStatement() throws Exception { + AnalyzedQuery analyzed = analyzeQueryWithCbo( + "select key from table1 order by 1 desc"); + assertTrue(analyzed.analyzer.getCboInfo(), + analyzed.analyzer.getCboInfo().contains("Plan optimized by CBO")); + assertReduceSinkHasSortKeys(analyzed.analyzer); + } + + private void assertOrderByOrdinalResolvedOnCboDecline(String query) throws Exception { + AnalyzedQuery analyzed = analyzeQueryWithCbo(query); + assertTrue(analyzed.analyzer.getCboInfo(), + analyzed.analyzer.getCboInfo() != null + && analyzed.analyzer.getCboInfo().contains("not optimized by CBO")); + ASTNode orderByRef = findFirstOrderByRef(analyzed.ast); + assertNotNull("expected an ORDER BY expression in " + query, orderByRef); + assertTrue("ORDER BY ordinal should be substituted, got token type=" + + orderByRef.getType() + " text=" + orderByRef.getText() + + " cboInfo=" + analyzed.analyzer.getCboInfo(), + orderByRef.getType() != HiveParser.Number); + assertReduceSinkHasSortKeys(analyzed.analyzer); + } + + private AnalyzedQuery analyzeQueryWithCbo(String query) throws Exception { + HiveConf cboConf = new HiveConf(conf); + cboConf.setBoolVar(HiveConf.ConfVars.HIVE_CBO_ENABLED, true); + cboConf.setVar(HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); + SessionState.start(cboConf); + Context ctx = new Context(cboConf); + ASTNode astNode = ParseUtils.parse(query, ctx); + QueryState queryState = new QueryState.Builder().withHiveConf(cboConf).build(); + BaseSemanticAnalyzer analyzer = SemanticAnalyzerFactory.get(queryState, astNode); + analyzer.initCtx(ctx); + try { + analyzer.analyze(astNode, ctx); + } finally { + analyzer.endAnalysis(astNode); + } + return new AnalyzedQuery(analyzer, astNode); + } + + private static void assertReduceSinkHasSortKeys(BaseSemanticAnalyzer analyzer) { + assertTrue("expected a SemanticAnalyzer with a sink op", + analyzer instanceof SemanticAnalyzer); + ReduceSinkOperator rs = findReduceSink(((SemanticAnalyzer) analyzer).getSinkOp()); + assertNotNull("expected a ReduceSink for ORDER BY", rs); + assertFalse("ReduceSink should keep the ORDER BY key; empty keys mean the ordinal was dropped", + rs.getConf().getKeyCols() == null || rs.getConf().getKeyCols().isEmpty()); + } + + private static ASTNode findFirstOrderByRef(ASTNode node) { + if (node.getType() == HiveParser.TOK_ORDERBY && node.getChildCount() > 0 + && node.getChild(0).getChildCount() > 0) { + ASTNode colNode = (ASTNode) node.getChild(0).getChild(0); + if (colNode != null && colNode.getChildCount() > 0) { + return (ASTNode) colNode.getChild(0); + } + } + if (node.getChildren() == null) { + return null; + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode found = findFirstOrderByRef((ASTNode) node.getChild(i)); + if (found != null) { + return found; + } + } + return null; + } + + @SuppressWarnings("rawtypes") + private static ReduceSinkOperator findReduceSink(Operator op) { + if (op instanceof ReduceSinkOperator) { + return (ReduceSinkOperator) op; + } + if (op == null || op.getParentOperators() == null) { + return null; + } + for (Operator parent : op.getParentOperators()) { + ReduceSinkOperator found = findReduceSink(parent); + if (found != null) { + return found; + } + } + return null; + } + + private static final class AnalyzedQuery { + final BaseSemanticAnalyzer analyzer; + final ASTNode ast; + + AnalyzedQuery(BaseSemanticAnalyzer analyzer, ASTNode ast) { + this.analyzer = analyzer; + this.ast = ast; + } + } + private static ColumnInfo stringCol(String internalName, String tab, String alias, boolean markedAmbiguous) { ColumnInfo colInfo = new ColumnInfo(internalName, TypeInfoFactory.stringTypeInfo, tab, false); colInfo.setAlias(alias); From 4c2fe60333d748662f89ff66d1162adf9753d75d Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Fri, 18 Sep 2026 07:22:21 +0000 Subject: [PATCH 2/8] HIVE-30037: Fix TestSemanticAnalyzer generic types for ReduceSink walk Co-authored-by: Sankalp Thakur --- .../apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index 0c72481982fb..f20aeec642ce 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -762,15 +762,14 @@ private static ASTNode findFirstOrderByRef(ASTNode node) { return null; } - @SuppressWarnings("rawtypes") - private static ReduceSinkOperator findReduceSink(Operator op) { + private static ReduceSinkOperator findReduceSink(Operator op) { if (op instanceof ReduceSinkOperator) { return (ReduceSinkOperator) op; } if (op == null || op.getParentOperators() == null) { return null; } - for (Operator parent : op.getParentOperators()) { + for (Operator parent : op.getParentOperators()) { ReduceSinkOperator found = findReduceSink(parent); if (found != null) { return found; From 3547a9a58cd85d947b6af2d04a5211fbbc3568cb Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Fri, 18 Sep 2026 07:24:20 +0000 Subject: [PATCH 3/8] HIVE-30037: Assert ORDER BY ordinal substitution on the CBO decline path Pin the AST rewrite (Number -> TOK_TABLE_OR_COL) instead of walking for a ReduceSink, which fetch-task conversion can remove. Co-authored-by: Sankalp Thakur --- .../hive/ql/parse/TestSemanticAnalyzer.java | 38 +++---------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index f20aeec642ce..fb12b2a50997 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -63,7 +63,6 @@ import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -697,8 +696,8 @@ public void testOrderByOrdinalStillSortedWhenCboHandlesStatement() throws Except AnalyzedQuery analyzed = analyzeQueryWithCbo( "select key from table1 order by 1 desc"); assertTrue(analyzed.analyzer.getCboInfo(), - analyzed.analyzer.getCboInfo().contains("Plan optimized by CBO")); - assertReduceSinkHasSortKeys(analyzed.analyzer); + analyzed.analyzer.getCboInfo() != null + && analyzed.analyzer.getCboInfo().contains("Plan optimized by CBO")); } private void assertOrderByOrdinalResolvedOnCboDecline(String query) throws Exception { @@ -708,11 +707,9 @@ private void assertOrderByOrdinalResolvedOnCboDecline(String query) throws Excep && analyzed.analyzer.getCboInfo().contains("not optimized by CBO")); ASTNode orderByRef = findFirstOrderByRef(analyzed.ast); assertNotNull("expected an ORDER BY expression in " + query, orderByRef); - assertTrue("ORDER BY ordinal should be substituted, got token type=" - + orderByRef.getType() + " text=" + orderByRef.getText() - + " cboInfo=" + analyzed.analyzer.getCboInfo(), - orderByRef.getType() != HiveParser.Number); - assertReduceSinkHasSortKeys(analyzed.analyzer); + assertEquals("ORDER BY ordinal should be substituted with the select expression, got text=" + + orderByRef.getText() + " cboInfo=" + analyzed.analyzer.getCboInfo(), + HiveParser.TOK_TABLE_OR_COL, orderByRef.getType()); } private AnalyzedQuery analyzeQueryWithCbo(String query) throws Exception { @@ -733,15 +730,6 @@ private AnalyzedQuery analyzeQueryWithCbo(String query) throws Exception { return new AnalyzedQuery(analyzer, astNode); } - private static void assertReduceSinkHasSortKeys(BaseSemanticAnalyzer analyzer) { - assertTrue("expected a SemanticAnalyzer with a sink op", - analyzer instanceof SemanticAnalyzer); - ReduceSinkOperator rs = findReduceSink(((SemanticAnalyzer) analyzer).getSinkOp()); - assertNotNull("expected a ReduceSink for ORDER BY", rs); - assertFalse("ReduceSink should keep the ORDER BY key; empty keys mean the ordinal was dropped", - rs.getConf().getKeyCols() == null || rs.getConf().getKeyCols().isEmpty()); - } - private static ASTNode findFirstOrderByRef(ASTNode node) { if (node.getType() == HiveParser.TOK_ORDERBY && node.getChildCount() > 0 && node.getChild(0).getChildCount() > 0) { @@ -762,22 +750,6 @@ private static ASTNode findFirstOrderByRef(ASTNode node) { return null; } - private static ReduceSinkOperator findReduceSink(Operator op) { - if (op instanceof ReduceSinkOperator) { - return (ReduceSinkOperator) op; - } - if (op == null || op.getParentOperators() == null) { - return null; - } - for (Operator parent : op.getParentOperators()) { - ReduceSinkOperator found = findReduceSink(parent); - if (found != null) { - return found; - } - } - return null; - } - private static final class AnalyzedQuery { final BaseSemanticAnalyzer analyzer; final ASTNode ast; From fec3e484ef9eaf39e23ec822f8ab6dbbbfd112be Mon Sep 17 00:00:00 2001 From: Sankalp Thakur <31366524+sankalpsthakur@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:45:14 +0000 Subject: [PATCH 4/8] HIVE-30037: Preserve resolved GROUP BY expressions on CBO decline Generated-by: ChatGPT GPT-6 Astra Pro --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 11 ++--- .../hive/ql/parse/TestSemanticAnalyzer.java | 45 +++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 74b7cbb23ab9..60eb8898aa3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -709,7 +709,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept // hive.cbo.enable is still true, so analyzeInternal skipped ORDER BY ordinal // substitution. Resolve them before the legacy planner compiles the ordinal // as a constant (and then drops the sort entirely). See HIVE-30037. - processPositionAlias(ast, true); + processPositionAlias(ast, false, true); sinkOp = super.genOPTree(ast, plannerCtx); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d70d6d134a77..d2d10ebdad45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -14241,12 +14241,13 @@ private void validateCreateView() void processPositionAlias(ASTNode ast) throws SemanticException { // When CBO is enabled, ORDER BY ordinals are resolved in CalcitePlanner.genSortByKey. // If CBO later declines the statement, CalcitePlanner.genOPTree calls this again with - // processOrderByPositionAlias=true so the legacy planner does not compile the ordinal - // as a constant sort key (HIVE-30037). - processPositionAlias(ast, !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)); + // only ORDER BY enabled; GROUP BY expressions have already been resolved and + // must not be interpreted as ordinals a second time (HIVE-30037). + processPositionAlias(ast, true, !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)); } - void processPositionAlias(ASTNode ast, boolean processOrderByPositionAlias) throws SemanticException { + void processPositionAlias(ASTNode ast, boolean processGroupByPositionAlias, + boolean processOrderByPositionAlias) throws SemanticException { boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS); boolean isGbyByPos = isBothByPos || HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_POSITION_ALIAS); @@ -14286,7 +14287,7 @@ void processPositionAlias(ASTNode ast, boolean processOrderByPositionAlias) thro int selectExpCnt = selectNode.getChildCount(); // replace each of the position alias in GROUPBY with the actual column name - if (groupbyNode != null) { + if (processGroupByPositionAlias && groupbyNode != null) { for (int child_pos = 0; child_pos < groupbyNode.getChildCount(); ++child_pos) { ASTNode node = (ASTNode) groupbyNode.getChild(child_pos); if (node.getToken().getType() == HiveParser.Number) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java index fb12b2a50997..8a5bb520ebf8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestSemanticAnalyzer.java @@ -700,6 +700,51 @@ public void testOrderByOrdinalStillSortedWhenCboHandlesStatement() throws Except && analyzed.analyzer.getCboInfo().contains("Plan optimized by CBO")); } + @Test + public void testCboDeclinePreservesOutOfRangeGroupByConstant() throws Exception { + assertGroupByConstantSurvivesCboDecline("100"); + } + + @Test + public void testCboDeclinePreservesInRangeGroupByConstant() throws Exception { + assertGroupByConstantSurvivesCboDecline("2"); + } + + private void assertGroupByConstantSurvivesCboDecline(String constant) throws Exception { + boolean original = conf.getBoolVar(HiveConf.ConfVars.HIVE_GROUPBY_POSITION_ALIAS); + conf.setBoolVar(HiveConf.ConfVars.HIVE_GROUPBY_POSITION_ALIAS, true); + try { + AnalyzedQuery analyzed = analyzeQueryWithCbo("select " + constant + + " as k, count(*) as n from table1 tablesample (2 rows) group by 1 order by 1"); + assertTrue(analyzed.analyzer.getCboInfo(), + analyzed.analyzer.getCboInfo() != null + && analyzed.analyzer.getCboInfo().contains("not optimized by CBO")); + ASTNode groupBy = findFirstNodeOfType(analyzed.ast, HiveParser.TOK_GROUPBY); + assertNotNull(groupBy); + assertEquals(HiveParser.Number, ((ASTNode) groupBy.getChild(0)).getType()); + assertEquals(constant, groupBy.getChild(0).getText()); + ASTNode orderByRef = findFirstOrderByRef(analyzed.ast); + assertNotNull(orderByRef); + assertEquals(HiveParser.Number, orderByRef.getType()); + assertEquals(constant, orderByRef.getText()); + } finally { + conf.setBoolVar(HiveConf.ConfVars.HIVE_GROUPBY_POSITION_ALIAS, original); + } + } + + private static ASTNode findFirstNodeOfType(ASTNode node, int type) { + if (node.getType() == type) { + return node; + } + for (int i = 0; i < node.getChildCount(); i++) { + ASTNode found = findFirstNodeOfType((ASTNode) node.getChild(i), type); + if (found != null) { + return found; + } + } + return null; + } + private void assertOrderByOrdinalResolvedOnCboDecline(String query) throws Exception { AnalyzedQuery analyzed = analyzeQueryWithCbo(query); assertTrue(analyzed.analyzer.getCboInfo(), From 7a19a8a579122b2c1d86d5ad17453ed28d464f44 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Sat, 19 Sep 2026 13:51:21 +0000 Subject: [PATCH 5/8] HIVE-30037: Add row-order evidence for CBO-decline ORDER BY 1 DESC Pin TABLESAMPLE + ORDER BY 1 DESC to return 3, 2, 1 with a MiniLlapLocal qtest and a ql HiveTestEnvSetup mini IT (fetch-task conversion off). Generated-by: Cursor Grok 4.6 Co-authored-by: Sankalp Thakur --- .../TestOrderByOrdinalCboDeclineRows.java | 126 ++++++++++++++++++ .../clientpositive/order_by_pos_cbo_decline.q | 12 ++ .../llap/order_by_pos_cbo_decline.q.out | 28 ++++ 3 files changed, 166 insertions(+) create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java create mode 100644 ql/src/test/queries/clientpositive/order_by_pos_cbo_decline.q create mode 100644 ql/src/test/results/clientpositive/llap/order_by_pos_cbo_decline.q.out diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java new file mode 100644 index 000000000000..7007c6484d70 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.DriverFactory; +import org.apache.hadoop.hive.ql.IDriver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.testutils.HiveTestEnvSetup; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +/** + * HIVE-30037: executed row order when CBO declines the statement. + * + * {@link TestSemanticAnalyzer} already pins AST substitution on this path. + * This class runs the query (Tez mini cluster via {@link HiveTestEnvSetup}) + * and asserts the result is 3, 2, 1 rather than the pre-fix unsorted order. + */ +public class TestOrderByOrdinalCboDeclineRows { + + private static final String TABLE = "hive30037_ob_t"; + private static final String QUERY = + "select d from " + TABLE + " tablesample (5 rows) s order by 1 desc"; + + @ClassRule + public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + + @Rule + public TestRule methodRule = env_setup.getMethodRule(); + + @BeforeClass + public static void beforeClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + driver.run("create table " + TABLE + " (d int)"); + driver.run("insert into " + TABLE + " values (1), (2), (3)"); + } + + @AfterClass + public static void afterClass() throws Exception { + IDriver driver = createDriver(); + dropTables(driver); + } + + public static void dropTables(IDriver driver) throws Exception { + driver.run("drop table if exists " + TABLE); + } + + @Test + public void testTablesampleOrderByOrdinalDescReturns321() throws Exception { + IDriver driver = createDriver(); + driver.run(QUERY); + + String cboInfo = driver.getPlan() == null ? null : driver.getPlan().getCboInfo(); + assertNotNull("expected CBO info after compile", cboInfo); + assertTrue("CBO should decline TABLESAMPLE, got cboInfo=" + cboInfo, + cboInfo.contains("not optimized by CBO")); + + assertEquals(Arrays.asList("3", "2", "1"), fetchRows(driver)); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static List fetchRows(IDriver driver) throws Exception { + driver.setMaxRows(100); + List rows = new ArrayList(); + if (driver.getFetchTask() != null) { + driver.getFetchTask().setMaxRows(100); + List batch = new ArrayList(); + while (driver.getFetchTask().fetch(batch)) { + rows.addAll(batch); + batch.clear(); + } + } else { + List batch = new ArrayList(); + while (driver.getResults(batch)) { + rows.addAll(batch); + batch.clear(); + } + } + List out = new ArrayList<>(); + for (Object row : rows) { + out.add(String.valueOf(row)); + } + return out; + } + + private static IDriver createDriver() { + HiveConf conf = env_setup.getTestCtx().hiveConf; + conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, + "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED, true); + HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_FETCH_TASK_CONVERSION, "none"); + SessionState.start(conf); + return DriverFactory.newDriver(conf); + } +} diff --git a/ql/src/test/queries/clientpositive/order_by_pos_cbo_decline.q b/ql/src/test/queries/clientpositive/order_by_pos_cbo_decline.q new file mode 100644 index 000000000000..26d27618f674 --- /dev/null +++ b/ql/src/test/queries/clientpositive/order_by_pos_cbo_decline.q @@ -0,0 +1,12 @@ +-- HIVE-30037: CBO declines TABLESAMPLE; ORDER BY 1 DESC must return 3, 2, 1. +-- MiniLlapLocal: mvn test -pl itests/qtest -Dtest=TestMiniLlapLocalCliDriver -Dqfile=order_by_pos_cbo_decline.q +-- Overwrite golden: add -Dtest.output.overwrite=true + +set hive.fetch.task.conversion=none; +set hive.cbo.enable=true; + +create table hive30037_ob_t (d int); + +insert into hive30037_ob_t values (1), (2), (3); + +select d from hive30037_ob_t tablesample (5 rows) s order by 1 desc; diff --git a/ql/src/test/results/clientpositive/llap/order_by_pos_cbo_decline.q.out b/ql/src/test/results/clientpositive/llap/order_by_pos_cbo_decline.q.out new file mode 100644 index 000000000000..177dcfa96d66 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/order_by_pos_cbo_decline.q.out @@ -0,0 +1,28 @@ +PREHOOK: query: create table hive30037_ob_t (d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@hive30037_ob_t +POSTHOOK: query: create table hive30037_ob_t (d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@hive30037_ob_t +PREHOOK: query: insert into hive30037_ob_t values (1), (2), (3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@hive30037_ob_t +POSTHOOK: query: insert into hive30037_ob_t values (1), (2), (3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@hive30037_ob_t +POSTHOOK: Lineage: hive30037_ob_t.d SCRIPT [] +PREHOOK: query: select d from hive30037_ob_t tablesample (5 rows) s order by 1 desc +PREHOOK: type: QUERY +PREHOOK: Input: default@hive30037_ob_t +#### A masked pattern was here #### +POSTHOOK: query: select d from hive30037_ob_t tablesample (5 rows) s order by 1 desc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@hive30037_ob_t +#### A masked pattern was here #### +3 +2 +1 From 61f49d35154fd1ed3d8e982b69a6e68ba662405f Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Sat, 19 Sep 2026 14:01:22 +0000 Subject: [PATCH 6/8] HIVE-30037: Assert CBO decline via EXPLAIN CBO in row-order IT QueryPlan is released after Driver.run, so read CBO decline from EXPLAIN CBO output before fetching 3, 2, 1. Generated-by: Cursor Grok 4.6 Co-authored-by: Sankalp Thakur --- .../ql/parse/TestOrderByOrdinalCboDeclineRows.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java index 7007c6484d70..be69751f9aec 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.util.ArrayList; @@ -78,13 +77,13 @@ public static void dropTables(IDriver driver) throws Exception { @Test public void testTablesampleOrderByOrdinalDescReturns321() throws Exception { IDriver driver = createDriver(); - driver.run(QUERY); - String cboInfo = driver.getPlan() == null ? null : driver.getPlan().getCboInfo(); - assertNotNull("expected CBO info after compile", cboInfo); - assertTrue("CBO should decline TABLESAMPLE, got cboInfo=" + cboInfo, - cboInfo.contains("not optimized by CBO")); + driver.run("explain cbo " + QUERY); + List explain = fetchRows(driver); + assertTrue("CBO should decline TABLESAMPLE, explain=" + explain, + explain.stream().anyMatch(line -> line.contains("not optimized by CBO"))); + driver.run(QUERY); assertEquals(Arrays.asList("3", "2", "1"), fetchRows(driver)); } From deb19e3bd09c9c548fd8f5fa4e30892dd1e27714 Mon Sep 17 00:00:00 2001 From: Sankalp Thakur Date: Sat, 19 Sep 2026 14:03:43 +0000 Subject: [PATCH 7/8] HIVE-30037: Fetch SELECT rows the same way as TestHiveTestEnvSetup EXPLAIN CBO does not leave a FetchTask after Driver.run. Pin executed order 3, 2, 1 from the TABLESAMPLE query itself. Generated-by: Cursor Grok 4.6 Co-authored-by: Sankalp Thakur --- .../parse/TestOrderByOrdinalCboDeclineRows.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java index be69751f9aec..604ee203466d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.parse; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Arrays; @@ -77,14 +76,10 @@ public static void dropTables(IDriver driver) throws Exception { @Test public void testTablesampleOrderByOrdinalDescReturns321() throws Exception { IDriver driver = createDriver(); - - driver.run("explain cbo " + QUERY); - List explain = fetchRows(driver); - assertTrue("CBO should decline TABLESAMPLE, explain=" + explain, - explain.stream().anyMatch(line -> line.contains("not optimized by CBO"))); - driver.run(QUERY); - assertEquals(Arrays.asList("3", "2", "1"), fetchRows(driver)); + List rows = fetchRows(driver); + assertEquals("TABLESAMPLE + ORDER BY 1 DESC should return 3,2,1; fetchTask=" + + driver.getFetchTask(), Arrays.asList("3", "2", "1"), rows); } @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -93,11 +88,7 @@ private static List fetchRows(IDriver driver) throws Exception { List rows = new ArrayList(); if (driver.getFetchTask() != null) { driver.getFetchTask().setMaxRows(100); - List batch = new ArrayList(); - while (driver.getFetchTask().fetch(batch)) { - rows.addAll(batch); - batch.clear(); - } + driver.getFetchTask().fetch(rows); } else { List batch = new ArrayList(); while (driver.getResults(batch)) { From 12e6f018b3d07151d55c5704db37c2f199cbf80d Mon Sep 17 00:00:00 2001 From: Sankalp Thakur <31366524+sankalpsthakur@users.noreply.github.com> Date: Sun, 20 Sep 2026 07:40:09 +0000 Subject: [PATCH 8/8] HIVE-30037: Use camelCase for the row-order test fixture Rename env_setup and its two references to envSetup to satisfy StaticVariableNameCheck. Only three identifier tokens change; query, assertions, annotations, and fixture lifecycle are unchanged. Validation: exact reported naming regex fails before and passes after; Java token comparison confirms no other token changes. Full Hive tests were not rerun locally for this test-fixture rename. Generated-by: ChatGPT --- .../hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java index 604ee203466d..8ba00de5ac90 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestOrderByOrdinalCboDeclineRows.java @@ -50,10 +50,10 @@ public class TestOrderByOrdinalCboDeclineRows { "select d from " + TABLE + " tablesample (5 rows) s order by 1 desc"; @ClassRule - public static HiveTestEnvSetup env_setup = new HiveTestEnvSetup(); + public static HiveTestEnvSetup envSetup = new HiveTestEnvSetup(); @Rule - public TestRule methodRule = env_setup.getMethodRule(); + public TestRule methodRule = envSetup.getMethodRule(); @BeforeClass public static void beforeClass() throws Exception { @@ -104,7 +104,7 @@ private static List fetchRows(IDriver driver) throws Exception { } private static IDriver createDriver() { - HiveConf conf = env_setup.getTestCtx().hiveConf; + HiveConf conf = envSetup.getTestCtx().hiveConf; conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);