diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index ad34eddbc1ae..2822c66d1c1e 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -48,10 +48,10 @@ catch (RecognitionException e) { //----------------------------------------------------------------------------------- tableAllColumns - : STAR - -> ^(TOK_ALLCOLREF) - | tableName DOT STAR - -> ^(TOK_ALLCOLREF tableName) + : STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)? + -> ^(TOK_ALLCOLREF columnNameList?) + | tableName DOT STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)? + -> ^(TOK_ALLCOLREF tableName columnNameList?) ; // (table|column) diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g index cb3404587d86..7ec8cacba327 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g @@ -398,6 +398,7 @@ KW_SPEC: 'SPEC'; KW_SYSTEM_TIME: 'SYSTEM_TIME'; KW_SYSTEM_VERSION: 'SYSTEM_VERSION'; KW_EXPIRE_SNAPSHOTS: 'EXPIRE_SNAPSHOTS'; +KW_EXCLUDE: 'EXCLUDE'; KW_SET_CURRENT_SNAPSHOT: 'SET_CURRENT_SNAPSHOT'; KW_BRANCH: 'BRANCH'; KW_SNAPSHOTS: 'SNAPSHOTS'; diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index ac9053cf284e..3bfe39536930 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -1026,6 +1026,7 @@ nonReserved | KW_TRIM | KW_SPEC | KW_SYSTEM_TIME | KW_SYSTEM_VERSION + | KW_EXCLUDE | KW_EXPIRE_SNAPSHOTS | KW_SET_CURRENT_SNAPSHOT | KW_BRANCH | KW_SNAPSHOTS | KW_RETAIN | KW_RETENTION diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 25489f477f55..8a79011d4614 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -4366,8 +4366,9 @@ private Pair internalGenSelectLogicalPlan(QB qb, RelNode s // 6.4 Build ExprNode corresponding to colums if (expr.getType() == HiveParser.TOK_ALLCOLREF) { - pos = genRexNodeRegex(".*", - expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), + // Parse SELECT * EXCLUDE columns and pass them to the Calcite engine for exclusion + String starTabAlias = SemanticAnalyzer.processAllColRefAndExclude(expr, inputRR, excludedColumns); + pos = genRexNodeRegex(".*", starTabAlias, expr, columnList, excludedColumns, inputRR, starRR, pos, outputRR, qb.getAliases(), true); } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b728bd854023..46b2981ad8a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -4733,6 +4733,35 @@ static boolean isRegex(String pattern, HiveConf conf) { return false; } + public static String processAllColRefAndExclude( + ASTNode expr, RowResolver inputRR, Set excludedColumns) throws SemanticException { + // Check if the query uses SELECT * EXCLUDE. If it does, grab the table + // alias (like t.*) and build a list of the columns the user wants to exclude. + String starTabAlias = null; + ASTNode excludeNode = null; + if (expr.getChildCount() > 0) { + ASTNode firstChild = (ASTNode) expr.getChild(0); + if (firstChild.getType() == HiveParser.TOK_TABCOLNAME) { + excludeNode = firstChild; + } else { + starTabAlias = getUnescapedName(firstChild).toLowerCase(); + if (expr.getChildCount() > 1) { + excludeNode = (ASTNode) expr.getChild(1); + } + } + } + + if (excludeNode != null) { + for (int e = 0; e < excludeNode.getChildCount(); e++) { + String excludeColName = unescapeIdentifier(excludeNode.getChild(e).getText()).toLowerCase(); + ColumnInfo colInfo = inputRR.get(starTabAlias, excludeColName); + if (colInfo != null) { + excludedColumns.add(colInfo); + } + } + } + return starTabAlias; + } private Operator genSelectPlan(String dest, QB qb, Operator input, Operator inputForSelectStar) throws SemanticException { @@ -4910,9 +4939,15 @@ private Operator genSelectPlan(String dest, ASTNode selExprList, QB qb, Opera // The real expression if (expr.getType() == HiveParser.TOK_ALLCOLREF) { int initPos = pos; - pos = genExprNodeDescRegex(".*", expr.getChildCount() == 0 ? null - : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(), - expr, colList, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); + + Set excludeCols = new HashSet<>(); + String starTabAlias = processAllColRefAndExclude(expr, inputRR, excludeCols); + if (excludeCols.isEmpty()) { + excludeCols = null; + } + + pos = genExprNodeDescRegex(".*", starTabAlias, + expr, colList, excludeCols, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false); if (unparseTranslator.isEnabled()) { offset += pos - initPos - 1; } diff --git a/ql/src/test/queries/clientpositive/select_exclude.q b/ql/src/test/queries/clientpositive/select_exclude.q new file mode 100644 index 000000000000..1ed816fd786f --- /dev/null +++ b/ql/src/test/queries/clientpositive/select_exclude.q @@ -0,0 +1,37 @@ +CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +); + +INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100'); +INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200'); + +-- Exclude a single column +EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude; +SELECT * EXCLUDE (email) FROM test_exclude; + +-- Exclude multiple columns +EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude; +SELECT * EXCLUDE (email, address, phone) FROM test_exclude; + +-- Exclude with table alias +EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t; +SELECT t.* EXCLUDE (id, phone) FROM test_exclude t; + +-- Exclude with JOIN +CREATE TABLE test_exclude_join ( + id INT, + department STRING +); +INSERT INTO test_exclude_join VALUES (1, 'Engineering'); +INSERT INTO test_exclude_join VALUES (2, 'Sales'); + +EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id; + +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id; diff --git a/ql/src/test/results/clientpositive/llap/select_exclude.q.out b/ql/src/test/results/clientpositive/llap/select_exclude.q.out new file mode 100644 index 000000000000..852e58541792 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/select_exclude.q.out @@ -0,0 +1,292 @@ +PREHOOK: query: CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_exclude +POSTHOOK: query: CREATE TABLE test_exclude ( + id INT, + name STRING, + email STRING, + address STRING, + phone STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_exclude +PREHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude +POSTHOOK: query: INSERT INTO test_exclude VALUES (1, 'Alice', 'alice@test.com', '123 Apple St', '555-0100') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude +POSTHOOK: Lineage: test_exclude.address SCRIPT [] +POSTHOOK: Lineage: test_exclude.email SCRIPT [] +POSTHOOK: Lineage: test_exclude.id SCRIPT [] +POSTHOOK: Lineage: test_exclude.name SCRIPT [] +POSTHOOK: Lineage: test_exclude.phone SCRIPT [] +PREHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude +POSTHOOK: query: INSERT INTO test_exclude VALUES (2, 'Bob', 'bob@test.com', '456 Banana Ave', '555-0200') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude +POSTHOOK: Lineage: test_exclude.address SCRIPT [] +POSTHOOK: Lineage: test_exclude.email SCRIPT [] +POSTHOOK: Lineage: test_exclude.id SCRIPT [] +POSTHOOK: Lineage: test_exclude.name SCRIPT [] +POSTHOOK: Lineage: test_exclude.phone SCRIPT [] +PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: test_exclude + Select Operator + expressions: id (type: int), name (type: string), address (type: string), phone (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +1 Alice 123 Apple St 555-0100 +2 Bob 456 Banana Ave 555-0200 +PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: test_exclude + Select Operator + expressions: id (type: int), name (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +1 Alice +2 Bob +PREHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: t + Select Operator + expressions: name (type: string), email (type: string), address (type: string) + outputColumnNames: _col0, _col1, _col2 + ListSink + +PREHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +#### A masked pattern was here #### +POSTHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +#### A masked pattern was here #### +Alice alice@test.com 123 Apple St +Bob bob@test.com 456 Banana Ave +PREHOOK: query: CREATE TABLE test_exclude_join ( + id INT, + department STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: CREATE TABLE test_exclude_join ( + id INT, + department STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_exclude_join +PREHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude_join +POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] +POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] +PREHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_exclude_join +POSTHOOK: query: INSERT INTO test_exclude_join VALUES (2, 'Sales') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_exclude_join +POSTHOOK: Lineage: test_exclude_join.department SCRIPT [] +POSTHOOK: Lineage: test_exclude_join.id SCRIPT [] +PREHOOK: query: EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +PREHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +POSTHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), name (type: string), email (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string), _col2 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + filterExpr: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: id is not null (type: boolean) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), department (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: int) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2 Data size: 198 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col4 + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +PREHOOK: type: QUERY +PREHOOK: Input: default@test_exclude +PREHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +POSTHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id) +FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_exclude +POSTHOOK: Input: default@test_exclude_join +#### A masked pattern was here #### +1 Alice alice@test.com Engineering +2 Bob bob@test.com Sales