diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index a905905c098e..b669b5201cdc 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -1495,7 +1495,7 @@ primitiveType dataType : complex=ARRAY (LT dataType GT)? #complexDataType | complex=MAP (LT dataType COMMA dataType GT)? #complexDataType - | complex=STRUCT ((LT complexColTypeList? GT) | NEQ)? #complexDataType + | complex=STRUCT ((LT complexColTypeList? GT) | NEQ {((SqlBaseLexer) getTokenStream().getTokenSource()).decComplexTypeLevelCounter();})? #complexDataType | primitiveType #primitiveDataType ; diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index 94e60db67ac7..3d064c904f19 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -1164,4 +1164,27 @@ class SparkSqlParserSuite extends AnalysisTest with SharedSparkSession { } } } + + test("SPARK-52709: STRUCT<> should not corrupt complex_type_level_counter") { + // STRUCT<> is tokenized as STRUCT + NEQ by the lexer. The parser must decrement + // the complex_type_level_counter so that subsequent >> is recognized as shift-right. + // Without the fix, this throws a parse error because >> is not recognized. + parser.parsePlan("SELECT CAST(null AS STRUCT<>), 2 >> 1") + + // Multiple empty structs should not corrupt the counter + parser.parsePlan("SELECT CAST(null AS STRUCT<>), CAST(null AS STRUCT<>), 4 >> 2") + + // Empty struct with unsigned shift right + parser.parsePlan("SELECT CAST(null AS STRUCT<>), 8 >>> 2") + + // ARRAY with <> as not-equal operator should still work + parser.parsePlan("SELECT ARRAY(1 <> 2)") + + // Nested complex types with >> should still work + parser.parsePlan("SELECT CAST(null AS MAP>)") + + // Mix of empty struct and nested complex types + parser.parsePlan( + "SELECT CAST(null AS STRUCT<>), CAST(null AS MAP>), 2 >> 1") + } }