diff --git a/iceberg/iceberg-handler/src/test/queries/positive/variant_cast.q b/iceberg/iceberg-handler/src/test/queries/positive/variant_cast.q new file mode 100644 index 000000000000..752c443b1759 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/variant_cast.q @@ -0,0 +1,47 @@ +set hive.explain.user=false; + +drop table if exists variant_cast; + +CREATE EXTERNAL TABLE variant_cast ( + id INT, + v VARIANT +) STORED BY ICEBERG +TBLPROPERTIES ( + 'format-version'='3' +); + +-- typed variant construction: values JSON cannot express keep their real type +INSERT INTO variant_cast VALUES +(1, CAST(true AS VARIANT)), +(2, CAST(42 AS VARIANT)), +(3, CAST(1.5BD AS VARIANT)), +(4, CAST(unhex('DEADBEEF') AS VARIANT)), +(5, CAST(DATE '2026-07-28' AS VARIANT)), +(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)), +(7, CAST('plain' AS VARIANT)), +(8, CAST(array(1, 2, 3) AS VARIANT)), +(9, CAST(NULL AS VARIANT)); + +SELECT id, v FROM variant_cast ORDER BY id; + +-- binary node is rendered as base64, matching Impala/Trino +SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4; + +-- cast over a column expression, not a constant +drop table if exists variant_src; + +CREATE TABLE variant_src (id INT, s STRING); + +INSERT INTO variant_src VALUES (11, 'from_col'); + +EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src; + +INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src; + +SELECT id, v FROM variant_cast WHERE id > 10; + +-- identity cast over a reader-produced variant column +SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10; + +drop table variant_cast; +drop table variant_src; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/variant_cast.q.out b/iceberg/iceberg-handler/src/test/results/positive/variant_cast.q.out new file mode 100644 index 000000000000..17daa4fba487 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/variant_cast.q.out @@ -0,0 +1,203 @@ +PREHOOK: query: drop table if exists variant_cast +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists variant_cast +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE EXTERNAL TABLE variant_cast ( + id INT, + v VARIANT +) STORED BY ICEBERG +TBLPROPERTIES ( + 'format-version'='3' +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@variant_cast +POSTHOOK: query: CREATE EXTERNAL TABLE variant_cast ( + id INT, + v VARIANT +) STORED BY ICEBERG +TBLPROPERTIES ( + 'format-version'='3' +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@variant_cast +PREHOOK: query: INSERT INTO variant_cast VALUES +(1, CAST(true AS VARIANT)), +(2, CAST(42 AS VARIANT)), +(3, CAST(1.5BD AS VARIANT)), +(4, CAST(unhex('DEADBEEF') AS VARIANT)), +(5, CAST(DATE '2026-07-28' AS VARIANT)), +(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)), +(7, CAST('plain' AS VARIANT)), +(8, CAST(array(1, 2, 3) AS VARIANT)), +(9, CAST(NULL AS VARIANT)) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@variant_cast +POSTHOOK: query: INSERT INTO variant_cast VALUES +(1, CAST(true AS VARIANT)), +(2, CAST(42 AS VARIANT)), +(3, CAST(1.5BD AS VARIANT)), +(4, CAST(unhex('DEADBEEF') AS VARIANT)), +(5, CAST(DATE '2026-07-28' AS VARIANT)), +(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)), +(7, CAST('plain' AS VARIANT)), +(8, CAST(array(1, 2, 3) AS VARIANT)), +(9, CAST(NULL AS VARIANT)) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@variant_cast +PREHOOK: query: SELECT id, v FROM variant_cast ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_cast +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, v FROM variant_cast ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_cast +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 true +2 42 +3 1.5 +4 "3q2+7w==" +5 "2026-07-28" +6 "2026-07-28 10:00:00" +7 "plain" +8 [1,2,3] +9 NULL +PREHOOK: query: SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4 +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_cast +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_cast +POSTHOOK: Output: hdfs://### HDFS PATH ### +3q2+7w== +PREHOOK: query: drop table if exists variant_src +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists variant_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE TABLE variant_src (id INT, s STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@variant_src +POSTHOOK: query: CREATE TABLE variant_src (id INT, s STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@variant_src +PREHOOK: query: INSERT INTO variant_src VALUES (11, 'from_col') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@variant_src +POSTHOOK: query: INSERT INTO variant_src VALUES (11, 'from_col') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@variant_src +POSTHOOK: Lineage: variant_src.id SCRIPT [] +POSTHOOK: Lineage: variant_src.s SCRIPT [] +PREHOOK: query: EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_src +PREHOOK: Output: default@variant_cast +POSTHOOK: query: EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_src +POSTHOOK: Output: default@variant_cast +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: variant_src + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: id (type: int), CAST( s AS VARIANT) (type: struct) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.variant_cast + Execution mode: vectorized + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: false + table: + input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat + output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat + serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe + name: default.variant_cast + + Stage: Stage-3 + Stats Work + Basic Stats Work: + +PREHOOK: query: INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_src +PREHOOK: Output: default@variant_cast +POSTHOOK: query: INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_src +POSTHOOK: Output: default@variant_cast +PREHOOK: query: SELECT id, v FROM variant_cast WHERE id > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_cast +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, v FROM variant_cast WHERE id > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_cast +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 "from_col" +PREHOOK: query: SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@variant_cast +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@variant_cast +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 "from_col" +PREHOOK: query: drop table variant_cast +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@variant_cast +PREHOOK: Output: database:default +PREHOOK: Output: default@variant_cast +POSTHOOK: query: drop table variant_cast +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@variant_cast +POSTHOOK: Output: database:default +POSTHOOK: Output: default@variant_cast +PREHOOK: query: drop table variant_src +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@variant_src +PREHOOK: Output: database:default +PREHOOK: Output: default@variant_src +POSTHOOK: query: drop table variant_src +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@variant_src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@variant_src diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index ecdb0f568692..cb7246aa1ae5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -788,6 +788,7 @@ public final class FunctionRegistry { system.registerFunction("ST_Z", ST_Z.class); // Iceberg UDFs + system.registerGenericUDF(serdeConstants.VARIANT_TYPE_NAME, GenericUDFToVariant.class); system.registerGenericUDF("parse_json", GenericUDFParseJson.class); system.registerGenericUDF("to_json", GenericUDFToJson.class); system.registerGenericUDF("variant_get", GenericUDFVariantGet.class); @@ -1637,7 +1638,7 @@ public static boolean isOpCast(GenericUDF genericUDF) { udfClass == GenericUDFToVarchar.class || udfClass == GenericUDFToChar.class || udfClass == GenericUDFTimestamp.class || udfClass == GenericUDFToBinary.class || udfClass == GenericUDFToDate.class || udfClass == GenericUDFToDecimal.class || - udfClass == GenericUDFToTimestampLocalTZ.class; + udfClass == GenericUDFToTimestampLocalTZ.class || udfClass == GenericUDFToVariant.class; } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java index 1eb1d17297f7..598d144f7fbb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java @@ -139,6 +139,8 @@ public class TypeCheckProcFactory { serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_DECIMAL, serdeConstants.DECIMAL_TYPE_NAME); + CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_VARIANT, + serdeConstants.VARIANT_TYPE_NAME); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_MAP, "toMap"); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_LIST, "toArray"); CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_STRUCT, "toStruct"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVariant.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVariant.java new file mode 100644 index 000000000000..d3b5a5598bd9 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToVariant.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.VariantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.variant.Variant; +import org.apache.hadoop.hive.serde2.variant.VariantBuilder; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +/** + * Converts a value to a variant, preserving the value's type as the corresponding variant + * primitive; the typed counterpart of parse_json for types JSON cannot express (binary, + * date, timestamp, decimal). Mirrors Spark's cast to variant: primitive, array and variant + * inputs are accepted; struct and map inputs are not. Timestamps are truncated to + * microseconds, the variant format's precision. + */ +@Description(name = "variant", value = "_FUNC_(x) - Converts x to a VARIANT, preserving its type", extended = """ + Example: + > SELECT CAST(unhex('DEADBEEF') AS VARIANT); + "3q2+7w==\"""") +public class GenericUDFToVariant extends GenericUDF { + private static final String VARIANT_TYPE_NAME = VariantObjectInspector.get().getTypeName(); + + private static final EnumSet SUPPORTED_PRIMITIVES = EnumSet.of( + PrimitiveCategory.VOID, PrimitiveCategory.BOOLEAN, PrimitiveCategory.BYTE, PrimitiveCategory.SHORT, + PrimitiveCategory.INT, PrimitiveCategory.LONG, PrimitiveCategory.FLOAT, PrimitiveCategory.DOUBLE, + PrimitiveCategory.DECIMAL, PrimitiveCategory.STRING, PrimitiveCategory.CHAR, PrimitiveCategory.VARCHAR, + PrimitiveCategory.BINARY, PrimitiveCategory.DATE, PrimitiveCategory.TIMESTAMP, + PrimitiveCategory.TIMESTAMPLOCALTZ); + + private transient ObjectInspector inputOI; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException("variant requires one argument"); + } + checkSupported(arguments[0]); + inputOI = arguments[0]; + return ObjectInspectorFactory.getVariantObjectInspector(); + } + + private static void checkSupported(ObjectInspector oi) throws UDFArgumentException { + boolean supported = switch (oi.getCategory()) { + case PRIMITIVE -> SUPPORTED_PRIMITIVES.contains(((PrimitiveObjectInspector) oi).getPrimitiveCategory()); + case LIST -> { + checkSupported(((ListObjectInspector) oi).getListElementObjectInspector()); + yield true; + } + case STRUCT -> VARIANT_TYPE_NAME.equals(oi.getTypeName()); + default -> false; + }; + if (!supported) { + throw new UDFArgumentTypeException(0, "Cannot cast " + oi.getTypeName() + " to VARIANT"); + } + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object value = arguments[0].get(); + if (value == null) { + return null; + } + try { + VariantBuilder builder = new VariantBuilder(false); + build(builder, value, inputOI); + Variant variant = builder.result(); + return List.of(variant.getMetadata(), variant.getValue()); + } catch (RuntimeException e) { + throw new HiveException("Cannot cast value to VARIANT: " + e.getMessage(), e); + } + } + + private static void build(VariantBuilder builder, Object o, ObjectInspector oi) throws HiveException { + if (o == null) { + builder.appendNull(); + return; + } + switch (oi.getCategory()) { + case PRIMITIVE: { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + switch (poi.getPrimitiveCategory()) { + case VOID: + builder.appendNull(); + break; + case BOOLEAN: + builder.appendBoolean(((BooleanObjectInspector) poi).get(o)); + break; + case BYTE: + case SHORT: + case INT: + case LONG: + builder.appendLong(PrimitiveObjectInspectorUtils.getLong(o, poi)); + break; + case FLOAT: + builder.appendFloat(((FloatObjectInspector) poi).get(o)); + break; + case DOUBLE: + builder.appendDouble(((DoubleObjectInspector) poi).get(o)); + break; + case DECIMAL: + builder.appendDecimal(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o).bigDecimalValue()); + break; + case STRING: + case CHAR: + case VARCHAR: + builder.appendString(PrimitiveObjectInspectorUtils.getString(o, poi)); + break; + case BINARY: + builder.appendBinary(((BinaryObjectInspector) poi).getPrimitiveJavaObject(o)); + break; + case DATE: + builder.appendDate(((DateObjectInspector) poi).getPrimitiveJavaObject(o).toEpochDay()); + break; + case TIMESTAMP: + builder.appendTimestampNtz(((TimestampObjectInspector) poi).getPrimitiveJavaObject(o).toEpochMicro()); + break; + case TIMESTAMPLOCALTZ: { + Instant instant = ((TimestampLocalTZObjectInspector) poi).getPrimitiveJavaObject(o) + .getZonedDateTime().toInstant(); + builder.appendTimestamp(instant.getEpochSecond() * 1_000_000 + instant.getNano() / 1_000); + break; + } + default: + throw new HiveException("Cannot cast " + poi.getTypeName() + " to VARIANT"); + } + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + int start = builder.getWritePos(); + int length = loi.getListLength(o); + List offsets = new ArrayList<>(length); + for (int i = 0; i < length; i++) { + offsets.add(builder.getWritePos() - start); + build(builder, loi.getListElement(o, i), loi.getListElementObjectInspector()); + } + builder.finishWritingArray(start, offsets); + break; + } + case STRUCT: + // the variant shape, guaranteed by initialize: embed as-is + builder.appendVariant(Variant.from(((StructObjectInspector) oi).getStructFieldsDataAsList(o))); + break; + default: + throw new HiveException("Cannot cast " + oi.getTypeName() + " to VARIANT"); + } + } + + @Override + public String getDisplayString(String[] children) { + return "CAST( " + children[0] + " AS VARIANT)"; + } +} \ No newline at end of file diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToVariant.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToVariant.java new file mode 100644 index 000000000000..ce3174c1f9c0 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToVariant.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.variant.Variant; +import org.apache.hadoop.hive.serde2.variant.VariantBuilder; +import org.junit.jupiter.api.Test; + +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class TestGenericUDFToVariant { + + @SuppressWarnings("unchecked") + private static String toJson(String typeString, Object value) throws Exception { + GenericUDFToVariant udf = new GenericUDFToVariant(); + udf.initialize(new ObjectInspector[] { TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo( + TypeInfoUtils.getTypeInfoFromTypeString(typeString)) }); + Object result = udf.evaluate(new GenericUDF.DeferredObject[] { new GenericUDF.DeferredJavaObject(value) }); + if (result == null) { + return null; + } + List variant = (List) result; + return new Variant(variant.get(1), variant.get(0)).toJson(ZoneOffset.UTC); + } + + @Test + public void testPrimitives() throws Exception { + assertEquals("true", toJson("boolean", true)); + assertEquals("42", toJson("int", 42)); + assertEquals("9007199254740993", toJson("bigint", 9007199254740993L)); + assertEquals("1.5", toJson("double", 1.5d)); + assertEquals("1.5", toJson("decimal(4,2)", HiveDecimal.create("1.50"))); + assertEquals("\"John\"", toJson("string", "John")); + assertEquals("\"ab\"", toJson("varchar(5)", new HiveVarchar("ab", 5))); + // char padding is stripped, consistent with Hive's char-to-string coercion + assertEquals("\"ab\"", toJson("char(5)", new HiveChar("ab", 5))); + assertNull(toJson("int", null)); + } + + @Test + public void testBinaryBecomesTypedNode() throws Exception { + // base64 of DEADBEEF; a real BINARY node, unreachable via parse_json + assertEquals("\"3q2+7w==\"", toJson("binary", new byte[] { (byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF })); + } + + @Test + public void testDateAndTimestamp() throws Exception { + assertEquals("\"2026-07-28\"", toJson("date", Date.valueOf("2026-07-28"))); + assertEquals("\"2026-07-28 10:00:00\"", toJson("timestamp", Timestamp.valueOf("2026-07-28 10:00:00"))); + } + + @Test + public void testArrayWithNulls() throws Exception { + assertEquals("[1,null,3]", toJson("array", Arrays.asList(1, null, 3))); + } + + @Test + public void testVariantInputIsEmbeddedAsIs() throws Exception { + Variant v = VariantBuilder.parseJson("{\"k\":1}", false); + assertEquals("{\"k\":1}", toJson("variant", List.of(v.getMetadata(), v.getValue()))); + } + + @Test + public void testStructAndMapRejected() { + assertThrows(UDFArgumentTypeException.class, () -> toJson("struct", null)); + assertThrows(UDFArgumentTypeException.class, () -> toJson("map", null)); + } + + @Test + public void testMalformedVariantInputThrows() { + // strict like parse_json and Spark's cast: bytes that are not variant encoding fail loudly + assertThrows(HiveException.class, + () -> toJson("struct", Arrays.asList(new byte[] { 9 }, new byte[] { 1 }))); + } +} \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/show_functions.q.out b/ql/src/test/results/clientpositive/llap/show_functions.q.out index 605e953d6349..dd17569bf0ac 100644 --- a/ql/src/test/results/clientpositive/llap/show_functions.q.out +++ b/ql/src/test/results/clientpositive/llap/show_functions.q.out @@ -491,6 +491,7 @@ validate_acid_sort_order var_pop var_samp variance +variant variant_get version weekofyear @@ -1133,6 +1134,7 @@ validate_acid_sort_order var_pop var_samp variance +variant variant_get version weekofyear