You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gs...@apache.org on 2023/08/07 21:11:37 UTC
[hive] branch master updated: HIVE-27523 - Implement array_union UDF in Hive (#4513) (Taraka Rama Rao Lethavadla, reviewed by Okumin, Sai Hemanth Gantasala)
This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b9918becd96 HIVE-27523 - Implement array_union UDF in Hive (#4513) (Taraka Rama Rao Lethavadla, reviewed by Okumin, Sai Hemanth Gantasala)
b9918becd96 is described below
commit b9918becd96a52659c6a99b78cf5531c6800b1d3
Author: tarak271 <ta...@gmail.com>
AuthorDate: Tue Aug 8 02:41:32 2023 +0530
HIVE-27523 - Implement array_union UDF in Hive (#4513) (Taraka Rama Rao Lethavadla, reviewed by Okumin, Sai Hemanth Gantasala)
---
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../hive/ql/udf/generic/GenericUDFArrayUnion.java | 73 +++++++
.../ql/udf/generic/TestGenericUDFArrayUnion.java | 228 +++++++++++++++++++++
.../test/queries/clientpositive/udf_array_union.q | 42 ++++
.../clientpositive/llap/show_functions.q.out | 2 +
.../clientpositive/llap/udf_array_union.q.out | 132 ++++++++++++
6 files changed, 478 insertions(+)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index b7107873445..bdcea8b8436 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -611,6 +611,7 @@ public final class FunctionRegistry {
system.registerGenericUDF("array_slice", GenericUDFArraySlice.class);
system.registerGenericUDF("array_except", GenericUDFArrayExcept.class);
system.registerGenericUDF("array_intersect", GenericUDFArrayIntersect.class);
+ system.registerGenericUDF("array_union", GenericUDFArrayUnion.class);
system.registerGenericUDF("deserialize", GenericUDFDeserialize.class);
system.registerGenericUDF("sentences", GenericUDFSentences.class);
system.registerGenericUDF("map_keys", GenericUDFMapKeys.class);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayUnion.java
new file mode 100644
index 00000000000..00d74d22599
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayUnion.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * GenericUDFArrayUnion
+ */
+@Description(name = "array_union", value = "_FUNC_(array1, array2) - Returns an array of the elements in the union of "
+ + "array1 and array2 without duplicates.", extended = "Example:\n"
+ + " > SELECT _FUNC_(array(1, 2, 2,4), array(2,3)) FROM src LIMIT 1;\n"
+ + " [1,2,3,4]")
+public class GenericUDFArrayUnion extends AbstractGenericUDFArrayBase {
+ static final int ARRAY2_IDX = 1;
+ static final String ERROR_NOT_COMPARABLE = "Input arrays are not comparable to use ARRAY_UNION udf";
+ private static final String FUNC_NAME = "ARRAY_UNION";
+
+ public GenericUDFArrayUnion() {
+ super(FUNC_NAME, 2, 2, ObjectInspector.Category.LIST);
+ }
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ ObjectInspector defaultOI = super.initialize(arguments);
+ //Array1 is already getting validated in Parent class
+ checkArgCategory(arguments, ARRAY2_IDX, ObjectInspector.Category.LIST, FUNC_NAME,
+ org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME);
+ if (!ObjectInspectorUtils.compareTypes(arrayOI.getListElementObjectInspector(),
+ ((ListObjectInspector) arguments[ARRAY2_IDX]).getListElementObjectInspector())) {
+ // check if elements of arrays are comparable
+ throw new UDFArgumentTypeException(1, ERROR_NOT_COMPARABLE);
+ }
+ return defaultOI;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ Object array = arguments[ARRAY_IDX].get();
+ Object array2 = arguments[ARRAY2_IDX].get();
+ if (array == null || array2 == null) {
+ return null;
+ }
+ return Stream.concat(((ListObjectInspector) argumentOIs[ARRAY_IDX]).getList(array).stream(),
+ ((ListObjectInspector) argumentOIs[ARRAY2_IDX]).getList(array2).stream()).distinct()
+ .map(o -> converter.convert(o)).collect(Collectors.toList());
+ }
+}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayUnion.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayUnion.java
new file mode 100644
index 00000000000..24104d3d413
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFArrayUnion.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.io.DateWritableV2;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.*;
+
+import static java.util.Arrays.asList;
+
+public class TestGenericUDFArrayUnion {
+ private final GenericUDFArrayUnion udf = new GenericUDFArrayUnion();
+
+ @Test
+ public void testPrimitive() throws HiveException {
+ ObjectInspector intObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector);
+ ObjectInspector floatObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
+ ObjectInspector doubleObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
+ ObjectInspector longObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableLongObjectInspector);
+ ObjectInspector stringObjectInspector = ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector);
+
+ Object i1 = new IntWritable(1);
+ Object i2 = new IntWritable(2);
+ Object i3 = new IntWritable(4);
+ Object i4 = new IntWritable(5);
+ Object i5 = new IntWritable(1);
+ Object i6 = new IntWritable(3);
+ Object i7 = new IntWritable(2);
+ Object i8 = new IntWritable(9);
+ List<Object> inputList = new ArrayList<>();
+ inputList.add(i1);
+ inputList.add(i2);
+ inputList.add(i3);
+ inputList.add(i4);
+
+ udf.initialize(new ObjectInspector[] { intObjectInspector, intObjectInspector });
+ runAndVerify(inputList, asList(i5, i6, i7, i8), asList(i1, i2, i3, i4, i6, i8));
+
+ i1 = new FloatWritable(3.3f);
+ i2 = new FloatWritable(1.1f);
+ i3 = new FloatWritable(4.3f);
+ i4 = new FloatWritable(2.22f);
+ i5 = new FloatWritable(3.3f);
+ i6 = new FloatWritable(1.1f);
+ i7 = new FloatWritable(2.28f);
+ i8 = new FloatWritable(2.20f);
+ List<Object> inputFloatList = new ArrayList<>();
+ inputFloatList.add(i1);
+ inputFloatList.add(i2);
+ inputFloatList.add(i3);
+ inputFloatList.add(i4);
+
+ udf.initialize(new ObjectInspector[] { floatObjectInspector, floatObjectInspector });
+ runAndVerify(new ArrayList<>(inputFloatList), asList(i5, i6, i7, i8), asList(i1, i2, i3, i4, i7, i8));
+
+ Object s1 = new Text("1");
+ Object s2 = new Text("2");
+ Object s3 = new Text("4");
+ Object s4 = new Text("5");
+ List<Object> inputStringList = new ArrayList<>();
+ inputStringList.add(s1);
+ inputStringList.add(s2);
+ inputStringList.add(s3);
+ inputStringList.add(s4);
+
+ udf.initialize(new ObjectInspector[] { stringObjectInspector, stringObjectInspector });
+ runAndVerify(inputStringList,asList(s1,s3),asList(s1,s2,s3,s4));
+
+ runAndVerify(inputStringList,inputStringList,inputStringList);
+ runAndVerify(inputStringList, Collections.emptyList(),inputStringList);
+ // Empty input arrays
+ runAndVerify(Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
+ // Int & float arrays
+ UDFArgumentTypeException exception = Assert.assertThrows(UDFArgumentTypeException.class,
+ () -> udf.initialize(new ObjectInspector[] { floatObjectInspector, intObjectInspector }));
+ Assert.assertEquals(GenericUDFArrayUnion.ERROR_NOT_COMPARABLE,exception.getMessage());
+ // float and string arrays
+ exception = Assert.assertThrows(UDFArgumentTypeException.class,
+ () -> udf.initialize(new ObjectInspector[] { floatObjectInspector, stringObjectInspector }));
+ Assert.assertEquals(GenericUDFArrayUnion.ERROR_NOT_COMPARABLE,exception.getMessage());
+ // long and double arrays
+ exception = Assert.assertThrows(UDFArgumentTypeException.class,
+ () -> udf.initialize(new ObjectInspector[] { longObjectInspector, doubleObjectInspector }));
+ Assert.assertEquals(GenericUDFArrayUnion.ERROR_NOT_COMPARABLE,exception.getMessage());
+ }
+
+ @Test public void testList() throws HiveException {
+ ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector)),
+ ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector)) };
+ udf.initialize(inputOIs);
+
+ Object i1 = asList(new Text("aa1"), new Text("dd"), new Text("cc"), new Text("bb"));
+ Object i2 = asList(new Text("aa2"), new Text("cc"), new Text("ba"), new Text("dd"));
+ Object i3 = asList(new Text("aa3"), new Text("cc"), new Text("dd"), new Text("ee"));
+ Object i4 = asList(new Text("aa4"), new Text("cc"), new Text("ddd"), new Text("bb"));
+ List<Object> inputList = new ArrayList<>();
+ inputList.add(i1);
+ inputList.add(i2);
+ inputList.add(i3);
+ inputList.add(i4);
+ runAndVerify(inputList, asList(i1, i2, i2), inputList);
+ }
+
+ @Test public void testStruct() throws HiveException {
+ ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getStandardStructObjectInspector(asList("f1", "f2", "f3", "f4"),
+ asList(PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDateObjectInspector,
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector)))),
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getStandardStructObjectInspector(asList("f1", "f2", "f3", "f4"),
+ asList(PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.writableDateObjectInspector,
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector)))) };
+ udf.initialize(inputOIs);
+
+ Object i1 = asList(new Text("a"), new DoubleWritable(3.1415),
+ new DateWritableV2(Date.of(2015, 5, 26)),
+ asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4)));
+
+ Object i2 = asList(new Text("b"), new DoubleWritable(3.14),
+ new DateWritableV2(Date.of(2015, 5, 26)),
+ asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4)));
+
+ Object i3 = asList(new Text("a"), new DoubleWritable(3.1415),
+ new DateWritableV2(Date.of(2015, 5, 25)),
+ asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(5)));
+
+ Object i4 = asList(new Text("a"), new DoubleWritable(3.1415),
+ new DateWritableV2(Date.of(2015, 5, 25)),
+ asList(new IntWritable(1), new IntWritable(3), new IntWritable(2), new IntWritable(4)));
+
+ List<Object> inputList = new ArrayList<>();
+ inputList.add(i1);
+ inputList.add(i2);
+ inputList.add(i3);
+ inputList.add(i4);
+ runAndVerify(inputList, asList(i1, i3), inputList);
+ }
+
+ @Test public void testMap() throws HiveException {
+ ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector(
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector)),
+ ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableIntObjectInspector)) };
+ udf.initialize(inputOIs);
+
+ Map<Text, IntWritable> m1 = new HashMap<>();
+ m1.put(new Text("a"), new IntWritable(4));
+ m1.put(new Text("b"), new IntWritable(3));
+ m1.put(new Text("c"), new IntWritable(1));
+ m1.put(new Text("d"), new IntWritable(2));
+
+ Map<Text, IntWritable> m2 = new HashMap<>();
+ m2.put(new Text("d"), new IntWritable(4));
+ m2.put(new Text("b"), new IntWritable(3));
+ m2.put(new Text("a"), new IntWritable(1));
+ m2.put(new Text("c"), new IntWritable(2));
+
+ Map<Text, IntWritable> m3 = new HashMap<>();
+ m3.put(new Text("d"), new IntWritable(4));
+ m3.put(new Text("b"), new IntWritable(3));
+ m3.put(new Text("a"), new IntWritable(1));
+
+ Map<Text, IntWritable> m4 = new HashMap<>();
+ m4.put(new Text("e"), new IntWritable(4));
+ m4.put(new Text("b"), new IntWritable(3));
+ m4.put(new Text("a"), new IntWritable(1));
+
+ List<Object> inputList = new ArrayList<>();
+ inputList.add(m1);
+ inputList.add(m3);
+ inputList.add(m2);
+ inputList.add(m1);
+
+ runAndVerify(inputList, asList(m1, m4), asList(m1, m3, m2, m4));
+ }
+
+ private void runAndVerify(List<Object> actual, List<Object> actual2, List<Object> expected) throws HiveException {
+ GenericUDF.DeferredJavaObject[] args =
+ { new GenericUDF.DeferredJavaObject(actual), new GenericUDF.DeferredJavaObject(actual2) };
+ List<?> result = (List<?>) udf.evaluate(args);
+ Assert.assertArrayEquals("Check content", expected.toArray(), result.toArray());
+ }
+}
diff --git a/ql/src/test/queries/clientpositive/udf_array_union.q b/ql/src/test/queries/clientpositive/udf_array_union.q
new file mode 100644
index 00000000000..bf0b36481fc
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_array_union.q
@@ -0,0 +1,42 @@
+--! qt:dataset:src
+
+-- SORT_QUERY_RESULTS
+
+set hive.fetch.task.conversion=more;
+
+DESCRIBE FUNCTION array_union;
+DESCRIBE FUNCTION EXTENDED array_union;
+
+-- evaluates function for array of primitives
+SELECT array_union(array(1, 2, 3, null,3,4),array(1, 3, null));
+
+SELECT array_union(array(),array());
+
+SELECT array_union(array(null),array(null));
+
+SELECT array_union(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),array(1.12,3.34,1.11,1.12));
+
+SELECT array(1,2,3),array_union(array(1, 2, 3),array(1,3,4));
+
+SELECT array_union(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),array(1.1234567890, 3.34567890, null,2.234567));
+
+SELECT array_union(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),array(11234567890, 2234567890, 334567890));
+
+SELECT array_union(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null));
+
+# handle null array cases
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/test_null_array;
+
+dfs -copyFromLocal ../../data/files/test_null_array.csv ${system:test.tmp.dir}/test_null_array/;
+
+create external table test_null_array (id int, value Array<String>) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY ':' collection items terminated by ',' location '${system:test.tmp.dir}/test_null_array';
+
+select value from test_null_array;
+
+select array_union(value,value) from test_null_array;
+
+select value, array_union(value,value) from test_null_array;
+
+dfs -rm -r ${system:test.tmp.dir}/test_null_array;
\ No newline at end of file
diff --git a/ql/src/test/results/clientpositive/llap/show_functions.q.out b/ql/src/test/results/clientpositive/llap/show_functions.q.out
index 28c7b37481f..201801c4f59 100644
--- a/ql/src/test/results/clientpositive/llap/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/llap/show_functions.q.out
@@ -54,6 +54,7 @@ array_join
array_max
array_min
array_slice
+array_union
ascii
asin
assert_true
@@ -678,6 +679,7 @@ array_join
array_max
array_min
array_slice
+array_union
ascii
asin
assert_true
diff --git a/ql/src/test/results/clientpositive/llap/udf_array_union.q.out b/ql/src/test/results/clientpositive/llap/udf_array_union.q.out
new file mode 100644
index 00000000000..e5ce200cfe0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/udf_array_union.q.out
@@ -0,0 +1,132 @@
+PREHOOK: query: DESCRIBE FUNCTION array_union
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION array_union
+POSTHOOK: type: DESCFUNCTION
+array_union(array1, array2) - Returns an array of the elements in the union of array1 and array2 without duplicates.
+PREHOOK: query: DESCRIBE FUNCTION EXTENDED array_union
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION EXTENDED array_union
+POSTHOOK: type: DESCFUNCTION
+array_union(array1, array2) - Returns an array of the elements in the union of array1 and array2 without duplicates.
+Example:
+ > SELECT array_union(array(1, 2, 2,4), array(2,3)) FROM src LIMIT 1;
+ [1,2,3,4]
+Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDFArrayUnion
+Function type:BUILTIN
+PREHOOK: query: SELECT array_union(array(1, 2, 3, null,3,4),array(1, 3, null))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(1, 2, 3, null,3,4),array(1, 3, null))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[1,2,3,null,4]
+PREHOOK: query: SELECT array_union(array(),array())
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(),array())
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[]
+PREHOOK: query: SELECT array_union(array(null),array(null))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(null),array(null))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[null]
+PREHOOK: query: SELECT array_union(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),array(1.12,3.34,1.11,1.12))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(1.12, 2.23, 3.34, null,1.11,1.12,2.9),array(1.12,3.34,1.11,1.12))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[1.12,2.23,3.34,null,1.11,2.9]
+PREHOOK: query: SELECT array(1,2,3),array_union(array(1, 2, 3),array(1,3,4))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array(1,2,3),array_union(array(1, 2, 3),array(1,3,4))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[1,2,3] [1,2,3,4]
+PREHOOK: query: SELECT array_union(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),array(1.1234567890, 3.34567890, null,2.234567))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(1.1234567890, 2.234567890, 3.34567890, null, 3.3456789, 2.234567,1.1234567890),array(1.1234567890, 3.34567890, null,2.234567))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[1.123456789,2.23456789,3.3456789,null,2.234567]
+PREHOOK: query: SELECT array_union(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),array(11234567890, 2234567890, 334567890))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(11234567890, 2234567890, 334567890, null, 11234567890, 2234567890, 334567890, null),array(11234567890, 2234567890, 334567890))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[11234567890,2234567890,334567890,null]
+PREHOOK: query: SELECT array_union(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT array_union(array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null,array("e","a","b","c","d")),array(array("a","b","c","d"),array("a","b","c","d"),array("a","b","c","d","e"),null))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+[["a","b","c","d"],["a","b","c","d","e"],null,["e","a","b","c","d"]]
+PREHOOK: query: create external table test_null_array (id int, value Array<String>) ROW FORMAT DELIMITED
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_null_array
+POSTHOOK: query: create external table test_null_array (id int, value Array<String>) ROW FORMAT DELIMITED
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_null_array
+PREHOOK: query: select value from test_null_array
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+POSTHOOK: query: select value from test_null_array
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+["NULL"]
+["null","null"]
+[]
+PREHOOK: query: select array_union(value,value) from test_null_array
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+POSTHOOK: query: select array_union(value,value) from test_null_array
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+["NULL"]
+["null"]
+[]
+PREHOOK: query: select value, array_union(value,value) from test_null_array
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+POSTHOOK: query: select value, array_union(value,value) from test_null_array
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_null_array
+#### A masked pattern was here ####
+["NULL"] ["NULL"]
+["null","null"] ["null"]
+[] []