You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2021/12/01 16:26:09 UTC
[hive] branch master updated: HIVE-25738: NullIf doesn't support complex types (#2816) (Zoltan Haindrich reviewed by Zhihua Deng and Stamatis Zampetakis)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1cafaef HIVE-25738: NullIf doesn't support complex types (#2816) (Zoltan Haindrich reviewed by Zhihua Deng and Stamatis Zampetakis)
1cafaef is described below
commit 1cafaef0456fd2714c20f57b7c81a4968f425d44
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Wed Dec 1 17:25:57 2021 +0100
HIVE-25738: NullIf doesn't support complex types (#2816) (Zoltan Haindrich reviewed by Zhihua Deng and Stamatis Zampetakis)
---
.../hive/ql/udf/generic/GenericUDFNullif.java | 24 ++++++----
.../hive/ql/udf/generic/TestGenericUDFNullif.java | 47 +++++++++++++++++++
ql/src/test/queries/clientnegative/nullif_union.q | 3 ++
ql/src/test/queries/clientpositive/udf_nullif.q | 9 ++++
.../test/results/clientnegative/nullif_union.q.out | 1 +
.../results/clientpositive/llap/udf_nullif.q.out | 54 ++++++++++++++++++++++
6 files changed, 130 insertions(+), 8 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java
index a47882e..b99efa1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNullif.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
@@ -49,6 +50,17 @@ public class GenericUDFNullif extends GenericUDF {
returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
returnOIResolver.update(arguments[0]);
+ switch (arguments[0].getCategory()) {
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case PRIMITIVE:
+ break;
+ case UNION:
+ default:
+ throw new UDFArgumentTypeException(0, "Unsupported Argument type category: " + arguments[0].getCategory());
+ }
+
boolean isPrimitive = (arguments[0] instanceof PrimitiveObjectInspector);
if (isPrimitive)
{
@@ -86,17 +98,13 @@ public class GenericUDFNullif extends GenericUDF {
public Object evaluate(DeferredObject[] arguments) throws HiveException {
Object arg0 = arguments[0].get();
Object arg1 = arguments[1].get();
- Object value0 = null;
- if (arg0 != null) {
- value0 = returnOIResolver.convertIfNecessary(arg0, argumentOIs[0], false);
- }
+ Object value0 = returnOIResolver.convertIfNecessary(arg0, argumentOIs[0], false);
if (arg0 == null || arg1 == null) {
return value0;
}
- PrimitiveObjectInspector compareOI = (PrimitiveObjectInspector) returnOIResolver.get();
- if (PrimitiveObjectInspectorUtils.comparePrimitiveObjects(
- value0, compareOI,
- returnOIResolver.convertIfNecessary(arg1, argumentOIs[1], false), compareOI)) {
+ Object value1 = returnOIResolver.convertIfNecessary(arg1, argumentOIs[1], false);
+ ObjectInspector compareOI = returnOIResolver.get();
+ if (ObjectInspectorUtils.compare(value0, compareOI, value1, compareOI) == 0) {
return null;
}
return value0;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java
index 281b0d5..2ff4408 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFNullif.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import static java.util.Arrays.asList;
+
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
@@ -27,7 +29,9 @@ import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.lazy.LazyInteger;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.DoubleWritable;
@@ -149,4 +153,47 @@ public class TestGenericUDFNullif {
Assert.assertEquals(TypeInfoFactory.intTypeInfo, oi.getTypeInfo());
Assert.assertEquals(null, udf.evaluate(args));
}
+
+ @Test
+ public void testArrayNull() throws Exception {
+ GenericUDFNullif udf = new GenericUDFNullif();
+
+ ObjectInspector[] inputOIs = {
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector),
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector) };
+
+ Object i1 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb"));
+ Object i2 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb"));
+
+ DeferredObject[] args = { new DeferredJavaObject(i1), new DeferredJavaObject(i2) };
+
+ StandardListObjectInspector oi = (StandardListObjectInspector) udf.initialize(inputOIs);
+ Assert.assertEquals("array<string>", oi.getTypeName());
+ Assert.assertEquals(null, udf.evaluate(args));
+
+ }
+
+ @Test
+ public void testArray1() throws Exception {
+ GenericUDFNullif udf = new GenericUDFNullif();
+
+ ObjectInspector[] inputOIs = {
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector),
+ ObjectInspectorFactory
+ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector) };
+
+ Object i1 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("bb"));
+ Object i2 = asList(new Text("aa"), new Text("dd"), new Text("cc"), new Text("xx"));
+
+ DeferredObject[] args = { new DeferredJavaObject(i1), new DeferredJavaObject(i2) };
+
+ StandardListObjectInspector oi = (StandardListObjectInspector) udf.initialize(inputOIs);
+ Assert.assertEquals("array<string>", oi.getTypeName());
+ Assert.assertEquals(i1, udf.evaluate(args));
+
+ }
+
}
diff --git a/ql/src/test/queries/clientnegative/nullif_union.q b/ql/src/test/queries/clientnegative/nullif_union.q
new file mode 100644
index 0000000..ca59fcb
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/nullif_union.q
@@ -0,0 +1,3 @@
+-- union is not supported
+SELECT NULLIF(create_union(0,1,2),create_union(0,1,3))
+
diff --git a/ql/src/test/queries/clientpositive/udf_nullif.q b/ql/src/test/queries/clientpositive/udf_nullif.q
index 00dfa23..3479426 100644
--- a/ql/src/test/queries/clientpositive/udf_nullif.q
+++ b/ql/src/test/queries/clientpositive/udf_nullif.q
@@ -27,3 +27,12 @@ select nullif(a,b),
nullif(b,c),
nullif(c,d),
nullif(d,a) from t0;
+
+SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null);
+SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null);
+
+SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null);
+SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null);
+
+SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null);
+SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null);
diff --git a/ql/src/test/results/clientnegative/nullif_union.q.out b/ql/src/test/results/clientnegative/nullif_union.q.out
new file mode 100644
index 0000000..8d7bc60
--- /dev/null
+++ b/ql/src/test/results/clientnegative/nullif_union.q.out
@@ -0,0 +1 @@
+FAILED: SemanticException [Error 10016]: Line 2:14 Argument type mismatch '2': Unsupported Argument type category: UNION
diff --git a/ql/src/test/results/clientpositive/llap/udf_nullif.q.out b/ql/src/test/results/clientpositive/llap/udf_nullif.q.out
index 492b412..916007a 100644
--- a/ql/src/test/results/clientpositive/llap/udf_nullif.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_nullif.q.out
@@ -213,3 +213,57 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@t0
#### A masked pattern was here ####
1 2 3.1 4.1
+PREHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(1,2,3)) is null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(array(1,2,3),array(3,2,1)) is not null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 1)) is null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(named_struct("c", 1),named_struct("c", 2)) is not null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',2)) is null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL
+PREHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT assert_true(NULLIF(map('a',1,'b',2),map('a',1,'b',3)) is not null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL