You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/08/11 17:29:00 UTC
hive git commit: HIVE-12954: NPE with str_to_map on null strings
(Marta Kuczora, reviewed by Aihua Xu)
Repository: hive
Updated Branches:
refs/heads/master 89080f557 -> 6447f5cd5
HIVE-12954: NPE with str_to_map on null strings (Marta Kuczora, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6447f5cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6447f5cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6447f5cd
Branch: refs/heads/master
Commit: 6447f5cd57d193c6ceb6aaf141fb12f29ac53cd4
Parents: 89080f5
Author: Marta Kuczora <ku...@cloudera.com>
Authored: Thu Aug 11 12:28:03 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Thu Aug 11 12:28:50 2016 -0500
----------------------------------------------------------------------
.../ql/udf/generic/GenericUDFStringToMap.java | 12 ++
.../udf/generic/TestGenericUDFStringToMap.java | 152 +++++++++++++++++++
2 files changed, 164 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6447f5cd/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
index ed60fbf..093f2a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
@@ -83,11 +83,23 @@ public class GenericUDFStringToMap extends GenericUDF {
public Object evaluate(DeferredObject[] arguments) throws HiveException {
ret.clear();
String text = (String) soi_text.convert(arguments[0].get());
+ if (text == null) {
+ return ret;
+ }
+
String delimiter1 = (soi_de1 == null) ?
default_de1 : (String) soi_de1.convert(arguments[1].get());
String delimiter2 = (soi_de2 == null) ?
default_de2 : (String) soi_de2.convert(arguments[2].get());
+ if (delimiter1 == null) {
+ delimiter1 = default_de1;
+ }
+
+ if (delimiter2 == null) {
+ delimiter2 = default_de2;
+ }
+
String[] keyValuePairs = text.split(delimiter1);
for (String keyValuePair : keyValuePairs) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6447f5cd/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java
new file mode 100644
index 0000000..1653936
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFStringToMap.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Test;
+
+public class TestGenericUDFStringToMap {
+
+ @Test
+ public void testStringToMapWithCustomDelimiters() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDF(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("key1", "value1");
+ expResult.put("key2", "value2");
+ expResult.put("key3", "value3");
+ runAndVerify("key1=value1;key2=value2;key3=value3", ";", "=", expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapWithDefaultDelimiters() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("key1", "value1");
+ expResult.put("key2", "value2");
+ expResult.put("key3", "value3");
+ runAndVerify("key1:value1,key2:value2,key3:value3", expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapWithNullDelimiters() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDF(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("key1", "value1");
+ expResult.put("key2", "value2");
+ expResult.put("key3", "value3");
+ runAndVerify("key1:value1,key2:value2,key3:value3", null, null, expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapWithNullText() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ runAndVerify(null, expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapWithEmptyText() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("", null);
+ runAndVerify("", expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapNoKey() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("", "value");
+ runAndVerify(":value", expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapNoValue() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("key", "");
+ runAndVerify("key:", expResult, udf);
+ }
+
+ @Test
+ public void testStringToMapNotMatchingDelimiter() throws HiveException {
+ GenericUDFStringToMap udf = new GenericUDFStringToMap();
+ initGenericUDFWithNoDelimiters(udf);
+ Map<String, String> expResult = new LinkedHashMap<String, String>();
+ expResult.put("key=value", null);
+ runAndVerify("key=value", expResult, udf);
+ }
+
+ private void initGenericUDF(GenericUDFStringToMap udf)
+ throws UDFArgumentException {
+
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI0, valueOI1, valueOI2 };
+ udf.initialize(arguments);
+ }
+
+ private void initGenericUDFWithNoDelimiters(GenericUDFStringToMap udf)
+ throws UDFArgumentException {
+
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI0 };
+ udf.initialize(arguments);
+ }
+
+ private void runAndVerify(String text, String delimiter1, String delimiter2,
+ Map<String, String> expResult, GenericUDF udf) throws HiveException {
+
+ DeferredObject valueObj0 = new DeferredJavaObject(text);
+ DeferredObject valueObj1 = new DeferredJavaObject(delimiter1);
+ DeferredObject valueObj2 = new DeferredJavaObject(delimiter2);
+ DeferredObject[] args = { valueObj0, valueObj1, valueObj2 };
+
+ @SuppressWarnings("unchecked")
+ LinkedHashMap<Object, Object> output = (LinkedHashMap<Object, Object>) udf.evaluate(args);
+ assertTrue("str_to_map() test", expResult.equals(output));
+ }
+
+ private void runAndVerify(String text, Map<String, String> expResult,
+ GenericUDF udf) throws HiveException {
+
+ DeferredObject valueObj0 = new DeferredJavaObject(text);
+ DeferredObject[] args = { valueObj0 };
+ @SuppressWarnings("unchecked")
+ LinkedHashMap<Object, Object> output = (LinkedHashMap<Object, Object>) udf.evaluate(args);
+ assertTrue("str_to_map() test", expResult.equals(output));
+ }
+}
\ No newline at end of file