You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/02/05 08:17:41 UTC
[incubator-hivemall] branch master updated: [HIVEMALL-236]
to_json/from_json cause KryoException/NullPointerException with ArrayList
due to Kryo bug
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new 5ff827d [HIVEMALL-236] to_json/from_json cause KryoException/NullPointerException with ArrayList due to Kryo bug
5ff827d is described below
commit 5ff827da3a57731b5ebc0a7a0763f025603fc2ab
Author: Makoto Yui <my...@apache.org>
AuthorDate: Tue Feb 5 17:17:37 2019 +0900
[HIVEMALL-236] to_json/from_json cause KryoException/NullPointerException with ArrayList due to Kryo bug
## What changes were proposed in this pull request?
Avoid NPE in Kryo serialization of List object created by `Arrays.asList`.
## What type of PR is it?
Bug Fix
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-236
## How was this patch tested?
unit tests
## Checklist
(Please remove this section if not needed; check `x` for YES, blank for NO)
- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [ ] Did you run system tests on Hive (or Spark)?
Author: Makoto Yui <my...@apache.org>
Closes #182 from myui/json_fix.
---
core/src/main/java/hivemall/tools/json/FromJsonUDF.java | 6 ++++--
core/src/main/java/hivemall/tools/json/ToJsonUDF.java | 7 ++++---
core/src/main/java/hivemall/utils/lang/ArrayUtils.java | 13 +++++++++++++
core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java | 9 +++++++++
core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java | 11 +++++++++++
5 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/core/src/main/java/hivemall/tools/json/FromJsonUDF.java b/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
index 97409dd..2a17f0c 100644
--- a/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
+++ b/core/src/main/java/hivemall/tools/json/FromJsonUDF.java
@@ -20,6 +20,7 @@ package hivemall.tools.json;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.hadoop.JsonSerdeUtils;
+import hivemall.utils.lang.ArrayUtils;
import hivemall.utils.lang.ExceptionUtils;
import hivemall.utils.lang.StringUtils;
@@ -109,9 +110,10 @@ public final class FromJsonUDF extends GenericUDF {
final ObjectInspector argOI2 = argOIs[2];
if (HiveUtils.isConstString(argOI2)) {
String names = HiveUtils.getConstString(argOI2);
- this.columnNames = Arrays.asList(names.split(","));
+ this.columnNames = ArrayUtils.asKryoSerializableList(names.split(","));
} else if (HiveUtils.isConstStringListOI(argOI2)) {
- this.columnNames = Arrays.asList(HiveUtils.getConstStringArray(argOI2));
+ this.columnNames =
+ ArrayUtils.asKryoSerializableList(HiveUtils.getConstStringArray(argOI2));
} else {
throw new UDFArgumentException("Expected `const array<string>` or `const string`"
+ " but got an unexpected OI type for the third argument: " + argOI2);
diff --git a/core/src/main/java/hivemall/tools/json/ToJsonUDF.java b/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
index 95d1af9..c37abe6 100644
--- a/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
+++ b/core/src/main/java/hivemall/tools/json/ToJsonUDF.java
@@ -20,10 +20,10 @@ package hivemall.tools.json;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.hadoop.JsonSerdeUtils;
+import hivemall.utils.lang.ArrayUtils;
import hivemall.utils.lang.ExceptionUtils;
import hivemall.utils.lang.StringUtils;
-import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
@@ -133,9 +133,10 @@ public final class ToJsonUDF extends GenericUDF {
final ObjectInspector argOI1 = argOIs[1];
if (HiveUtils.isConstString(argOI1)) {
String names = HiveUtils.getConstString(argOI1);
- this.columnNames = Arrays.asList(names.split(","));
+ this.columnNames = ArrayUtils.asKryoSerializableList(names.split(","));
} else if (HiveUtils.isConstStringListOI(argOI1)) {
- this.columnNames = Arrays.asList(HiveUtils.getConstStringArray(argOI1));
+ this.columnNames =
+ ArrayUtils.asKryoSerializableList(HiveUtils.getConstStringArray(argOI1));
} else {
throw new UDFArgumentException("Expected `const array<string>` or `const string`"
+ " but got an unexpected OI type for the third argument: " + argOI1);
diff --git a/core/src/main/java/hivemall/utils/lang/ArrayUtils.java b/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
index e06400a..1bce603 100644
--- a/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
+++ b/core/src/main/java/hivemall/utils/lang/ArrayUtils.java
@@ -21,6 +21,7 @@ package hivemall.utils.lang;
import hivemall.math.random.PRNG;
import java.lang.reflect.Array;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@@ -750,4 +751,16 @@ public final class ArrayUtils {
return ret;
}
+ /**
+ * Workaround for org.apache.hive.com.esotericsoftware.kryo.KryoException
+ */
+ @Nonnull
+ public static List<String> asKryoSerializableList(@Nonnull final String[] array) {
+ final List<String> list = new ArrayList<>(array.length);
+ for (String e : array) {
+ list.add(e);
+ }
+ return list;
+ }
+
}
diff --git a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
index 738a939..46be75d 100644
--- a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
@@ -91,4 +91,13 @@ public class FromJsonUDFTest {
new Object[] {"[0.1,1.1,2.2]"});
}
+ @Test
+ public void testSerializationThreeArgs() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(FromJsonUDF.class,
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ HiveUtils.getConstStringObjectInspector("struct<name:string,age:int>"),
+ HiveUtils.getConstStringObjectInspector("person")},
+ new Object[] {"{ \"person\" : { \"name\" : \"makoto\" , \"age\" : 37 } }"});
+ }
+
}
diff --git a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
index f7f698c..c2209e5 100644
--- a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
@@ -19,6 +19,7 @@
package hivemall.tools.json;
import hivemall.TestUtils;
+import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
@@ -62,4 +63,14 @@ public class ToJsonUDFTest {
new Object[] {Arrays.asList(0.1d, 1.1d, 2.1d)});
}
+ @Test
+ public void testSerializationTwoArgs() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(ToJsonUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ HiveUtils.getConstStringObjectInspector("person")},
+ new Object[] {Arrays.asList(0.1d, 1.1d, 2.1d)});
+ }
+
}