You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2018/04/25 08:04:09 UTC
[1/2] incubator-hivemall git commit: [HIVEMALL-191] Add Kryo
serialization test to existing workaround code
Repository: incubator-hivemall
Updated Branches:
refs/heads/master 36fb839d9 -> b64b94f92
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java b/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java
index fc5572d..685139d 100644
--- a/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/SelectKBestUDFTest.java
@@ -21,6 +21,7 @@ package hivemall.tools.array;
import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
+import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -70,17 +71,22 @@ public class SelectKBestUDFTest {
}
@Test
- public void testSerialization() throws HiveException {
- final SelectKBestUDF selectKBest = new SelectKBestUDF();
+ public void testSerialization() throws HiveException, IOException {
final int k = 2;
- selectKBest.initialize(new ObjectInspector[] {
- ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
- ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
- ObjectInspectorUtils.getConstantObjectInspector(
- PrimitiveObjectInspectorFactory.javaIntObjectInspector, k)});
+ final double[] data = new double[] {250.29999999999998, 170.90000000000003, 73.2,
+ 12.199999999999996};
+ final double[] importanceList = new double[] {292.1666753739119, 152.70000455081467,
+ 187.93333893418327, 59.93333511948589};
- byte[] serialized = TestUtils.serializeObjectByKryo(selectKBest);
- TestUtils.deserializeObjectByKryo(serialized, SelectKBestUDF.class);
+ TestUtils.testGenericUDFSerialization(
+ SelectKBestUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector, k)},
+ new Object[] {WritableUtils.toWritableList(data),
+ WritableUtils.toWritableList(importanceList), k});
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
index 6f7d4cf..a5b3550 100644
--- a/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/FromJsonUDFTest.java
@@ -18,11 +18,14 @@
*/
package hivemall.tools.json;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.HiveUtils;
+import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -79,4 +82,12 @@ public class FromJsonUDFTest {
udf.close();
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(FromJsonUDF.class,
+ new ObjectInspector[] {PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ HiveUtils.getConstStringObjectInspector("array<double>")},
+ new Object[] {"[0.1,1.1,2.2]"});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
index 4da6b9a..5223849 100644
--- a/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
+++ b/core/src/test/java/hivemall/tools/json/ToJsonUDFTest.java
@@ -18,8 +18,10 @@
*/
package hivemall.tools.json;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -29,6 +31,9 @@ import org.apache.hadoop.io.Text;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+import java.util.Arrays;
+
public class ToJsonUDFTest {
@Test
@@ -47,4 +52,12 @@ public class ToJsonUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ToJsonUDF.class,
+ new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
+ new Object[] {Arrays.asList(0.1d, 1.1d, 2.1d)});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
index 9012c63..23018f8 100644
--- a/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
+++ b/core/src/test/java/hivemall/tools/vector/VectorAddUDFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.tools.vector;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
@@ -77,4 +78,14 @@ public class VectorAddUDFTest {
udf.close();
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ VectorAddUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaFloatObjectInspector)},
+ new Object[] {Arrays.asList(1.d, 2.d, 3.d), Arrays.asList(2.f, 3.f, 4.f)});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
index 6d7c05e..f0fe9d1 100644
--- a/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
+++ b/core/src/test/java/hivemall/tools/vector/VectorDotUDFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.tools.vector;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
@@ -76,4 +77,14 @@ public class VectorDotUDFTest {
udf.close();
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ VectorDotUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaFloatObjectInspector)},
+ new Object[] {Arrays.asList(1.d, 2.d, 3.d), Arrays.asList(2.f, 3.f, 4.f)});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/topicmodel/LDAUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/topicmodel/LDAUDTFTest.java b/core/src/test/java/hivemall/topicmodel/LDAUDTFTest.java
index 4cbb668..fc725b9 100644
--- a/core/src/test/java/hivemall/topicmodel/LDAUDTFTest.java
+++ b/core/src/test/java/hivemall/topicmodel/LDAUDTFTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.Arrays;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -161,6 +162,21 @@ public class LDAUDTFTest {
udtf.getWordScore("アボカド", k2) > udtf.getWordScore("健康", k2));
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ LDAUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-topics 2 -num_docs 2 -s 1 -iter 32 -eps 1e-3")},
+ new Object[][] {
+ {Arrays.asList("fruits:1", "healthy:1", "vegetables:1")},
+ {Arrays.asList("apples:1", "avocados:1", "colds:1", "flu:1", "like:2",
+ "oranges:1")}});
+ }
+
private static void println(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/topicmodel/PLSAUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/topicmodel/PLSAUDTFTest.java b/core/src/test/java/hivemall/topicmodel/PLSAUDTFTest.java
index e5045a5..87b2f4c 100644
--- a/core/src/test/java/hivemall/topicmodel/PLSAUDTFTest.java
+++ b/core/src/test/java/hivemall/topicmodel/PLSAUDTFTest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.Arrays;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -162,6 +163,21 @@ public class PLSAUDTFTest {
udtf.getWordScore("アボカド", k2) > udtf.getWordScore("健康", k2));
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ PLSAUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-topics 2 -alpha 0.1 -delta 0.00001 -iter 10000")},
+ new Object[][] {
+ {Arrays.asList("fruits:1", "healthy:1", "vegetables:1")},
+ {Arrays.asList("apples:1", "avocados:1", "colds:1", "flu:1", "like:2",
+ "oranges:1")}});
+ }
+
private static void println(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java
----------------------------------------------------------------------
diff --git a/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java b/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java
index 96d9e4b..745ec30 100644
--- a/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java
+++ b/nlp/src/main/java/hivemall/nlp/tokenizer/KuromojiUDF.java
@@ -72,13 +72,10 @@ public final class KuromojiUDF extends GenericUDF {
private static final int READ_TIMEOUT_MS = 60000; // 60 sec
private static final long MAX_INPUT_STREAM_SIZE = 32L * 1024L * 1024L; // ~32MB
- private Mode _mode;
-
- // lazy instantiation to avoid org.apache.hive.com.esotericsoftware.kryo.KryoException: java.lang.NullPointerException
- private transient CharArraySet _stopWords;
-
- private Set<String> _stopTags;
- private UserDictionary _userDict;
+ private String _modeString;
+ private String[] _stopWordsArray;
+ private String[] _stopTagsArray;
+ private Object _userDictObj;
// workaround to avoid org.apache.hive.com.esotericsoftware.kryo.KryoException: java.util.ConcurrentModificationException
private transient JapaneseAnalyzer _analyzer;
@@ -91,12 +88,29 @@ public final class KuromojiUDF extends GenericUDF {
+ arglen);
}
- this._mode = (arglen >= 2) ? tokenizationMode(arguments[1]) : Mode.NORMAL;
- this._stopWords = (arglen >= 3) ? stopWords(arguments[2])
- : JapaneseAnalyzer.getDefaultStopSet();
- this._stopTags = (arglen >= 4) ? stopTags(arguments[3])
- : JapaneseAnalyzer.getDefaultStopTags();
- this._userDict = (arglen >= 5) ? userDictionary(arguments[4]) : null;
+ this._modeString = (arglen >= 2) ? HiveUtils.getConstString(arguments[1]) : "NORMAL";
+
+ this._stopWordsArray = null;
+ if (arglen >= 3 && !HiveUtils.isVoidOI(arguments[2])) {
+ this._stopWordsArray = HiveUtils.getConstStringArray(arguments[2]);
+ }
+
+ this._stopTagsArray = null;
+ if (arglen >= 4 && !HiveUtils.isVoidOI(arguments[3])) {
+ this._stopTagsArray = HiveUtils.getConstStringArray(arguments[3]);
+ }
+
+ this._userDictObj = null;
+ if (arglen >= 5) {
+ if (HiveUtils.isConstListOI(arguments[4])) {
+ this._userDictObj = HiveUtils.getConstStringArray(arguments[4]);
+ } else if (HiveUtils.isConstString(arguments[4])) {
+ this._userDictObj = HiveUtils.getConstString(arguments[4]);
+ } else {
+ throw new UDFArgumentException(
+ "User dictionary MUST be given as an array of constant string or constant string (URL)");
+ }
+ }
this._analyzer = null;
@@ -106,7 +120,18 @@ public final class KuromojiUDF extends GenericUDF {
@Override
public List<Text> evaluate(DeferredObject[] arguments) throws HiveException {
if (_analyzer == null) {
- this._analyzer = new JapaneseAnalyzer(_userDict, _mode, _stopWords, _stopTags);
+ Mode mode = tokenizationMode(_modeString);
+ CharArraySet stopWords = stopWords(_stopWordsArray);
+ Set<String> stopTags = stopTags(_stopTagsArray);
+
+ UserDictionary userDict = null;
+ if (_userDictObj instanceof String[]) {
+ userDict = userDictionary((String[]) _userDictObj);
+ } else if (_userDictObj instanceof String) {
+ userDict = userDictionary((String) _userDictObj);
+ }
+
+ this._analyzer = new JapaneseAnalyzer(userDict, mode, stopWords, stopTags);
}
Object arg0 = arguments[0].get();
@@ -137,9 +162,7 @@ public final class KuromojiUDF extends GenericUDF {
}
@Nonnull
- private static Mode tokenizationMode(@Nonnull final ObjectInspector oi)
- throws UDFArgumentException {
- final String arg = HiveUtils.getConstString(oi);
+ private static Mode tokenizationMode(@Nullable final String arg) throws UDFArgumentException {
if (arg == null) {
return Mode.NORMAL;
}
@@ -160,12 +183,8 @@ public final class KuromojiUDF extends GenericUDF {
}
@Nonnull
- private static CharArraySet stopWords(@Nonnull final ObjectInspector oi)
+ private static CharArraySet stopWords(@Nullable final String[] array)
throws UDFArgumentException {
- if (HiveUtils.isVoidOI(oi)) {
- return JapaneseAnalyzer.getDefaultStopSet();
- }
- final String[] array = HiveUtils.getConstStringArray(oi);
if (array == null) {
return JapaneseAnalyzer.getDefaultStopSet();
}
@@ -177,12 +196,7 @@ public final class KuromojiUDF extends GenericUDF {
}
@Nonnull
- private static Set<String> stopTags(@Nonnull final ObjectInspector oi)
- throws UDFArgumentException {
- if (HiveUtils.isVoidOI(oi)) {
- return JapaneseAnalyzer.getDefaultStopTags();
- }
- final String[] array = HiveUtils.getConstStringArray(oi);
+ private static Set<String> stopTags(@Nullable final String[] array) throws UDFArgumentException {
if (array == null) {
return JapaneseAnalyzer.getDefaultStopTags();
}
@@ -201,19 +215,6 @@ public final class KuromojiUDF extends GenericUDF {
}
@Nullable
- private static UserDictionary userDictionary(@Nonnull final ObjectInspector oi)
- throws UDFArgumentException {
- if (HiveUtils.isConstListOI(oi)) {
- return userDictionary(HiveUtils.getConstStringArray(oi));
- } else if (HiveUtils.isConstString(oi)) {
- return userDictionary(HiveUtils.getConstString(oi));
- } else {
- throw new UDFArgumentException(
- "User dictionary MUST be given as an array of constant string or constant string (URL)");
- }
- }
-
- @Nullable
private static UserDictionary userDictionary(@Nullable final String[] userDictArray)
throws UDFArgumentException {
if (userDictArray == null) {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/nlp/src/main/java/hivemall/nlp/tokenizer/SmartcnUDF.java
----------------------------------------------------------------------
diff --git a/nlp/src/main/java/hivemall/nlp/tokenizer/SmartcnUDF.java b/nlp/src/main/java/hivemall/nlp/tokenizer/SmartcnUDF.java
index afaa485..d185b0d 100644
--- a/nlp/src/main/java/hivemall/nlp/tokenizer/SmartcnUDF.java
+++ b/nlp/src/main/java/hivemall/nlp/tokenizer/SmartcnUDF.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -102,7 +103,7 @@ public final class SmartcnUDF extends GenericUDF {
}
@Nonnull
- private static CharArraySet stopWords(@Nonnull final String[] array)
+ private static CharArraySet stopWords(@Nullable final String[] array)
throws UDFArgumentException {
if (array == null) {
return SmartChineseAnalyzer.getDefaultStopSet();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/nlp/src/test/java/hivemall/TestUtils.java
----------------------------------------------------------------------
diff --git a/nlp/src/test/java/hivemall/TestUtils.java b/nlp/src/test/java/hivemall/TestUtils.java
new file mode 100644
index 0000000..c886c2d
--- /dev/null
+++ b/nlp/src/test/java/hivemall/TestUtils.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall;
+
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hive.com.esotericsoftware.kryo.Kryo;
+import org.apache.hive.com.esotericsoftware.kryo.io.Input;
+import org.apache.hive.com.esotericsoftware.kryo.io.Output;
+
+import javax.annotation.Nonnull;
+import java.io.ByteArrayOutputStream;
+
+public final class TestUtils {
+
+ @Nonnull
+ public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
+ Kryo kryo = getKryo();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Output output = new Output(bos);
+ kryo.writeObject(output, obj);
+ output.close();
+ return bos.toByteArray();
+ }
+
+ @Nonnull
+ public static <T> T deserializeObjectByKryo(@Nonnull byte[] in, @Nonnull Class<T> clazz) {
+ Kryo kryo = getKryo();
+ Input inp = new Input(in);
+ T t = kryo.readObject(inp, clazz);
+ inp.close();
+ return t;
+ }
+
+ @Nonnull
+ private static Kryo getKryo() {
+ return Utilities.runtimeSerializationKryo.get();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/nlp/src/test/java/hivemall/nlp/tokenizer/KuromojiUDFTest.java
----------------------------------------------------------------------
diff --git a/nlp/src/test/java/hivemall/nlp/tokenizer/KuromojiUDFTest.java b/nlp/src/test/java/hivemall/nlp/tokenizer/KuromojiUDFTest.java
index f9acc82..1c3db9f 100644
--- a/nlp/src/test/java/hivemall/nlp/tokenizer/KuromojiUDFTest.java
+++ b/nlp/src/test/java/hivemall/nlp/tokenizer/KuromojiUDFTest.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import hivemall.TestUtils;
+
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -33,10 +35,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.io.Text;
import org.junit.Assert;
import org.junit.Test;
-import org.objenesis.strategy.StdInstantiatorStrategy;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.io.Output;
public class KuromojiUDFTest {
@@ -80,7 +78,7 @@ public class KuromojiUDFTest {
}
@Test(expected = UDFArgumentException.class)
- public void testInvalidMode() throws UDFArgumentException, IOException {
+ public void testInvalidMode() throws IOException, HiveException {
GenericUDF udf = new KuromojiUDF();
ObjectInspector[] argOIs = new ObjectInspector[2];
// line
@@ -91,6 +89,18 @@ public class KuromojiUDFTest {
argOIs[1] = PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
stringType, new Text("unsupported mode"));
udf.initialize(argOIs);
+
+ DeferredObject[] args = new DeferredObject[1];
+ args[0] = new DeferredObject() {
+ public Text get() throws HiveException {
+ return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
+ }
+
+ @Override
+ public void prepare(int arg) throws HiveException {}
+ };
+ udf.evaluate(args);
+
udf.close();
}
@@ -365,16 +375,31 @@ public class KuromojiUDFTest {
}
@Test
- public void testSerializeByKryo() throws UDFArgumentException {
+ public void testSerialization() throws IOException, HiveException {
final KuromojiUDF udf = new KuromojiUDF();
ObjectInspector[] argOIs = new ObjectInspector[1];
argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
udf.initialize(argOIs);
- Kryo kryo = new Kryo();
- kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());
- Output output = new Output(1024 * 16);
- kryo.writeObject(output, udf);
- output.close();
+ // serialization after initialization
+ byte[] serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, KuromojiUDF.class);
+
+ DeferredObject[] args = new DeferredObject[1];
+ args[0] = new DeferredObject() {
+ public Text get() throws HiveException {
+ return new Text("クロモジのJapaneseAnalyzerを使ってみる。テスト。");
+ }
+
+ @Override
+ public void prepare(int arg) throws HiveException {}
+ };
+ List<Text> tokens = udf.evaluate(args);
+
+ // serialization after evaluation
+ serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, KuromojiUDF.class);
+
+ udf.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/nlp/src/test/java/hivemall/nlp/tokenizer/SmartcnUDFTest.java
----------------------------------------------------------------------
diff --git a/nlp/src/test/java/hivemall/nlp/tokenizer/SmartcnUDFTest.java b/nlp/src/test/java/hivemall/nlp/tokenizer/SmartcnUDFTest.java
index 83ccb0c..342e48a 100644
--- a/nlp/src/test/java/hivemall/nlp/tokenizer/SmartcnUDFTest.java
+++ b/nlp/src/test/java/hivemall/nlp/tokenizer/SmartcnUDFTest.java
@@ -21,6 +21,8 @@ package hivemall.nlp.tokenizer;
import java.io.IOException;
import java.util.List;
+import hivemall.TestUtils;
+
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -78,4 +80,33 @@ public class SmartcnUDFTest {
Assert.assertNotNull(tokens);
udf.close();
}
+
+ @Test
+ public void testSerialization() throws IOException, HiveException {
+ final SmartcnUDF udf = new SmartcnUDF();
+ ObjectInspector[] argOIs = new ObjectInspector[1];
+ argOIs[0] = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ udf.initialize(argOIs);
+
+ // serialization after initialization
+ byte[] serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, SmartcnUDF.class);
+
+ DeferredObject[] args = new DeferredObject[1];
+ args[0] = new DeferredObject() {
+ public Text get() throws HiveException {
+ return new Text("Smartcn为Apache2.0协议的开源中文分词系统,Java语言编写,修改的中科院计算所ICTCLAS分词系统。");
+ }
+
+ @Override
+ public void prepare(int arg) throws HiveException {}
+ };
+ List<Text> tokens = udf.evaluate(args);
+
+ // serialization after evaluation
+ serialized = TestUtils.serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, SmartcnUDF.class);
+
+ udf.close();
+ }
}
[2/2] incubator-hivemall git commit: [HIVEMALL-191] Add Kryo
serialization test to existing workaround code
Posted by my...@apache.org.
[HIVEMALL-191] Add Kryo serialization test to existing workaround code
## What changes were proposed in this pull request?
Add Kryo serialization test to existing workaround code as: https://github.com/apache/incubator-hivemall/commit/f6765dff7be67e1a3327709bbb9bfdc6eba7b97f
To be more precise, currently two UDFs `quantified_features` and `tokenize_ja` explicitly have the workaround lazy instantiation code. So, this PR makes their `transient` keyword unnecessary.
## What type of PR is it?
Improvement
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-191
## How was this patch tested?
Added some unit tests, and manually tested as well
## Checklist
- [x] Did you apply source code formatter, i.e., `mvn formatter:format`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?
Author: Takuya Kitazawa <k....@gmail.com>
Closes #145 from takuti/HIVEMALL-191.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/b64b94f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/b64b94f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/b64b94f9
Branch: refs/heads/master
Commit: b64b94f92b27886bb8e36b27e5463be5fda1e52f
Parents: 36fb839
Author: Takuya Kitazawa <k....@gmail.com>
Authored: Wed Apr 25 17:04:02 2018 +0900
Committer: Makoto Yui <my...@apache.org>
Committed: Wed Apr 25 17:04:02 2018 +0900
----------------------------------------------------------------------
.../ftvec/trans/QuantifiedFeaturesUDTF.java | 33 ++++----
core/src/test/java/hivemall/TestUtils.java | 70 ++++++++++++++++
.../classifier/GeneralClassifierUDTFTest.java | 12 +++
...ernelExpansionPassiveAggressiveUDTFTest.java | 12 +++
.../classifier/PassiveAggressiveUDTFTest.java | 23 +++++
.../hivemall/classifier/PerceptronUDTFTest.java | 15 ++++
.../fm/FactorizationMachineUDTFTest.java | 15 ++++
.../FieldAwareFactorizationMachineUDTFTest.java | 15 ++++
.../java/hivemall/ftvec/FeatureUDFTest.java | 13 +++
.../ftvec/hashing/FeatureHashingUDFTest.java | 20 +++++
.../ftvec/trans/QuantifiedFeaturesUDTFTest.java | 88 ++++++++++++++++++++
.../ftvec/trans/TestBinarizeLabelUDTF.java | 15 ++++
.../ftvec/trans/VectorizeFeaturesUDFTest.java | 27 ++++--
.../geospatial/HaversineDistanceUDFTest.java | 20 +++++
.../hivemall/geospatial/Lat2TileYUDFTest.java | 9 ++
.../hivemall/geospatial/Lon2TileXUDFTest.java | 9 ++
.../hivemall/geospatial/TileX2LonUDFTest.java | 8 ++
.../hivemall/geospatial/TileY2LatUDFTest.java | 8 ++
.../knn/distance/EuclidDistanceUDFTest.java | 16 ++++
.../knn/similarity/CosineSimilarityUDFTest.java | 15 ++++
.../knn/similarity/DIMSUMMapperUDTFTest.java | 24 ++++++
.../mf/BPRMatrixFactorizationUDTFTest.java | 14 ++++
.../mf/MatrixFactorizationAdaGradUDTFTest.java | 14 ++++
.../mf/MatrixFactorizationSGDUDTFTest.java | 14 ++++
.../java/hivemall/recommend/SlimUDTFTest.java | 72 ++++++++++++++++
.../hivemall/regression/AdaGradUDTFTest.java | 14 ++++
.../regression/GeneralRegressorUDTFTest.java | 16 +++-
.../RandomForestClassifierUDTFTest.java | 35 ++++++++
.../smile/tools/TreePredictUDFTest.java | 50 +++++++++++
.../smile/tools/TreePredictUDFv1Test.java | 49 +++++++++++
.../statistics/MovingAverageUDTFTest.java | 12 +++
.../java/hivemall/tools/TryCastUDFTest.java | 14 ++++
.../tools/array/ArrayAppendUDFTest.java | 14 +++-
.../tools/array/ArrayElementAtUDFTest.java | 12 +++
.../tools/array/ArrayFlattenUDFTest.java | 10 +++
.../hivemall/tools/array/ArraySliceUDFTest.java | 14 ++++
.../hivemall/tools/array/ArrayUnionUDFTest.java | 11 +++
.../tools/array/ConditionalEmitUDTFTest.java | 12 +++
.../tools/array/FirstElementUDFTest.java | 10 +++
.../tools/array/LastElementUDFTest.java | 10 +++
.../tools/array/SelectKBestUDFTest.java | 24 ++++--
.../hivemall/tools/json/FromJsonUDFTest.java | 11 +++
.../java/hivemall/tools/json/ToJsonUDFTest.java | 13 +++
.../hivemall/tools/vector/VectorAddUDFTest.java | 11 +++
.../hivemall/tools/vector/VectorDotUDFTest.java | 11 +++
.../java/hivemall/topicmodel/LDAUDTFTest.java | 16 ++++
.../java/hivemall/topicmodel/PLSAUDTFTest.java | 16 ++++
.../hivemall/nlp/tokenizer/KuromojiUDF.java | 83 +++++++++---------
.../java/hivemall/nlp/tokenizer/SmartcnUDF.java | 3 +-
nlp/src/test/java/hivemall/TestUtils.java | 55 ++++++++++++
.../hivemall/nlp/tokenizer/KuromojiUDFTest.java | 47 ++++++++---
.../hivemall/nlp/tokenizer/SmartcnUDFTest.java | 31 +++++++
52 files changed, 1101 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java b/core/src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java
index 9299976..5b2eefe 100644
--- a/core/src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java
+++ b/core/src/main/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTF.java
@@ -48,8 +48,7 @@ public final class QuantifiedFeaturesUDTF extends GenericUDTF {
private Identifier<String>[] identifiers;
private DoubleWritable[] columnValues;
- // lazy instantiation to avoid org.apache.hive.com.esotericsoftware.kryo.KryoException: java.lang.NullPointerException
- private transient Object[] forwardObjs;
+ private Object[] forwardObjs;
@SuppressWarnings("unchecked")
@Override
@@ -63,20 +62,21 @@ public final class QuantifiedFeaturesUDTF extends GenericUDTF {
int outputSize = size - 1;
this.doubleOIs = new PrimitiveObjectInspector[outputSize];
- this.columnValues = new DoubleWritable[outputSize];
this.identifiers = new Identifier[outputSize];
- this.forwardObjs = null;
+ this.columnValues = new DoubleWritable[outputSize];
for (int i = 0; i < outputSize; i++) {
- columnValues[i] = new DoubleWritable(Double.NaN);
ObjectInspector argOI = argOIs[i + 1];
if (HiveUtils.isNumberOI(argOI)) {
doubleOIs[i] = HiveUtils.asDoubleCompatibleOI(argOI);
} else {
identifiers[i] = new Identifier<String>();
}
+ columnValues[i] = new DoubleWritable(Double.NaN);
}
+ this.forwardObjs = null;
+
List<String> fieldNames = new ArrayList<String>(outputSize);
List<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(outputSize);
fieldNames.add("features");
@@ -87,32 +87,37 @@ public final class QuantifiedFeaturesUDTF extends GenericUDTF {
@Override
public void process(Object[] args) throws HiveException {
- if (forwardObjs == null) {
- this.forwardObjs = new Object[] {Arrays.asList(columnValues)};
- }
-
+ int outputSize = args.length - 1;
boolean outputRow = boolOI.get(args[0]);
if (outputRow) {
- final DoubleWritable[] values = this.columnValues;
- for (int i = 0, outputSize = args.length - 1; i < outputSize; i++) {
+ if (forwardObjs == null) {
+ // forwardObjs internally references columnValues
+ List<DoubleWritable> column = new ArrayList<>(outputSize);
+ this.forwardObjs = new Object[] {column};
+ for (int i = 0; i < outputSize; i++) {
+ column.add(columnValues[i]);
+ }
+ }
+ // updating columnValues simultaneously changes forwardObjs
+ for (int i = 0; i < outputSize; i++) {
Object arg = args[i + 1];
Identifier<String> identifier = identifiers[i];
if (identifier == null) {
double v = PrimitiveObjectInspectorUtils.getDouble(arg, doubleOIs[i]);
- values[i].set(v);
+ columnValues[i].set(v);
} else {
if (arg == null) {
throw new HiveException("Found Null in the input: " + Arrays.toString(args));
} else {
String k = arg.toString();
int id = identifier.valueOf(k);
- values[i].set(id);
+ columnValues[i].set(id);
}
}
}
forward(forwardObjs);
} else {// load only
- for (int i = 0, outputSize = args.length - 1; i < outputSize; i++) {
+ for (int i = 0; i < outputSize; i++) {
Identifier<String> identifier = identifiers[i];
if (identifier != null) {
Object arg = args[i + 1];
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/TestUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/TestUtils.java b/core/src/test/java/hivemall/TestUtils.java
index 12d921e..7bd1ac3 100644
--- a/core/src/test/java/hivemall/TestUtils.java
+++ b/core/src/test/java/hivemall/TestUtils.java
@@ -19,16 +19,86 @@
package hivemall;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import javax.annotation.Nonnull;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hive.com.esotericsoftware.kryo.Kryo;
import org.apache.hive.com.esotericsoftware.kryo.io.Input;
import org.apache.hive.com.esotericsoftware.kryo.io.Output;
public final class TestUtils {
+ public static <T extends GenericUDF> void testGenericUDFSerialization(@Nonnull Class<T> clazz,
+ @Nonnull ObjectInspector[] ois, @Nonnull Object[] row) throws HiveException,
+ IOException {
+ final T udf;
+ try {
+ udf = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new HiveException(e);
+ }
+
+ udf.initialize(ois);
+
+ // serialization after initialization
+ byte[] serialized = serializeObjectByKryo(udf);
+ deserializeObjectByKryo(serialized, clazz);
+
+ int size = row.length;
+ GenericUDF.DeferredObject[] rowDeferred = new GenericUDF.DeferredObject[size];
+ for (int i = 0; i < size; i++) {
+ rowDeferred[i] = new GenericUDF.DeferredJavaObject(row[i]);
+ }
+
+ udf.evaluate(rowDeferred);
+
+ // serialization after evaluating row
+ serialized = serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized, clazz);
+
+ udf.close();
+ }
+
+ public static <T extends GenericUDTF> void testGenericUDTFSerialization(
+ @Nonnull Class<T> clazz, @Nonnull ObjectInspector[] ois, @Nonnull Object[][] rows)
+ throws HiveException {
+ final T udtf;
+ try {
+ udtf = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new HiveException(e);
+ }
+
+ udtf.initialize(ois);
+
+ // serialization after initialization
+ byte[] serialized = serializeObjectByKryo(udtf);
+ deserializeObjectByKryo(serialized, clazz);
+
+ udtf.setCollector(new Collector() {
+ public void collect(Object input) throws HiveException {
+ // noop
+ }
+ });
+
+ for (Object[] row : rows) {
+ udtf.process(row);
+ }
+
+ // serialization after processing row
+ serialized = serializeObjectByKryo(udtf);
+ TestUtils.deserializeObjectByKryo(serialized, clazz);
+
+ udtf.close();
+ }
+
@Nonnull
public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
Kryo kryo = getKryo();
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java b/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
index dba4a00..9b0885d 100644
--- a/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
+++ b/core/src/test/java/hivemall/classifier/GeneralClassifierUDTFTest.java
@@ -21,6 +21,8 @@ package hivemall.classifier;
import static hivemall.utils.hadoop.HiveUtils.lazyInteger;
import static hivemall.utils.hadoop.HiveUtils.lazyLong;
import static hivemall.utils.hadoop.HiveUtils.lazyString;
+
+import hivemall.TestUtils;
import hivemall.utils.math.MathUtils;
import java.io.BufferedReader;
@@ -368,6 +370,16 @@ public class GeneralClassifierUDTFTest {
Assert.assertTrue(accuracy > 0.8f);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ GeneralClassifierUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 0}});
+ }
+
private static void println(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/classifier/KernelExpansionPassiveAggressiveUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/classifier/KernelExpansionPassiveAggressiveUDTFTest.java b/core/src/test/java/hivemall/classifier/KernelExpansionPassiveAggressiveUDTFTest.java
index 1af0ebc..ec5a382 100644
--- a/core/src/test/java/hivemall/classifier/KernelExpansionPassiveAggressiveUDTFTest.java
+++ b/core/src/test/java/hivemall/classifier/KernelExpansionPassiveAggressiveUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.classifier;
+import hivemall.TestUtils;
import hivemall.model.FeatureValue;
import hivemall.utils.math.MathUtils;
@@ -27,6 +28,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;
@@ -146,6 +148,16 @@ public class KernelExpansionPassiveAggressiveUDTFTest {
Assert.assertTrue(accuracy > 0.82f);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ KernelExpansionPassiveAggressiveUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 0}});
+ }
+
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
InputStream is = KernelExpansionPassiveAggressiveUDTFTest.class.getResourceAsStream(fileName);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/classifier/PassiveAggressiveUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/classifier/PassiveAggressiveUDTFTest.java b/core/src/test/java/hivemall/classifier/PassiveAggressiveUDTFTest.java
index 2322c07..721b777 100644
--- a/core/src/test/java/hivemall/classifier/PassiveAggressiveUDTFTest.java
+++ b/core/src/test/java/hivemall/classifier/PassiveAggressiveUDTFTest.java
@@ -19,9 +19,12 @@
package hivemall.classifier;
import static org.junit.Assert.assertEquals;
+
+import hivemall.TestUtils;
import hivemall.model.PredictionResult;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -221,4 +224,24 @@ public class PassiveAggressiveUDTFTest {
assertEquals(expectedLearningRate2, udtf.eta(loss, margin2), 1e-5f);
}
+ @Test
+ public void testPA1Serialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ PassiveAggressiveUDTF.PA1.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 0}});
+ }
+
+ @Test
+ public void testPA2Serialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ PassiveAggressiveUDTF.PA2.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 0}});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/classifier/PerceptronUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/classifier/PerceptronUDTFTest.java b/core/src/test/java/hivemall/classifier/PerceptronUDTFTest.java
index e04adb7..6d3dd12 100644
--- a/core/src/test/java/hivemall/classifier/PerceptronUDTFTest.java
+++ b/core/src/test/java/hivemall/classifier/PerceptronUDTFTest.java
@@ -19,9 +19,12 @@
package hivemall.classifier;
import static org.junit.Assert.assertEquals;
+
+import hivemall.TestUtils;
import hivemall.model.FeatureValue;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -30,6 +33,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.junit.Test;
+import java.util.Arrays;
+
public class PerceptronUDTFTest {
@Test
@@ -118,4 +123,14 @@ public class PerceptronUDTFTest {
assertEquals(-1.f, udtf.model.get(word3.getFeature()).get(), 1e-5f);
assertEquals(0.f, udtf.model.get(word4.getFeature()).get(), 1e-5f);
}
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ PerceptronUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 0}});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/fm/FactorizationMachineUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/fm/FactorizationMachineUDTFTest.java b/core/src/test/java/hivemall/fm/FactorizationMachineUDTFTest.java
index 479f5cf..fb2b4a2 100644
--- a/core/src/test/java/hivemall/fm/FactorizationMachineUDTFTest.java
+++ b/core/src/test/java/hivemall/fm/FactorizationMachineUDTFTest.java
@@ -23,12 +23,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.StringTokenizer;
import java.util.zip.GZIPInputStream;
import javax.annotation.Nonnull;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -102,6 +104,19 @@ public class FactorizationMachineUDTFTest {
Assert.assertTrue(udtf._params.l2norm);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ FactorizationMachineUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-factors 5 -min 1 -max 5 -iters 1 -init_v gaussian -eta0 0.01 -seed 31")},
+ new Object[][] {{Arrays.asList("1:-2", "2:-1"), 1.0}});
+ }
+
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
InputStream is = FactorizationMachineUDTFTest.class.getResourceAsStream(fileName);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java b/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
index 585392b..604b355 100644
--- a/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
+++ b/core/src/test/java/hivemall/fm/FieldAwareFactorizationMachineUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.fm;
+import hivemall.TestUtils;
import hivemall.utils.lang.NumberUtils;
import java.io.BufferedReader;
@@ -26,6 +27,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.zip.GZIPInputStream;
@@ -212,6 +214,19 @@ public class FieldAwareFactorizationMachineUDTFTest {
Assert.assertTrue("Last loss was greater than expected: " + loss, loss < lossThreshold);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ FieldAwareFactorizationMachineUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-opt sgd -classification -factors 10 -w0 -seed 43")}, new Object[][] {{
+ Arrays.asList("0:1:-2", "1:2:-1"), 1.0}});
+ }
+
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
InputStream is;
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/ftvec/FeatureUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/FeatureUDFTest.java b/core/src/test/java/hivemall/ftvec/FeatureUDFTest.java
index 55dd803..17a937e 100644
--- a/core/src/test/java/hivemall/ftvec/FeatureUDFTest.java
+++ b/core/src/test/java/hivemall/ftvec/FeatureUDFTest.java
@@ -18,7 +18,9 @@
*/
package hivemall.ftvec;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
@@ -29,6 +31,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class FeatureUDFTest {
FeatureUDF udf = null;
@@ -225,4 +229,13 @@ public class FeatureUDFTest {
Assert.assertNull(ret);
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(FeatureUDF.class, new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}, new Object[] {"f1",
+ 2.5d});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java b/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
index 061a971..82b5cde 100644
--- a/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
+++ b/core/src/test/java/hivemall/ftvec/hashing/FeatureHashingUDFTest.java
@@ -18,11 +18,20 @@
*/
package hivemall.ftvec.hashing;
+import hivemall.TestUtils;
import hivemall.utils.hashing.MurmurHash3;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
+import java.util.Arrays;
+
public class FeatureHashingUDFTest {
@Test
@@ -41,4 +50,15 @@ public class FeatureHashingUDFTest {
FeatureHashingUDF.mhash("0", MurmurHash3.DEFAULT_NUM_FEATURES) + ":1.1", actual);
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ FeatureHashingUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-features 1")},
+ new Object[] {Arrays.asList("aaa#xxx", "bbb:10")});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTFTest.java b/core/src/test/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTFTest.java
new file mode 100644
index 0000000..4cfc20e
--- /dev/null
+++ b/core/src/test/java/hivemall/ftvec/trans/QuantifiedFeaturesUDTFTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package hivemall.ftvec.trans;
+
+import hivemall.TestUtils;
+import hivemall.utils.hadoop.WritableUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+import org.apache.hadoop.hive.ql.udf.generic.Collector;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class QuantifiedFeaturesUDTFTest {
+
+ @Test
+ public void test() throws HiveException {
+ final QuantifiedFeaturesUDTF udtf = new QuantifiedFeaturesUDTF();
+
+ udtf.initialize(new ObjectInspector[] {
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true),
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});
+
+ final List<List<Double>> quantifiedInputs = new ArrayList<>();
+ udtf.setCollector(new Collector() {
+ public void collect(Object input) throws HiveException {
+ Object[] row = (Object[]) input;
+ List<DoubleWritable> column = (List<DoubleWritable>) row[0];
+ List<Double> quantifiedInput = new ArrayList<>();
+ for (DoubleWritable elem : column) {
+ quantifiedInput.add(elem.get());
+ }
+ quantifiedInputs.add(quantifiedInput);
+ }
+ });
+
+ udtf.process(new Object[] {WritableUtils.val(true), "aaa", 1.0});
+ udtf.process(new Object[] {WritableUtils.val(true), "bbb", 2.0});
+
+ udtf.close();
+
+ Assert.assertEquals(2, quantifiedInputs.size());
+
+ List<Double> quantifiedInput = quantifiedInputs.get(0);
+ Assert.assertTrue(quantifiedInput.get(0) == 0.d);
+ Assert.assertTrue(quantifiedInput.get(1) == 1.d);
+
+ quantifiedInput = quantifiedInputs.get(1);
+ Assert.assertTrue(quantifiedInput.get(0) == 1.d);
+ Assert.assertTrue(quantifiedInput.get(1) == 2.d);
+ }
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ QuantifiedFeaturesUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true),
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}, new Object[][] {{
+ WritableUtils.val(true), "aaa", 1.0}});
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/ftvec/trans/TestBinarizeLabelUDTF.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/trans/TestBinarizeLabelUDTF.java b/core/src/test/java/hivemall/ftvec/trans/TestBinarizeLabelUDTF.java
index a8d2beb..67e0a3e 100644
--- a/core/src/test/java/hivemall/ftvec/trans/TestBinarizeLabelUDTF.java
+++ b/core/src/test/java/hivemall/ftvec/trans/TestBinarizeLabelUDTF.java
@@ -24,6 +24,8 @@ import static org.mockito.Mockito.times;
import static org.powermock.api.mockito.PowerMockito.doNothing;
import static org.powermock.api.mockito.PowerMockito.spy;
import static org.powermock.api.mockito.PowerMockito.verifyPrivate;
+
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.util.Arrays;
@@ -104,4 +106,17 @@ public class TestBinarizeLabelUDTF {
verifyPrivate(udtf, times(0)).invoke("forward", any(Object[].class));
}
+
+ @Test
+ public void testSerialization() throws HiveException {
+ final List<String> featureNames = Arrays.asList("positive", "negative", "features");
+ TestUtils.testGenericUDTFSerialization(
+ BinarizeLabelUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorFactory.getStandardConstantListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames)},
+ new Object[][] {{new Integer(0), new Integer(0), WritableUtils.val("a:1", "b:2")}});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java b/core/src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java
index e826963..cf28350 100644
--- a/core/src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java
+++ b/core/src/test/java/hivemall/ftvec/trans/VectorizeFeaturesUDFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.ftvec.trans;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
@@ -88,7 +89,7 @@ public class VectorizeFeaturesUDFTest {
arguments[2] = new DeferredJavaObject("1.1");
List<Text> actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
List<Text> expected = WritableUtils.val("a:0.1", "b:1.1");
Assert.assertEquals(expected, actuals);
@@ -111,7 +112,7 @@ public class VectorizeFeaturesUDFTest {
arguments[2] = new DeferredJavaObject("0");
List<Text> actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
Assert.assertEquals(expected, actuals);
@@ -134,7 +135,7 @@ public class VectorizeFeaturesUDFTest {
arguments[2] = new DeferredJavaObject(new Boolean(false));
List<Text> actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
List<Text> expected = WritableUtils.val(new String[] {"a:0.1"});
Assert.assertEquals(expected, actuals);
@@ -163,19 +164,19 @@ public class VectorizeFeaturesUDFTest {
arguments[2] = new DeferredJavaObject("dayofweek");
List<Text> actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
List<Text> expected = WritableUtils.val("a:0.1", "b#dayofweek");
Assert.assertEquals(expected, actuals);
arguments[2] = new DeferredJavaObject("1.0");
actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
expected = WritableUtils.val("a:0.1", "b:1.0");
Assert.assertEquals(expected, actuals);
arguments[2] = new DeferredJavaObject("1");
actuals = udf.evaluate(arguments);
- //System.out.println(actuals);
+ //System.out.println(actuals);
expected = WritableUtils.val("a:0.1", "b:1.0");
Assert.assertEquals(expected, actuals);
@@ -188,4 +189,18 @@ public class VectorizeFeaturesUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ final List<String> featureNames = Arrays.asList("q", "c");
+
+ TestUtils.testGenericUDFSerialization(
+ VectorizeFeaturesUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardConstantListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, featureNames),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector}, new Object[] {
+ featureNames, 0.1d, "dayofweek"});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java b/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java
index af5316a..5ec709b 100644
--- a/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java
+++ b/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java
@@ -20,6 +20,7 @@ package hivemall.geospatial;
import java.io.IOException;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -102,4 +103,23 @@ public class HaversineDistanceUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ // Tokyo
+ double lat1 = 35.6833d, lon1 = 139.7667d;
+ // Osaka
+ double lat2 = 34.6603d, lon2 = 135.5232d;
+
+ TestUtils.testGenericUDFSerialization(
+ HaversineDistanceUDF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)},
+ new Object[] {lat1, lon1, lat2, lon2, true});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java b/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java
index 8fa1ada..e4006db 100644
--- a/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java
+++ b/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java
@@ -20,6 +20,7 @@ package hivemall.geospatial;
import java.io.IOException;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -45,4 +46,12 @@ public class Lat2TileYUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(Lat2TileYUDF.class, new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[] {49.60055d,
+ 13});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java b/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java
index cd82826..6191321 100644
--- a/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java
+++ b/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java
@@ -20,6 +20,7 @@ package hivemall.geospatial;
import java.io.IOException;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -45,4 +46,12 @@ public class Lon2TileXUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(Lon2TileXUDF.class, new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[] {11.01296d,
+ 13});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java b/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java
index ce2b0c9..1bbe4a4 100644
--- a/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java
+++ b/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java
@@ -20,6 +20,7 @@ package hivemall.geospatial;
import java.io.IOException;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -45,4 +46,11 @@ public class TileX2LonUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(TileX2LonUDF.class, new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[] {3551, 13});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java b/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java
index 5f4b516..688b509 100644
--- a/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java
+++ b/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java
@@ -20,6 +20,7 @@ package hivemall.geospatial;
import java.io.IOException;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -45,4 +46,11 @@ public class TileY2LatUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(TileY2LatUDF.class, new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector}, new Object[] {503, 14});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/knn/distance/EuclidDistanceUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/knn/distance/EuclidDistanceUDFTest.java b/core/src/test/java/hivemall/knn/distance/EuclidDistanceUDFTest.java
index e91ec6d..64bfe1b 100644
--- a/core/src/test/java/hivemall/knn/distance/EuclidDistanceUDFTest.java
+++ b/core/src/test/java/hivemall/knn/distance/EuclidDistanceUDFTest.java
@@ -18,9 +18,15 @@
*/
package hivemall.knn.distance;
+import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import hivemall.TestUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.junit.Assert;
import org.junit.Test;
@@ -42,4 +48,14 @@ public class EuclidDistanceUDFTest {
Assert.assertEquals(Math.sqrt(1.0 + 9.0 + 9.0), d, 0.f);
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ EuclidDistanceUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)},
+ new Object[] {Arrays.asList("1:1.0", "2:3.0", "3:3.0"), Arrays.asList("1:2.0", "3:6.0")});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/knn/similarity/CosineSimilarityUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/knn/similarity/CosineSimilarityUDFTest.java b/core/src/test/java/hivemall/knn/similarity/CosineSimilarityUDFTest.java
index 9825e9b..9f319a9 100644
--- a/core/src/test/java/hivemall/knn/similarity/CosineSimilarityUDFTest.java
+++ b/core/src/test/java/hivemall/knn/similarity/CosineSimilarityUDFTest.java
@@ -22,6 +22,11 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import hivemall.TestUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.junit.Assert;
import org.junit.Test;
@@ -107,4 +112,14 @@ public class CosineSimilarityUDFTest {
CosineSimilarityUDF.cosineSimilarity(Arrays.asList("1", "2"), Arrays.asList("1", "2")),
0.0);
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ CosineSimilarityUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)},
+ new Object[] {Arrays.asList("1:1.0", "2:3.0", "3:3.0"), Arrays.asList("1:2.0", "3:6.0")});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/knn/similarity/DIMSUMMapperUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/knn/similarity/DIMSUMMapperUDTFTest.java b/core/src/test/java/hivemall/knn/similarity/DIMSUMMapperUDTFTest.java
index 898b4cb..766dade 100644
--- a/core/src/test/java/hivemall/knn/similarity/DIMSUMMapperUDTFTest.java
+++ b/core/src/test/java/hivemall/knn/similarity/DIMSUMMapperUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.knn.similarity;
+import hivemall.TestUtils;
import hivemall.mf.BPRMatrixFactorizationUDTFTest;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.lang.StringUtils;
@@ -357,6 +358,29 @@ public class DIMSUMMapperUDTFTest {
emitCounter.getValue() < numMaxEmits);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ final Integer[] itemIDs = new Integer[] {1, 2, 3};
+
+ final List<String> user = new ArrayList<String>();
+ convertRowToFeatures(0, user, itemIDs);
+
+ final Map<Integer, Double> norms = new HashMap<Integer, Double>();
+ computeColumnNorms(norms, itemIDs);
+
+ TestUtils.testGenericUDTFSerialization(
+ DIMSUMMapperUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-threshold 0.999999 -disable_symmetric_output")}, new Object[][] {{user,
+ norms}});
+ }
+
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
// use MF's resource file
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/mf/BPRMatrixFactorizationUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/mf/BPRMatrixFactorizationUDTFTest.java b/core/src/test/java/hivemall/mf/BPRMatrixFactorizationUDTFTest.java
index 41f1f97..d8dd820 100644
--- a/core/src/test/java/hivemall/mf/BPRMatrixFactorizationUDTFTest.java
+++ b/core/src/test/java/hivemall/mf/BPRMatrixFactorizationUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.mf;
+import hivemall.TestUtils;
import hivemall.utils.lang.StringUtils;
import java.io.BufferedReader;
@@ -111,6 +112,19 @@ public class BPRMatrixFactorizationUDTFTest {
Assert.assertTrue("finishedIter: " + finishedIter, finishedIter < iterations);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ BPRMatrixFactorizationUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-factor 10 -iter 1")}, new Object[][] {{0, 0, 1}});
+ }
+
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
InputStream is = BPRMatrixFactorizationUDTFTest.class.getResourceAsStream(fileName);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/mf/MatrixFactorizationAdaGradUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/mf/MatrixFactorizationAdaGradUDTFTest.java b/core/src/test/java/hivemall/mf/MatrixFactorizationAdaGradUDTFTest.java
index e587f01..64f54fa 100644
--- a/core/src/test/java/hivemall/mf/MatrixFactorizationAdaGradUDTFTest.java
+++ b/core/src/test/java/hivemall/mf/MatrixFactorizationAdaGradUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.mf;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -80,4 +81,17 @@ public class MatrixFactorizationAdaGradUDTFTest {
}
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ MatrixFactorizationAdaGradUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-factor 10")},
+ new Object[][] {{0, 0, 5.f}});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/mf/MatrixFactorizationSGDUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/mf/MatrixFactorizationSGDUDTFTest.java b/core/src/test/java/hivemall/mf/MatrixFactorizationSGDUDTFTest.java
index 53983c1..1f35da1 100644
--- a/core/src/test/java/hivemall/mf/MatrixFactorizationSGDUDTFTest.java
+++ b/core/src/test/java/hivemall/mf/MatrixFactorizationSGDUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.mf;
+import hivemall.TestUtils;
import hivemall.mf.FactorizedModel.RankInitScheme;
import hivemall.utils.lang.mutable.MutableInt;
@@ -343,4 +344,17 @@ public class MatrixFactorizationSGDUDTFTest {
Assert.assertEquals(5, numCollected.intValue());
Assert.assertFalse(tmpFile.exists());
}
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ MatrixFactorizationSGDUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-factor 10")},
+ new Object[][] {{0, 0, 5.f}});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/recommend/SlimUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/recommend/SlimUDTFTest.java b/core/src/test/java/hivemall/recommend/SlimUDTFTest.java
index 00b78f0..0bb57d3 100644
--- a/core/src/test/java/hivemall/recommend/SlimUDTFTest.java
+++ b/core/src/test/java/hivemall/recommend/SlimUDTFTest.java
@@ -21,6 +21,7 @@ package hivemall.recommend;
import java.util.HashMap;
import java.util.Map;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -96,4 +97,75 @@ public class SlimUDTFTest {
slim.finalizeTraining();
}
+ @Test
+ public void testSerialization() throws HiveException {
+ int numUser = 4;
+ int numItem = 5;
+
+ float[][] data = { {1.f, 4.f, 0.f, 0.f, 0.f}, {0.f, 3.f, 0.f, 1.f, 2.f},
+ {2.f, 2.f, 0.f, 0.f, 3.f}, {0.f, 1.f, 1.f, 0.f, 0.f}};
+
+ Object[][] rows = new Object[numItem * (numItem - 1)][5];
+ int ri = 0;
+
+ for (int i = 0; i < numItem; i++) {
+ Map<Integer, Float> Ri = new HashMap<>();
+ for (int u = 0; u < numUser; u++) {
+ if (data[u][i] != 0.) {
+ Ri.put(u, data[u][i]);
+ }
+ }
+
+ // most similar data
+ Map<Integer, Map<Integer, Float>> knnRatesOfI = new HashMap<>();
+ for (int u = 0; u < numUser; u++) {
+ Map<Integer, Float> Ru = new HashMap<>();
+ for (int k = 0; k < numItem; k++) {
+ if (k == i)
+ continue;
+ Ru.put(k, data[u][k]);
+ }
+ knnRatesOfI.put(u, Ru);
+ }
+
+ for (int j = 0; j < numItem; j++) {
+ if (i == j)
+ continue;
+ Map<Integer, Float> Rj = new HashMap<>();
+ for (int u = 0; u < numUser; u++) {
+ if (data[u][j] != 0.) {
+ Rj.put(u, data[u][j]);
+ }
+ }
+
+ rows[ri][0] = i;
+ rows[ri][1] = Ri;
+ rows[ri][2] = knnRatesOfI;
+ rows[ri][3] = j;
+ rows[ri][4] = Rj;
+ ri += 1;
+ }
+ }
+
+ TestUtils.testGenericUDTFSerialization(
+ SlimUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector),
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector)),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorFactory.getStandardMapObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-l2 0.01 -l1 0.01")}, rows);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/regression/AdaGradUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/regression/AdaGradUDTFTest.java b/core/src/test/java/hivemall/regression/AdaGradUDTFTest.java
index fa7e28a..f80c579 100644
--- a/core/src/test/java/hivemall/regression/AdaGradUDTFTest.java
+++ b/core/src/test/java/hivemall/regression/AdaGradUDTFTest.java
@@ -20,7 +20,9 @@ package hivemall.regression;
import static org.junit.Assert.assertEquals;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -28,6 +30,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.junit.Test;
+import java.util.Arrays;
+
public class AdaGradUDTFTest {
@SuppressWarnings("deprecation")
@@ -57,4 +61,14 @@ public class AdaGradUDTFTest {
labelOI});
assertEquals("struct<feature:bigint,weight:float>", longListSOI.getTypeName());
}
+
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ AdaGradUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector}, new Object[][] {{
+ Arrays.asList("1:-2", "2:-1"), 1.f}});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/regression/GeneralRegressorUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/regression/GeneralRegressorUDTFTest.java b/core/src/test/java/hivemall/regression/GeneralRegressorUDTFTest.java
index efc0699..da33f5d 100644
--- a/core/src/test/java/hivemall/regression/GeneralRegressorUDTFTest.java
+++ b/core/src/test/java/hivemall/regression/GeneralRegressorUDTFTest.java
@@ -28,6 +28,7 @@ import java.util.List;
import javax.annotation.Nonnull;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.Collector;
@@ -165,7 +166,7 @@ public class GeneralRegressorUDTFTest {
}
@Test(expected = IllegalArgumentException.class)
- public void testIlleagalStringFeature() throws Exception {
+ public void testIllegalStringFeature() throws Exception {
List<String> x = Arrays.asList("1:-2jjjj", "2:-1");
ObjectInspector featureOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
testFeature(x, featureOI, String.class, String.class);
@@ -323,6 +324,19 @@ public class GeneralRegressorUDTFTest {
}
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ GeneralRegressorUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ "-loss SquaredLoss")},
+ new Object[][] {{Arrays.asList("1:-2", "2:-1"), 10.f}});
+ }
+
private static void println(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/smile/classification/RandomForestClassifierUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/smile/classification/RandomForestClassifierUDTFTest.java b/core/src/test/java/hivemall/smile/classification/RandomForestClassifierUDTFTest.java
index 578689c..903822b 100644
--- a/core/src/test/java/hivemall/smile/classification/RandomForestClassifierUDTFTest.java
+++ b/core/src/test/java/hivemall/smile/classification/RandomForestClassifierUDTFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.smile.classification;
+import hivemall.TestUtils;
import hivemall.classifier.KernelExpansionPassiveAggressiveUDTF;
import hivemall.utils.codec.Base91;
import hivemall.utils.lang.mutable.MutableInt;
@@ -360,6 +361,40 @@ public class RandomForestClassifierUDTFTest {
Assert.assertTrue("oob error rate is too high: " + oobErrorRate, oobErrorRate < 0.3);
}
+ @Test
+ public void testSerialization() throws HiveException, IOException, ParseException {
+ URL url = new URL(
+ "https://gist.githubusercontent.com/myui/143fa9d05bd6e7db0114/raw/500f178316b802f1cade6e3bf8dc814a96e84b1e/iris.arff");
+ InputStream is = new BufferedInputStream(url.openStream());
+
+ ArffParser arffParser = new ArffParser();
+ arffParser.setResponseIndex(4);
+
+ AttributeDataset iris = arffParser.parse(is);
+ int size = iris.size();
+ double[][] x = iris.toArray(new double[size][]);
+ int[] y = iris.toArray(new int[size]);
+
+ final Object[][] rows = new Object[size][2];
+ for (int i = 0; i < size; i++) {
+ double[] row = x[i];
+ final List<String> xi = new ArrayList<String>(x[0].length);
+ for (int j = 0; j < row.length; j++) {
+ xi.add(j + ":" + row[j]);
+ }
+ rows[i][0] = xi;
+ rows[i][1] = y[i];
+ }
+
+ TestUtils.testGenericUDTFSerialization(
+ RandomForestClassifierUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "-trees 49")},
+ rows);
+ }
@Nonnull
private static BufferedReader readFile(@Nonnull String fileName) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/smile/tools/TreePredictUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/smile/tools/TreePredictUDFTest.java b/core/src/test/java/hivemall/smile/tools/TreePredictUDFTest.java
index 31713d9..1c21e0d 100644
--- a/core/src/test/java/hivemall/smile/tools/TreePredictUDFTest.java
+++ b/core/src/test/java/hivemall/smile/tools/TreePredictUDFTest.java
@@ -18,6 +18,7 @@
*/
package hivemall.smile.tools;
+import hivemall.TestUtils;
import hivemall.math.matrix.dense.RowMajorDenseMatrix2d;
import hivemall.smile.classification.DecisionTree;
import hivemall.smile.data.Attribute;
@@ -206,6 +207,55 @@ public class TreePredictUDFTest {
return result.get();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException, ParseException {
+ URL url = new URL(
+ "https://gist.githubusercontent.com/myui/ef17aabecf0c0c5bcb69/raw/aac0575b4d43072c6f3c82d9072fdefb61892694/cpu.arff");
+ InputStream is = new BufferedInputStream(url.openStream());
+
+ ArffParser arffParser = new ArffParser();
+ arffParser.setResponseIndex(6);
+ AttributeDataset data = arffParser.parse(is);
+ double[] datay = data.toArray(new double[data.size()]);
+ double[][] datax = data.toArray(new double[data.size()][]);
+
+ int n = datax.length;
+ int m = 3 * n / 4;
+ int[] index = Math.permutate(n);
+
+ double[][] trainx = new double[m][];
+ double[] trainy = new double[m];
+ for (int i = 0; i < m; i++) {
+ trainx[i] = datax[index[i]];
+ trainy[i] = datay[index[i]];
+ }
+
+ double[][] testx = new double[n - m][];
+ double[] testy = new double[n - m];
+ for (int i = m; i < n; i++) {
+ testx[i - m] = datax[index[i]];
+ testy[i - m] = datay[index[i]];
+ }
+
+ Attribute[] attrs = SmileExtUtils.convertAttributeTypes(data.attributes());
+ RegressionTree tree = new RegressionTree(attrs, new RowMajorDenseMatrix2d(trainx,
+ trainx[0].length), trainy, 20);
+
+ byte[] b = tree.serialize(true);
+ byte[] encoded = Base91.encode(b);
+ Text model = new Text(encoded);
+
+ TestUtils.testGenericUDFSerialization(
+ TreePredictUDF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)},
+ new Object[] {"model_id#1", model, ArrayUtils.toList(testx[0])});
+ }
+
private static void debugPrint(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/smile/tools/TreePredictUDFv1Test.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/smile/tools/TreePredictUDFv1Test.java b/core/src/test/java/hivemall/smile/tools/TreePredictUDFv1Test.java
index 4251ca9..779eb73 100644
--- a/core/src/test/java/hivemall/smile/tools/TreePredictUDFv1Test.java
+++ b/core/src/test/java/hivemall/smile/tools/TreePredictUDFv1Test.java
@@ -20,6 +20,7 @@ package hivemall.smile.tools;
import static org.junit.Assert.assertEquals;
+import hivemall.TestUtils;
import hivemall.math.matrix.dense.RowMajorDenseMatrix2d;
import hivemall.smile.classification.DecisionTree;
import hivemall.smile.data.Attribute;
@@ -228,6 +229,54 @@ public class TreePredictUDFv1Test {
return result.get();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException, ParseException {
+ URL url = new URL(
+ "https://gist.githubusercontent.com/myui/ef17aabecf0c0c5bcb69/raw/aac0575b4d43072c6f3c82d9072fdefb61892694/cpu.arff");
+ InputStream is = new BufferedInputStream(url.openStream());
+
+ ArffParser arffParser = new ArffParser();
+ arffParser.setResponseIndex(6);
+ AttributeDataset data = arffParser.parse(is);
+ double[] datay = data.toArray(new double[data.size()]);
+ double[][] datax = data.toArray(new double[data.size()][]);
+
+ int n = datax.length;
+ int m = 3 * n / 4;
+ int[] index = Math.permutate(n);
+
+ double[][] trainx = new double[m][];
+ double[] trainy = new double[m];
+ for (int i = 0; i < m; i++) {
+ trainx[i] = datax[index[i]];
+ trainy[i] = datay[index[i]];
+ }
+
+ double[][] testx = new double[n - m][];
+ double[] testy = new double[n - m];
+ for (int i = m; i < n; i++) {
+ testx[i - m] = datax[index[i]];
+ testy[i - m] = datay[index[i]];
+ }
+
+ Attribute[] attrs = SmileExtUtils.convertAttributeTypes(data.attributes());
+ RegressionTree tree = new RegressionTree(attrs, new RowMajorDenseMatrix2d(trainx,
+ trainx[0].length), trainy, 20);
+ String opScript = tree.predictOpCodegen(StackMachine.SEP);
+
+ TestUtils.testGenericUDFSerialization(
+ TreePredictUDFv1.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)},
+ new Object[] {"model_id#1", ModelType.opscode.getId(), opScript,
+ ArrayUtils.toList(testx[0])});
+ }
+
private static void debugPrint(String msg) {
if (DEBUG) {
System.out.println(msg);
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java b/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
index 3ab7e8b..4439bf5 100644
--- a/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
+++ b/core/src/test/java/hivemall/statistics/MovingAverageUDTFTest.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.Collector;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
@@ -65,4 +66,15 @@ public class MovingAverageUDTFTest {
Assert.assertEquals(Arrays.asList(1.d, 1.5d, 2.d, 3.d, 4.d, 5.d, 6.d), results);
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ MovingAverageUDTF.class,
+ new ObjectInspector[] {
+ PrimitiveObjectInspectorFactory.javaFloatObjectInspector,
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector, 3)},
+ new Object[][] { {1.f}, {2.f}, {3.f}, {4.f}, {5.f}});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/TryCastUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/TryCastUDFTest.java b/core/src/test/java/hivemall/tools/TryCastUDFTest.java
index 7cd75ba..09235b2 100644
--- a/core/src/test/java/hivemall/tools/TryCastUDFTest.java
+++ b/core/src/test/java/hivemall/tools/TryCastUDFTest.java
@@ -18,15 +18,18 @@
*/
package hivemall.tools;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Text;
@@ -55,4 +58,15 @@ public class TryCastUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ TryCastUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorUtils.getConstantObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, "array<string>")},
+ new Object[] {Arrays.asList(1.d, 2.d, 3.d)});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
index 113e993..74ae2d5 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayAppendUDFTest.java
@@ -18,9 +18,11 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -85,7 +87,7 @@ public class ArrayAppendUDFTest {
ArrayAppendUDF udf = new ArrayAppendUDF();
udf.initialize(new ObjectInspector[] {
- ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
PrimitiveObjectInspectorFactory.javaDoubleObjectInspector});
DeferredObject[] args = new DeferredObject[] {new GenericUDF.DeferredJavaObject(null),
@@ -98,4 +100,14 @@ public class ArrayAppendUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ArrayAppendUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}, new Object[] {
+ Arrays.asList(0.d, 1.d, 2.d), 3.d});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ArrayElementAtUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayElementAtUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayElementAtUDFTest.java
index 95ef1a2..39a5cd9 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayElementAtUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayElementAtUDFTest.java
@@ -18,9 +18,11 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -78,4 +80,14 @@ public class ArrayElementAtUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ArrayElementAtUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector},
+ new Object[] {Arrays.asList(0.d, 1.d, 2.d), 1});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
index 3824fd5..309dfad 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayFlattenUDFTest.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -51,4 +52,13 @@ public class ArrayFlattenUDFTest {
udf.close();
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ArrayFlattenUDF.class,
+ new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector))},
+ new Object[] {Arrays.asList(Arrays.asList(0, 1, 2, 3), Arrays.asList(4, 5),
+ Arrays.asList(6, 7))});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java b/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
index b18ae54..760e476 100644
--- a/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArraySliceUDFTest.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
@@ -110,4 +111,17 @@ public class ArraySliceUDFTest {
udf.close();
}
+
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ArraySliceUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector),
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector},
+ new Object[] {
+ Arrays.asList("zero", "one", "two", "three", "four", "five", "six", "seven",
+ "eight", "nine", "ten"), 2, 5});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java b/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
index 1138e9a..eca4212 100644
--- a/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ArrayUnionUDFTest.java
@@ -18,9 +18,11 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -58,4 +60,13 @@ public class ArrayUnionUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ ArrayUnionUDF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
+ new Object[] {Arrays.asList(0.d, 1.d), Arrays.asList(2.d, 3.d)});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
index 9564cef..70ad25e 100644
--- a/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
+++ b/core/src/test/java/hivemall/tools/array/ConditionalEmitUDTFTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import hivemall.TestUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.Collector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -65,4 +66,15 @@ public class ConditionalEmitUDTFTest {
udtf.close();
}
+ @Test
+ public void testSerialization() throws HiveException {
+ TestUtils.testGenericUDTFSerialization(
+ ConditionalEmitUDTF.class,
+ new ObjectInspector[] {
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector),
+ ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector)},
+ new Object[][] {
+ {Arrays.asList(true, false, true), Arrays.asList("one", "two", "three")},
+ {Arrays.asList(true, true, false), Arrays.asList("one", "two", "three")}});
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java b/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java
index 73ba47a..241b8b9 100644
--- a/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/FirstElementUDFTest.java
@@ -18,9 +18,11 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -61,4 +63,12 @@ public class FirstElementUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ FirstElementUDF.class,
+ new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
+ new Object[] {Arrays.asList(0.d, 1.d, 2.d)});
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/b64b94f9/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java b/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java
index 38c7361..325c28a 100644
--- a/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java
+++ b/core/src/test/java/hivemall/tools/array/LastElementUDFTest.java
@@ -18,9 +18,11 @@
*/
package hivemall.tools.array;
+import hivemall.TestUtils;
import hivemall.utils.hadoop.WritableUtils;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -61,4 +63,12 @@ public class LastElementUDFTest {
udf.close();
}
+ @Test
+ public void testSerialization() throws HiveException, IOException {
+ TestUtils.testGenericUDFSerialization(
+ FirstElementUDF.class,
+ new ObjectInspector[] {ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector)},
+ new Object[] {Arrays.asList(0.d, 1.d, 2.d)});
+ }
+
}