You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2019/02/06 08:16:55 UTC
[incubator-hivemall] branch master updated (5ff827d -> 5afa926)
This is an automated email from the ASF dual-hosted git repository.
myui pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git.
from 5ff827d [HIVEMALL-236] to_json/from_json cause KryoException/NullPointerException with ArrayList due to Kryo bug
new 8d8d701 Modified to_string_array to be a generic UDF
new 5afa926 Added usages
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hivemall/tools/array/ArrayAvgGenericUDAF.java | 17 +++++-
.../java/hivemall/tools/array/ArraySumUDAF.java | 16 +++++-
.../hivemall/tools/array/ToStringArrayUDF.java | 64 +++++++++++++++++++---
3 files changed, 86 insertions(+), 11 deletions(-)
[incubator-hivemall] 02/02: Added usages
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
commit 5afa926de99181827e205a4192e97c2b5f4dcad9
Author: Makoto Yui <my...@apache.org>
AuthorDate: Wed Feb 6 17:16:24 2019 +0900
Added usages
---
.../java/hivemall/tools/array/ArrayAvgGenericUDAF.java | 17 ++++++++++++++++-
.../main/java/hivemall/tools/array/ArraySumUDAF.java | 16 +++++++++++++++-
2 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java b/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
index caaf84e..ad9c39a 100644
--- a/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
+++ b/core/src/main/java/hivemall/tools/array/ArrayAvgGenericUDAF.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hive.ql.util.JavaDataModel.JAVA64_ARRAY_META;
import static org.apache.hadoop.hive.ql.util.JavaDataModel.JAVA64_REF;
import static org.apache.hadoop.hive.ql.util.JavaDataModel.PRIMITIVES1;
import static org.apache.hadoop.hive.ql.util.JavaDataModel.PRIMITIVES2;
+
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.hadoop.WritableUtils;
@@ -57,8 +58,22 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
+// @formatter:off
@Description(name = "array_avg", value = "_FUNC_(array<number>) - Returns an array<double>"
- + " in which each element is the mean of a set of numbers")
+ + " in which each element is the mean of a set of numbers",
+ extended = "WITH input as (\n" +
+ " select array(1.0, 2.0, 3.0) as nums\n" +
+ " UNION ALL\n" +
+ " select array(2.0, 3.0, 4.0) as nums\n" +
+ ")\n" +
+ "select\n" +
+ " array_avg(nums)\n" +
+ "from\n" +
+ " input;\n" +
+ "\n" +
+ "[\"1.5\",\"2.5\",\"3.5\"]"
+ )
+// @formatter:on
public final class ArrayAvgGenericUDAF extends AbstractGenericUDAFResolver {
@Override
diff --git a/core/src/main/java/hivemall/tools/array/ArraySumUDAF.java b/core/src/main/java/hivemall/tools/array/ArraySumUDAF.java
index 8a47957..a1431c4 100644
--- a/core/src/main/java/hivemall/tools/array/ArraySumUDAF.java
+++ b/core/src/main/java/hivemall/tools/array/ArraySumUDAF.java
@@ -31,9 +31,23 @@ import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+//@formatter:off
@SuppressWarnings("deprecation")
@Description(name = "array_sum", value = "_FUNC_(array<number>) - Returns an array<double>"
- + " in which each element is summed up")
+ + " in which each element is summed up",
+ extended = "WITH input as (\n" +
+ " select array(1.0, 2.0, 3.0) as nums\n" +
+ " UNION ALL\n" +
+ " select array(2.0, 3.0, 4.0) as nums\n" +
+ ")\n" +
+ "select\n" +
+ " array_sum(nums)\n" +
+ "from\n" +
+ " input;\n" +
+ "\n" +
+ "[\"3.0\",\"5.0\",\"7.0\"]"
+ )
+// @formatter:on
public final class ArraySumUDAF extends UDAF {
public ArraySumUDAF() {}
[incubator-hivemall] 01/02: Modified to_string_array to be a
generic UDF
Posted by my...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
commit 8d8d70138ef69cfedf02bacd3a2f937596141730
Author: Makoto Yui <my...@apache.org>
AuthorDate: Wed Feb 6 17:15:47 2019 +0900
Modified to_string_array to be a generic UDF
---
.../hivemall/tools/array/ToStringArrayUDF.java | 64 +++++++++++++++++++---
1 file changed, 55 insertions(+), 9 deletions(-)
diff --git a/core/src/main/java/hivemall/tools/array/ToStringArrayUDF.java b/core/src/main/java/hivemall/tools/array/ToStringArrayUDF.java
index 306673c..d1b7454 100644
--- a/core/src/main/java/hivemall/tools/array/ToStringArrayUDF.java
+++ b/core/src/main/java/hivemall/tools/array/ToStringArrayUDF.java
@@ -18,22 +18,68 @@
*/
package hivemall.tools.array;
-import java.util.List;
+import hivemall.utils.hadoop.HiveUtils;
+import hivemall.utils.lang.StringUtils;
-import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-@Description(name = "to_string_array", value = "_FUNC_(array<ANY>) - Returns an array of strings")
+@Description(name = "to_string_array", value = "_FUNC_(array<ANY>) - Returns an array of strings",
+ extended = "select to_string_array(array(1.0,2.0,3.0));\n\n" + "[\"1.0\",\"2.0\",\"3.0\"]")
@UDFType(deterministic = true, stateful = false)
-public final class ToStringArrayUDF extends UDF {
+public final class ToStringArrayUDF extends GenericUDF {
+
+ private ListObjectInspector listOI;
+ private List<String> result;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
+ if (argOIs.length != 1) {
+ throw new UDFArgumentException(
+ "to_string_array expects exactly one argument: " + argOIs.length);
+ }
+ this.listOI = HiveUtils.asListOI(argOIs[0]);
+ this.result = new ArrayList<>();
+
+ return ObjectInspectorFactory.getStandardListObjectInspector(
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector);
+ }
+
+ @Override
+ public List<String> evaluate(DeferredObject[] arguments) throws HiveException {
+ final Object arg0 = arguments[0].get();
+ if (arg0 == null) {
+ return null;
+ }
+
+ result.clear();
+
+ final int len = listOI.getListLength(arg0);
+ for (int i = 0; i < len; i++) {
+ final Object e = listOI.getListElement(arg0, i);
+ if (e == null) {
+ result.add(null);
+ } else {
+ result.add(e.toString());
+ }
+ }
+
+ return result;
+ }
- @Nullable
- public List<Text> evaluate(@Nullable final List<Text> inArray) {
- return inArray;
+ @Override
+ public String getDisplayString(String[] children) {
+ return "to_string_array(" + StringUtils.join(children, ',') + ')';
}
}