You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2020/06/04 05:21:17 UTC
[incubator-hivemall] branch master updated: [HIVEMALL-295][BUGFIX]
transpose_and_dot throws UDFArgumentException for 0 rows input
This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new 04fa754 [HIVEMALL-295][BUGFIX] transpose_and_dot throws UDFArgumentException for 0 rows input
04fa754 is described below
commit 04fa7545b4e4b540a6a8c610c36450a8e2320ba7
Author: Makoto Yui <my...@apache.org>
AuthorDate: Thu Jun 4 14:21:05 2020 +0900
[HIVEMALL-295][BUGFIX] transpose_and_dot throws UDFArgumentException for 0 rows input
## What changes were proposed in this pull request?
transpose_and_dot throws UDFArgumentException for 0 rows input.
```
WITH INPUT AS(
SELECT
ARRAY(1.0,2.0,3.0) AS X,
ARRAY(0,1) AS Y
)
SELECT
transpose_and_dot(Y,X) AS observed
FROM
INPUT
WHERE false
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.exec.UDFArgumentException
at org.apache.hadoop.hive.ql.exec.GroupByOperator.closeOp(GroupByOperator.java:1126)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:697)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:711)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:711)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:711)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:711)
at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.close(MapRecordProcessor.java:464)
... 15 more
Caused by: org.apache.hadoop.hive.ql.exec.UDFArgumentException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at hivemall.utils.lang.Preconditions.checkNotNull(Preconditions.java:43)
at hivemall.tools.matrix.TransposeAndDotUDAF$TransposeAndDotUDAFEvaluator.iterate(TransposeAndDotUDAF.java:172)
at org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.aggregate(GenericUDAFEvaluator.java:192)
at org.apache.hadoop.hive.ql.exec.GroupByOperator.closeOp(GroupByOperator.java:1117)
... 21 more
```
## What type of PR is it?
Bug Fix
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-295
## How was this patch tested?
manual tests on EMR
## Checklist
(Please remove this section if not needed; check `x` for YES, blank for NO)
- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`, for your commit?
- [x] Did you run system tests on Hive (or Spark)?
Author: Makoto Yui <my...@apache.org>
Closes #229 from myui/HIVEMALL-295.
---
.../java/hivemall/knn/similarity/AngularSimilarityUDF.java | 2 +-
.../main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java | 12 ++++++++----
.../hivemall/utils/collections/BoundedPriorityQueue.java | 2 +-
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/hivemall/knn/similarity/AngularSimilarityUDF.java b/core/src/main/java/hivemall/knn/similarity/AngularSimilarityUDF.java
index e381d36..b2284bd 100644
--- a/core/src/main/java/hivemall/knn/similarity/AngularSimilarityUDF.java
+++ b/core/src/main/java/hivemall/knn/similarity/AngularSimilarityUDF.java
@@ -74,7 +74,7 @@ public final class AngularSimilarityUDF extends GenericUDF {
@Override
public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException {
- if (argOIs.length != 2) {
+ if (argOIs.length != 2) {
throw new UDFArgumentException("angular_similarity takes 2 arguments");
}
this.arg0ListOI = HiveUtils.asListOI(argOIs, 0);
diff --git a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
index 8366c83..b6978fb 100644
--- a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
+++ b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java
@@ -20,7 +20,6 @@ package hivemall.tools.matrix;
import hivemall.utils.hadoop.HiveUtils;
import hivemall.utils.hadoop.WritableUtils;
-import hivemall.utils.lang.Preconditions;
import hivemall.utils.lang.SizeOf;
import java.util.ArrayList;
@@ -28,7 +27,6 @@ import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -169,8 +167,10 @@ public final class TransposeAndDotUDAF extends AbstractGenericUDAFResolver {
final Object matrix0RowObj = parameters[0];
final Object matrix1RowObj = parameters[1];
- Preconditions.checkNotNull(matrix0RowObj, UDFArgumentException.class);
- Preconditions.checkNotNull(matrix1RowObj, UDFArgumentException.class);
+ // need to care about NULL since NULL is passed when aggregating with zero row.
+ if (matrix0RowObj == null || matrix1RowObj == null) {
+ return;
+ }
final TransposeAndDotAggregationBuffer myAgg = (TransposeAndDotAggregationBuffer) agg;
@@ -231,6 +231,10 @@ public final class TransposeAndDotUDAF extends AbstractGenericUDAFResolver {
throws HiveException {
final TransposeAndDotAggregationBuffer myAgg = (TransposeAndDotAggregationBuffer) agg;
+ if (myAgg.aggMatrix == null) {
+ return null;
+ }
+
final List<List<DoubleWritable>> result = new ArrayList<List<DoubleWritable>>();
for (double[] row : myAgg.aggMatrix) {
result.add(WritableUtils.toWritableList(row));
diff --git a/core/src/main/java/hivemall/utils/collections/BoundedPriorityQueue.java b/core/src/main/java/hivemall/utils/collections/BoundedPriorityQueue.java
index 34fb0e4..9fc7754 100644
--- a/core/src/main/java/hivemall/utils/collections/BoundedPriorityQueue.java
+++ b/core/src/main/java/hivemall/utils/collections/BoundedPriorityQueue.java
@@ -45,7 +45,7 @@ public final class BoundedPriorityQueue<E> {
this.comparator = comparator;
this.queue = new PriorityQueue<E>(size + 10, comparator);
}
-
+
public boolean contains(@Nonnull E e) {
return queue.contains(e);
}