You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jx...@apache.org on 2015/11/06 18:33:10 UTC
[28/55] [abbrv] hive git commit: HIVE-12206: ClassNotFound Exception
during query compilation with Tez and Union query and GenericUDFs (Jason Dere,
reviewed by Ashutosh Chauhan)
HIVE-12206: ClassNotFound Exception during query compilation with Tez and Union query and GenericUDFs (Jason Dere, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe6ebf77
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe6ebf77
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe6ebf77
Branch: refs/heads/master-fixed
Commit: fe6ebf77a5494b737479af3f1159e46ed6aa9d24
Parents: 7073ce3
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue Nov 3 15:38:31 2015 -0800
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue Nov 3 15:38:31 2015 -0800
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/udf/example/GenericUDFExampleAdd.java | 48 ++++++++++++++++++++
.../apache/hadoop/hive/ql/exec/Utilities.java | 2 +
.../queries/clientpositive/tez_union_with_udf.q | 13 ++++++
.../clientpositive/tez/tez_union_with_udf.q.out | 36 +++++++++++++++
5 files changed, 100 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 7416d32..2d1d274 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -424,6 +424,7 @@ minillap.query.files=bucket_map_join_tez1.q,\
tez_union_view.q,\
tez_union_decimal.q,\
tez_union_group_by.q,\
+ tez_union_with_udf.q,\
tez_smb_main.q,\
tez_smb_1.q,\
vectorized_dynamic_partition_pruning.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java
----------------------------------------------------------------------
diff --git a/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java b/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java
new file mode 100644
index 0000000..85906c2
--- /dev/null
+++ b/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java
@@ -0,0 +1,48 @@
+package org.apache.hadoop.hive.udf.example;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+
+public class GenericUDFExampleAdd extends GenericUDF {
+
+ Converter converter0;
+ Converter converter1;
+ DoubleWritable result = new DoubleWritable();
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ ObjectInspector doubleOI = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.DOUBLE);
+ converter0 = (Converter) ObjectInspectorConverters.getConverter(arguments[0], doubleOI);
+ converter1 = (Converter) ObjectInspectorConverters.getConverter(arguments[1], doubleOI);
+ return doubleOI;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ DoubleWritable dw0 = (DoubleWritable) converter0.convert(arguments[0].get());
+ DoubleWritable dw1 = (DoubleWritable) converter1.convert(arguments[0].get());
+ if (dw0 == null || dw1 == null) {
+ return null;
+ }
+ result.set(dw0.get() + dw1.get());
+ return result;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "GenericUDFExampleAdd";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 665b3f7..02adf0c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1057,6 +1057,7 @@ public final class Utilities {
*/
private static void serializeObjectByKryo(Kryo kryo, Object plan, OutputStream out) {
Output output = new Output(out);
+ kryo.setClassLoader(getSessionSpecifiedClassLoader());
kryo.writeObject(output, plan);
output.close();
}
@@ -1080,6 +1081,7 @@ public final class Utilities {
private static <T> T deserializeObjectByKryo(Kryo kryo, InputStream in, Class<T> clazz ) {
Input inp = new Input(in);
+ kryo.setClassLoader(getSessionSpecifiedClassLoader());
T t = kryo.readObject(inp,clazz);
inp.close();
return t;
http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/test/queries/clientpositive/tez_union_with_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/tez_union_with_udf.q b/ql/src/test/queries/clientpositive/tez_union_with_udf.q
new file mode 100644
index 0000000..6826530
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_union_with_udf.q
@@ -0,0 +1,13 @@
+select * from (select key + key from src limit 1) a
+union all
+select * from (select key + key from src limit 1) b;
+
+
+add jar ${system:maven.local.repository}/org/apache/hive/hive-it-test-serde/${system:hive.version}/hive-it-test-serde-${system:hive.version}.jar;
+
+create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd';
+
+-- Now try the query with the UDF
+select example_add(key, key)from (select key from src limit 1) a
+union all
+select example_add(key, key)from (select key from src limit 1) b;
http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out b/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out
new file mode 100644
index 0000000..923e098
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out
@@ -0,0 +1,36 @@
+PREHOOK: query: select * from (select key + key from src limit 1) a
+union all
+select * from (select key + key from src limit 1) b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select key + key from src limit 1) a
+union all
+select * from (select key + key from src limit 1) b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+476.0
+476.0
+PREHOOK: query: create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: example_add
+POSTHOOK: query: create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: example_add
+PREHOOK: query: -- Now try the query with the UDF
+select example_add(key, key)from (select key from src limit 1) a
+union all
+select example_add(key, key)from (select key from src limit 1) b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- Now try the query with the UDF
+select example_add(key, key)from (select key from src limit 1) a
+union all
+select example_add(key, key)from (select key from src limit 1) b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+476.0
+476.0