You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2020/04/27 23:42:53 UTC
[bigtop] branch master updated: BIGTOP-3333. Fix Hive build after
upgrading Spark to 2.4.5 (#622)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 5795aea BIGTOP-3333. Fix Hive build after upgrading Spark to 2.4.5 (#622)
5795aea is described below
commit 5795aea07fd3fd74e2865b60445ce18d39fa3da8
Author: Kengo Seki <se...@apache.org>
AuthorDate: Tue Apr 28 08:42:43 2020 +0900
BIGTOP-3333. Fix Hive build after upgrading Spark to 2.4.5 (#622)
---
.../src/common/hive/patch2-HIVE-18436.diff | 73 ++++++++++++++++++++++
1 file changed, 73 insertions(+)
diff --git a/bigtop-packages/src/common/hive/patch2-HIVE-18436.diff b/bigtop-packages/src/common/hive/patch2-HIVE-18436.diff
new file mode 100644
index 0000000..442629d
--- /dev/null
+++ b/bigtop-packages/src/common/hive/patch2-HIVE-18436.diff
@@ -0,0 +1,73 @@
+diff --git a/pom.xml b/pom.xml
+index f91f7f43a6..71a889746a 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -178,7 +178,7 @@
+ <orc.version>1.3.4</orc.version>
+ <mockito-all.version>1.9.5</mockito-all.version>
+ <mina.version>2.0.0-M5</mina.version>
+- <netty.version>4.0.52.Final</netty.version>
++ <netty.version>4.1.17.Final</netty.version>
+ <parquet.version>1.8.1</parquet.version>
+ <pig.version>0.16.0</pig.version>
+ <protobuf.version>2.5.0</protobuf.version>
+@@ -189,7 +189,7 @@
+ <tez.version>0.8.4</tez.version>
+ <slider.version>0.90.2-incubating</slider.version>
+ <super-csv.version>2.2.0</super-csv.version>
+- <spark.version>2.0.0</spark.version>
++ <spark.version>2.3.0</spark.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.8</scala.version>
+ <tempus-fugit.version>1.1</tempus-fugit.version>
+diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+index beeafd0672..c871ba6c33 100644
+--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
++++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+@@ -84,7 +84,7 @@ public static synchronized LocalHiveSparkClient getInstance(SparkConf sparkConf)
+ private LocalHiveSparkClient(SparkConf sparkConf) {
+ sc = new JavaSparkContext(sparkConf);
+ jobMetricsListener = new JobMetricsListener();
+- sc.sc().listenerBus().addListener(jobMetricsListener);
++ sc.sc().addSparkListener(jobMetricsListener);
+ }
+
+ @Override
+diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
+index 24858d7cef..fb736471b2 100644
+--- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
++++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
+@@ -72,7 +72,8 @@ public void testEmbeddedChannel() throws Exception {
+ c.writeAndFlush(MESSAGE);
+ assertEquals(1, c.outboundMessages().size());
+ assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass()));
+- c.writeInbound(c.readOutbound());
++ Object readOutboundResult = c.readOutbound();
++ c.writeInbound(readOutboundResult);
+ assertEquals(1, c.inboundMessages().size());
+ assertEquals(MESSAGE, c.readInbound());
+ c.close();
+diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
+index 5a4801c5fa..21b3d4e494 100644
+--- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
++++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
+@@ -287,7 +287,8 @@ private void transfer(Rpc serverRpc, Rpc clientRpc) {
+
+ int count = 0;
+ while (!client.outboundMessages().isEmpty()) {
+- server.writeInbound(client.readOutbound());
++ Object readOutboundResult = client.readOutbound();
++ server.writeInbound(readOutboundResult);
+ count++;
+ }
+ server.flush();
+@@ -295,7 +296,8 @@ private void transfer(Rpc serverRpc, Rpc clientRpc) {
+
+ count = 0;
+ while (!server.outboundMessages().isEmpty()) {
+- client.writeInbound(server.readOutbound());
++ Object readOutboundResult = server.readOutbound();
++ client.writeInbound(readOutboundResult);
+ count++;
+ }
+ client.flush();