You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by st...@apache.org on 2018/03/10 02:55:45 UTC
hive git commit: HIVE-18436: Upgrade to Spark 2.3.0 (Sahil Takiar,
reviewed by Rui Li)
Repository: hive
Updated Branches:
refs/heads/master b362de387 -> 41afa4e21
HIVE-18436: Upgrade to Spark 2.3.0 (Sahil Takiar, reviewed by Rui Li)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/41afa4e2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/41afa4e2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/41afa4e2
Branch: refs/heads/master
Commit: 41afa4e2185b45b0bb6816baaea330d9ed40d127
Parents: b362de3
Author: Sahil Takiar <ta...@gmail.com>
Authored: Fri Mar 9 18:54:42 2018 -0800
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Fri Mar 9 18:54:42 2018 -0800
----------------------------------------------------------------------
itests/pom.xml | 4 ++--
pom.xml | 4 ++--
.../apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java | 2 +-
.../org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java | 3 ++-
.../test/java/org/apache/hive/spark/client/rpc/TestRpc.java | 6 ++++--
5 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index a782cd2..40bb054 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -99,10 +99,10 @@
cd -
fi
tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
- mv $BASE_DIR/spark-${spark.version}-bin-hadoop2-without-hive $BASE_DIR/$finalName
+ mv $BASE_DIR/spark-${spark.version}-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName
}
mkdir -p $DOWNLOAD_DIR
- download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
+ download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop3-beta1-without-hive.tgz" "spark"
cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/
</echo>
</target>
http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a242fbf..5be30f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -185,7 +185,7 @@
<orc.version>1.4.3</orc.version>
<mockito-all.version>1.10.19</mockito-all.version>
<mina.version>2.0.0-M5</mina.version>
- <netty.version>4.0.52.Final</netty.version>
+ <netty.version>4.1.17.Final</netty.version>
<netty3.version>3.10.5.Final</netty3.version>
<parquet.version>1.9.0</parquet.version>
<pig.version>0.16.0</pig.version>
@@ -198,7 +198,7 @@
<tez.version>0.9.1</tez.version>
<slider.version>0.92.0-incubating</slider.version>
<super-csv.version>2.2.0</super-csv.version>
- <spark.version>2.2.0</spark.version>
+ <spark.version>2.3.0</spark.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.8</scala.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
index f43b449..dff3b0f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java
@@ -104,7 +104,7 @@ public class LocalHiveSparkClient implements HiveSparkClient {
sc.addJar(regJar);
}
jobMetricsListener = new JobMetricsListener();
- sc.sc().listenerBus().addListener(jobMetricsListener);
+ sc.sc().addSparkListener(jobMetricsListener);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
----------------------------------------------------------------------
diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
index 24858d7..fb73647 100644
--- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
+++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java
@@ -72,7 +72,8 @@ public class TestKryoMessageCodec {
c.writeAndFlush(MESSAGE);
assertEquals(1, c.outboundMessages().size());
assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass()));
- c.writeInbound(c.readOutbound());
+ Object readOutboundResult = c.readOutbound();
+ c.writeInbound(readOutboundResult);
assertEquals(1, c.inboundMessages().size());
assertEquals(MESSAGE, c.readInbound());
c.close();
http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
----------------------------------------------------------------------
diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
index 21e4595..fafdff7 100644
--- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
+++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java
@@ -323,7 +323,8 @@ public class TestRpc {
int count = 0;
while (!client.outboundMessages().isEmpty()) {
- server.writeInbound(client.readOutbound());
+ Object readOutboundResult = client.readOutbound();
+ server.writeInbound(readOutboundResult);
count++;
}
server.flush();
@@ -331,7 +332,8 @@ public class TestRpc {
count = 0;
while (!server.outboundMessages().isEmpty()) {
- client.writeInbound(server.readOutbound());
+ Object readOutboundResult = server.readOutbound();
+ client.writeInbound(readOutboundResult);
count++;
}
client.flush();