You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/10/10 11:00:31 UTC
[kylin] branch master updated: KYLIN-3272 Upgrade Spark to 2.3.2
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new d2a2864 KYLIN-3272 Upgrade Spark to 2.3.2
d2a2864 is described below
commit d2a2864968f451938881d55073518851d60a6bd1
Author: Lijun Cao <64...@qq.com>
AuthorDate: Wed Oct 10 10:12:14 2018 +0800
KYLIN-3272 Upgrade Spark to 2.3.2
---
build/script/download-spark.sh | 4 ++--
.../main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java | 3 +++
pom.xml | 2 +-
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
index ed904b5..1b61145 100755
--- a/build/script/download-spark.sh
+++ b/build/script/download-spark.sh
@@ -27,8 +27,8 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-spark_version="2.1.2"
-spark_pkg_md5="b47cf199bf78042a0471b9a5ae0ceeac"
+spark_version="2.3.2"
+spark_pkg_md5="01b3e6aef0ef574e066d15e035b9b2dd"
if [ ! -f "build/spark-${spark_version}-bin-hadoop2.7.tgz" ]
then
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index a13f96a..ed623d7 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -111,6 +111,9 @@ public class KylinKryoRegistrator implements KryoRegistrator {
addClassQuitely(kyroClasses, "org.apache.spark.sql.catalyst.expressions.GenericInternalRow");
addClassQuitely(kyroClasses, "org.apache.spark.unsafe.types.UTF8String");
+ addClassQuitely(kyroClasses, "org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage");
+ addClassQuitely(kyroClasses, "scala.collection.immutable.Set$EmptySet$");
+
for (Class kyroClass : kyroClasses) {
kryo.register(kyroClass);
}
diff --git a/pom.xml b/pom.xml
index c973d1a..264583d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,7 +60,7 @@
<kafka.version>1.0.0</kafka.version>
<!-- Spark versions -->
- <spark.version>2.1.2</spark.version>
+ <spark.version>2.3.2</spark.version>
<kryo.version>4.0.0</kryo.version>
<!-- mysql versions -->