[kylin] branch master updated: KYLIN-3571 Not build Spark in Kylin's binary package

[sh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new d240e0d  KYLIN-3571 Not build Spark in Kylin's binary package
d240e0d is described below

commit d240e0d3ea939483d0d07a638c47aae610958c1f
Author: chao long <wa...@qq.com>
AuthorDate: Fri Mar 8 15:55:05 2019 +0800

    KYLIN-3571 Not build Spark in Kylin's binary package
---
 build/bin/download-spark.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++
 build/script/compress.sh    |  6 ++--
 build/script/package.sh     |  1 -
 3 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/build/bin/download-spark.sh b/build/bin/download-spark.sh
new file mode 100755
index 0000000..4ce5f8e
--- /dev/null
+++ b/build/bin/download-spark.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
+if [ -d "${KYLIN_HOME}/spark" ]
+then
+    echo "Spark binary exists"
+    exit 0;
+else
+    echo "Downloading spark package..."
+fi
+
+spark_package_dir=/tmp/spark_package
+
+mkdir -p -- "${spark_package_dir}" && cd -P -- "${spark_package_dir}"
+
+alias md5cmd="md5sum"
+if [[ `uname -a` =~ "Darwin" ]]; then
+    alias md5cmd="md5 -q"
+fi
+
+spark_version="2.3.2"
+spark_pkg_md5="01b3e6aef0ef574e066d15e035b9b2dd"
+
+if [ ! -f "spark-${spark_version}-bin-hadoop2.7.tgz" ]
+then
+    echo "No binary file found, start to download package to ${spark_package_dir}"
+    wget http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz || echo "Download spark failed"
+else
+    if [ `md5cmd spark-${spark_version}-bin-hadoop2.7.tgz | awk '{print $1}'` != "${spark_pkg_md5}" ]
+    then
+        echo "md5 check failed"
+        rm spark-${spark_version}-bin-hadoop2.7.tgz
+        wget http://archive.apache.org/dist/spark/spark-${spark_version}/spark-${spark_version}-bin-hadoop2.7.tgz || echo "Download spark failed"
+    else
+        echo "Spark package found in ${spark_package_dir}"
+    fi
+fi
+unalias md5cmd
+
+echo "Start to decompress package"
+tar -zxvf spark-${spark_version}-bin-hadoop2.7.tgz  || { exit 1; }
+mv spark-${spark_version}-bin-hadoop2.7 spark
+
+# Remove unused components in Spark
+rm -rf spark/lib/spark-examples-*
+rm -rf spark/examples
+rm -rf spark/data
+rm -rf spark/R
+
+# mv spark binary to KYLIN_HOME
+mv spark ${KYLIN_HOME}
+
+echo "Download spark binary done"
+
+rm -rf ${spark_package_dir}
diff --git a/build/script/compress.sh b/build/script/compress.sh
index cc94f47..666dcec 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -34,8 +34,8 @@ package_name=apache-kylin-${version}-bin
 cd build/
 rm -rf ${package_name}
 mkdir ${package_name}
-cp -r ext lib tool bin conf tomcat spark ../examples/sample_cube commit_SHA1 ${package_name}
-rm -rf ext lib tomcat spark commit_SHA1
+cp -r ext lib tool bin conf tomcat ../examples/sample_cube commit_SHA1 ${package_name}
+rm -rf ext lib tomcat commit_SHA1
 
 ## comment all default properties, and append them to the user visible kylin.properties
 ## first 16 lines are license, just skip them
@@ -44,7 +44,7 @@ sed '1,16d' ../core-common/src/main/resources/kylin-defaults.properties | awk '{
 find ${package_name} -type d -exec chmod 755 {} \;
 find ${package_name} -type f -exec chmod 644 {} \;
 find ${package_name} -type f -name "*.sh" -exec chmod 755 {} \;
-find ${package_name}/spark/bin/ -type f -exec chmod +x {} \;
+
 mkdir -p ../dist
 tar -cvzf ../dist/${package_name}.tar.gz ${package_name}
 rm -rf ${package_name}
diff --git a/build/script/package.sh b/build/script/package.sh
index dedd10d..70e6a60 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -76,6 +76,5 @@ git rev-parse HEAD >> build/commit_SHA1
 
 sh build/script/build.sh $@         || { exit 1; }
 sh build/script/download-tomcat.sh  || { exit 1; }
-sh build/script/download-spark.sh   || { exit 1; }
 sh build/script/prepare.sh          || { exit 1; }
 sh build/script/compress.sh         || { exit 1; }