(kylin) branch kylin5 updated: KYLIN-5969 Package gluten dependencies into spark

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/kylin5 by this push:
     new 0fdc49b3ec KYLIN-5969 Package gluten dependencies into spark
0fdc49b3ec is described below

commit 0fdc49b3ecb30a0f56c476dcc0bd6aa922c03693
Author: zhong.zhu <mi...@icloud.com>
AuthorDate: Wed Sep 4 16:41:23 2024 +0800

    KYLIN-5969 Package gluten dependencies into spark
---
 build/release/compress.sh         |  4 +++-
 build/sbin/download-spark-user.sh | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/build/release/compress.sh b/build/release/compress.sh
index 3f887fafb6..f20e4deab9 100755
--- a/build/release/compress.sh
+++ b/build/release/compress.sh
@@ -93,7 +93,9 @@ rm -rf ext lib commit_SHA1 VERSION # keep the spark folder on purpose
 cp -rf server/webapp/dist ${package_name}/server/public
 cp -rf server/newten.jar ${package_name}/server/
 cp -rf server/jars ${package_name}/server/
-echo $WITH_GLUTEN
+
+gluten_version=$(sed -n 's:.*<gluten.version>\(.*\)</gluten.version>.*:\1:p'  ./../pom.xml)
+echo "$gluten_version" > ${package_name}/GLUTEN_VERSION
 if [[ "${WITH_GLUTEN}" = "1" ]]; then
     mkdir -p ${package_name}/lib/gluten/
     cp -rf gluten/jars/spark33/* ${package_name}/lib/gluten/
diff --git a/build/sbin/download-spark-user.sh b/build/sbin/download-spark-user.sh
index 6161f6780f..72017d24a2 100755
--- a/build/sbin/download-spark-user.sh
+++ b/build/sbin/download-spark-user.sh
@@ -76,9 +76,23 @@ fi
 tar -zxf ${KYLIN_HOME}/hive_1_2_2.tar.gz -C ${KYLIN_HOME}/spark/ || { exit 1; }
 
 # add gluten relevant dependencies to spark
-if [ ! -f ${KYLIN_HOME}/lib/gluten ]
+if [ -d ${KYLIN_HOME}/lib/gluten/ ]
 then
   find ${KYLIN_HOME}/spark/jars/ -name "protobuf-java*" -delete
   cp -rf ${KYLIN_HOME}/server/jars/*.jar ${KYLIN_HOME}/spark/jars/
   cp -rf ${KYLIN_HOME}/server/libch.so ${KYLIN_HOME}/spark/
+else
+  gluten_version=$(cat ${KYLIN_HOME}/GLUTEN_VERSION)
+  gluten_platform="ubuntu22.04-x86_64"
+  if [ ! -f ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}.tar.gz ]
+  then
+    wget --directory-prefix=${KYLIN_HOME} https://repository.kyligence.io/repository/open-raw/org/apache/gluten/${gluten_version}-${gluten_platform}/gluten-${gluten_version}-${gluten_platform}.tar.gz  || echo "Download gluten failed"
+  fi
+  tar -zxf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}.tar.gz -C ${KYLIN_HOME}/ || { exit 1; }
+  cp -rf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}/libs/libch.so ${KYLIN_HOME}/server/
+  cp -rf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}/jars/spark33/gluten.jar ${KYLIN_HOME}/lib/ext/
+  find ${KYLIN_HOME}/spark/jars/ -name "protobuf-java*" -delete
+  cp -rf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}/libs/libch.so ${KYLIN_HOME}/spark/
+  cp -rf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}/jars/spark33/* ${KYLIN_HOME}/spark/jars/
+  rm -rf ${KYLIN_HOME}/gluten-${gluten_version}-${gluten_platform}/
 fi
\ No newline at end of file