(impala) branch master updated: IMPALA-11157: Use native-toolchain hadoop for aarch64

[mi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


The following commit(s) were added to refs/heads/master by this push:
     new 1d404838a IMPALA-11157: Use native-toolchain hadoop for aarch64
1d404838a is described below

commit 1d404838a07743cdf806c0729c04ebd153634ece
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Wed Nov 22 23:35:51 2023 +0000

    IMPALA-11157: Use native-toolchain hadoop for aarch64
    
    Uses hadoop binaries from native-toolchain for aarch64 rather than
    pre-built binaries from an individual's GitHub repo.
    
    Change-Id: Ide5ad327d6ce7c2a6b7d0ec4cf1dd53fef987720
    Reviewed-on: http://gerrit.cloudera.org:8080/20737
    Reviewed-by: Laszlo Gaal <la...@cloudera.com>
    Tested-by: Michael Smith <mi...@cloudera.com>
---
 bin/bootstrap_toolchain.py |  3 +++
 bin/impala-config.sh       | 14 +++++++++-----
 buildall.sh                |  9 +++------
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 48f0caece..211036001 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -485,6 +485,9 @@ def get_toolchain_downloads():
   protobuf_package_clang = ToolchainPackage(
       "protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION"))
   toolchain_packages += [protobuf_package_clang]
+  if platform.machine() == 'aarch64':
+    toolchain_packages.append(ToolchainPackage("hadoop",
+        explicit_version=os.environ.get("IMPALA_HADOOP_CLIENT_BINARY_VERSION")))
   # Check whether this platform is supported (or whether a valid custom toolchain
   # has been provided).
   if not try_get_platform_release_label() \
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 2bf174002..817e4977e 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -85,13 +85,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
 # moving to a different build of the toolchain, e.g. when a version is bumped or a
 # compile option is changed. The build id can be found in the output of the toolchain
 # build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=3-c22e6eb3b9
-export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=377-c22e6eb3b9
+export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=8-62067ab072
+export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=378-62067ab072
 export IMPALA_TOOLCHAIN_REPO=\
 ${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
 export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
 export IMPALA_TOOLCHAIN_COMMIT_HASH=\
-${IMPALA_TOOLCHAIN_COMMIT_HASH-c22e6eb3b95979bdd7a2ec8f86abb751b439f7a4}
+${IMPALA_TOOLCHAIN_COMMIT_HASH-62067ab072750a6395307499caf9ebaa65cb79b7}
 # Compare the build ref in build IDs by removing everything up-to-and-including the
 # first hyphen.
 if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
@@ -101,6 +101,8 @@ if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
   exit 1
 fi
 
+export ARCH_NAME=$(uname -p)
+
 # Versions of toolchain dependencies.
 # -----------------------------------
 if $USE_AVRO_CPP; then
@@ -201,6 +203,10 @@ export IMPALA_CLOUDFLAREZLIB_VERSION=9e601a3f37
 unset IMPALA_CLOUDFLAREZLIB_URL
 export IMPALA_CALLONCEHACK_VERSION=1.0.0
 unset IMPALA_CALLONCEHACK_URL
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  export IMPALA_HADOOP_CLIENT_BINARY_VERSION=3.3.6
+  unset IMPALA_HADOOP_CLIENT_BINARY_URL
+fi
 # Thrift related environment variables.
 # IMPALA_THRIFT_POM_VERSION is used to populate IMPALA_THRIFT_JAVA_VERSION and
 # thrift.version in java/pom.xml.
@@ -261,8 +267,6 @@ export APACHE_HIVE_VERSION=3.1.3
 export APACHE_HIVE_STORAGE_API_VERSION=2.7.0
 export APACHE_OZONE_VERSION=1.3.0
 
-export ARCH_NAME=$(uname -p)
-
 # Java dependencies that are not also runtime components. Declaring versions here allows
 # other branches to override them in impala-config-branch.sh for cleaner patches.
 export IMPALA_BOUNCY_CASTLE_VERSION=1.68
diff --git a/buildall.sh b/buildall.sh
index cdaf319f3..e39eaa922 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -443,13 +443,10 @@ bootstrap_dependencies() {
     "$IMPALA_HOME/bin/bootstrap_toolchain.py"
     echo "Toolchain bootstrap complete."
   fi
-  # Download prebuilt Hadoop native binaries for aarch64
+  # Use prebuilt Hadoop native binaries for aarch64
   if [[ "$(uname -p)" = "aarch64" ]]; then
-    if ! [[ -d "$IMPALA_HOME/../hadoopAarch64NativeLibs" ]]; then
-      git clone https://github.com/zhaorenhai/hadoopAarch64NativeLibs \
-        "$IMPALA_HOME/../hadoopAarch64NativeLibs"
-    fi
-    cp $IMPALA_HOME/../hadoopAarch64NativeLibs/lib*  $HADOOP_HOME/lib/native/
+    cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-$IMPALA_HADOOP_CLIENT_BINARY_VERSION/lib/* \
+        $HADOOP_HOME/lib/native/
   fi
   if [[ "${USE_APACHE_HIVE}" = true ]]; then
     "$IMPALA_HOME/testdata/bin/patch_hive.sh"