You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2022/04/30 11:25:49 UTC
[bigtop] branch master updated: BIGTOP-3677. Building HBase for CentOS 7 on arm64 and ppc64le fails due to missing libraries. (#896)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 8ab99718 BIGTOP-3677. Building HBase for CentOS 7 on arm64 and ppc64le fails due to missing libraries. (#896)
8ab99718 is described below
commit 8ab9971866e40f35cac19bd9c7a3aad76127bbe8
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Sat Apr 30 20:25:43 2022 +0900
BIGTOP-3677. Building HBase for CentOS 7 on arm64 and ppc64le fails due to missing libraries. (#896)
* BIGTOP-3677. Building HBase for CentOS 7 on arm64 fails due to missing libraries.
* needs the fix for ppc64le too.
* bumped the version of protobuf used for HBase on ppc64le.
---
.../src/common/hbase/do-component-build | 22 ++++----
.../protoc-3.5.1-add-support-for-ppc64le.patch | 65 ----------------------
bigtop_toolchain/manifests/protobuf.pp | 21 +++----
3 files changed, 20 insertions(+), 88 deletions(-)
diff --git a/bigtop-packages/src/common/hbase/do-component-build b/bigtop-packages/src/common/hbase/do-component-build
index 89c56e8b..be6d8ef2 100644
--- a/bigtop-packages/src/common/hbase/do-component-build
+++ b/bigtop-packages/src/common/hbase/do-component-build
@@ -20,6 +20,14 @@ set -ex
export MAVEN_OPTS="-Xmx1024m"
+MVN_ARGS="-Phadoop-3.0 "
+MVN_ARGS+="-Dhadoop-three.version=${HADOOP_VERSION} "
+MVN_ARGS+="-Dhadoop.guava.version=27.0-jre "
+MVN_ARGS+="-Djetty.version=9.3.29.v20201019 "
+MVN_ARGS+="-Dzookeeper.version=${ZOOKEEPER_VERSION} "
+MVN_ARGS+="-DskipTests "
+MVN_ARGS+="-Dcheckstyle.skip=true "
+
if [ "${BIGTOP_JDK:=8}" == "8" ]; then
MAVEN_ADDITIONAL="-Dadditionalparam=-Xdoclint:none"
fi
@@ -30,8 +38,9 @@ if [ $HOSTTYPE = "powerpc64le" ] ; then
sed -i 's|<jruby.version>.*</jruby.version>|<jruby.version>1.7.23</jruby.version>|' pom.xml
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
-Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc
- mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.5.1-1 \
- -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.5.1.1/bin/protoc
+ mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.17.3 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe -Dfile=/usr/local/protobuf-3.17.3/bin/protoc
+ MVN_ARGS+="-Dexternal.protobuf.groupid=com.google.protobuf -Dexternal.protoc.version=2.5.0 "
fi
if [ $HOSTTYPE = "aarch64" ] ; then
@@ -43,16 +52,9 @@ if [ $HOSTTYPE = "aarch64" ] ; then
'\ <\/dependency>' pom.xml
mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=2.5.0 \
-Dclassifier=linux-aarch_64 -Dpackaging=exe -Dfile=/usr/local/bin/protoc
+ MVN_ARGS+="-Dexternal.protobuf.groupid=com.google.protobuf -Dexternal.protoc.version=2.5.0 "
fi
-MVN_ARGS="-Phadoop-3.0 "
-MVN_ARGS+="-Dhadoop-three.version=${HADOOP_VERSION} "
-MVN_ARGS+="-Dhadoop.guava.version=27.0-jre "
-MVN_ARGS+="-Djetty.version=9.3.29.v20201019 "
-MVN_ARGS+="-Dzookeeper.version=${ZOOKEEPER_VERSION} "
-MVN_ARGS+="-DskipTests "
-MVN_ARGS+="-Dcheckstyle.skip=true "
-
# HBASE-21513: separate site and assembly:single to avoid assembly issues.
mvn ${MVN_ARGS} ${MAVEN_ADDITIONAL} clean install "$@"
mvn ${MVN_ARGS} ${MAVEN_ADDITIONAL} site "$@"
diff --git a/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch b/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch
deleted file mode 100644
index 581ac4ec..00000000
--- a/bigtop_toolchain/files/protoc-3.5.1-add-support-for-ppc64le.patch
+++ /dev/null
@@ -1,65 +0,0 @@
-diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh
-index fe1dec2..f31597a 100755
---- a/protoc-artifacts/build-protoc.sh
-+++ b/protoc-artifacts/build-protoc.sh
-@@ -81,6 +81,8 @@ checkArch ()
- assertEq $format "elf64-x86-64" $LINENO
- elif [[ "$ARCH" == aarch_64 ]]; then
- assertEq $format "elf64-little" $LINENO
-+ elif [[ "$ARCH" == ppcle_64 ]]; then
-+ assertEq $format "elf64-powerpcle" $LINENO
- else
- fail "Unsupported arch: $ARCH"
- fi
-@@ -127,6 +129,8 @@ checkDependencies ()
- elif [[ "$ARCH" == aarch_64 ]]; then
- dump_cmd='objdump -p '"$1"' | grep NEEDED'
- white_list="libpthread\.so\.0\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
-+ elif [[ "$ARCH" == ppcle_64 ]]; then
-+ white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2"
- fi
- elif [[ "$OS" == osx ]]; then
- dump_cmd='otool -L '"$1"' | fgrep dylib'
-@@ -193,6 +197,8 @@ elif [[ "$(uname)" == Linux* ]]; then
- CXXFLAGS="$CXXFLAGS -m32"
- elif [[ "$ARCH" == aarch_64 ]]; then
- CONFIGURE_ARGS="$CONFIGURE_ARGS --host=aarch64-linux-gnu"
-+ elif [[ "$ARCH" == ppcle_64 ]]; then
-+ CXXFLAGS="$CXXFLAGS -m64"
- else
- fail "Unsupported arch: $ARCH"
- fi
-diff --git a/protoc-artifacts/build-zip.sh b/protoc-artifacts/build-zip.sh
-index f08e275..e87bb39 100755
---- a/protoc-artifacts/build-zip.sh
-+++ b/protoc-artifacts/build-zip.sh
-@@ -20,6 +20,7 @@ included. Each invocation will create 6 zip packages:
- dist/<TARGET>-<VERSION_NUMBER>-linux-x86_32.zip
- dist/<TARGET>-<VERSION_NUMBER>-linux-x86_64.zip
- dist/<TARGET>-<VERSION_NUMBER>-linux-aarch_64.zip
-+ dist/<TARGET>-<VERSION_NUMBER>-linux-ppcle_64.zip
- EOF
- exit 1
- fi
-@@ -35,6 +36,8 @@ declare -a FILE_NAMES=( \
- linux-x86_32.zip linux-x86_32.exe \
- linux-x86_64.zip linux-x86_64.exe \
- linux-aarch_64.zip linux-aarch_64.exe \
-+ linux-ppcle_64.zip linux-ppcle_64.exe \
-+
- )
-
- # List of all well-known types to be included.
-diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml
-index 0f9dd9f..4a0892e 100644
---- a/protoc-artifacts/pom.xml
-+++ b/protoc-artifacts/pom.xml
-@@ -37,7 +37,7 @@
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
-- <version>1.2.3.Final</version>
-+ <version>1.5.0.Final</version>
- </extension>
- </extensions>
- <plugins>
diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp
index 47ea5a5b..db402eb8 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -54,22 +54,17 @@ class bigtop_toolchain::protobuf {
}
if ($architecture == 'ppc64le') {
- exec { "download protobuf 3.5.1.1":
+ exec { "download protobuf 3.17.3":
cwd => "/usr/src",
- command => "/usr/bin/wget https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.5.1.1.tar.gz && mkdir -p protobuf-3.5.1.1 && /bin/tar -xvzf v3.5.1.1.tar.gz -C protobuf-3.5.1.1 --strip-components=1",
- creates => "/usr/src/protobuf-3.5.1.1",
+ command => "/usr/bin/wget https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.17.3.tar.gz && mkdir -p protobuf-3.17.3 && /bin/tar -xvzf v3.17.3.tar.gz -C protobuf-3.17.3 --strip-components=1",
+ creates => "/usr/src/protobuf-3.17.3",
}
- file { "/usr/src/protobuf-3.5.1.1/protoc-3.5.1-add-support-for-ppc64le.patch":
- source => "puppet:///modules/bigtop_toolchain/protoc-3.5.1-add-support-for-ppc64le.patch",
- require => Exec["download protobuf 3.5.1.1"],
- }
-
- exec { "install protobuf 3.5.1.1":
- cwd => "/usr/src/protobuf-3.5.1.1",
- command => "/usr/bin/patch -p1 < protoc-3.5.1-add-support-for-ppc64le.patch && /usr/src/protobuf-3.5.1.1/autogen.sh && /usr/src/protobuf-3.5.1.1/configure --prefix=/usr/local/protobuf-3.5.1.1 --disable-shared --with-pic && /usr/bin/make install",
- creates => "/usr/local/protobuf-3.5.1.1",
- require => File["/usr/src/protobuf-3.5.1.1/protoc-3.5.1-add-support-for-ppc64le.patch"],
+ exec { "install protobuf 3.17.3":
+ cwd => "/usr/src/protobuf-3.17.3",
+ command => "/usr/src/protobuf-3.17.3/autogen.sh && /usr/src/protobuf-3.17.3/configure --prefix=/usr/local/protobuf-3.17.3 --disable-shared --with-pic && /usr/bin/make install",
+ creates => "/usr/local/protobuf-3.17.3",
+ require => Exec["download protobuf 3.17.3"],
timeout => 3000
}
}