You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by iw...@apache.org on 2023/08/03 03:46:40 UTC
[bigtop] branch master updated: BIGTOP-3964: Add isal installation in bigtop toolchain and use isal library when compiling hadoop (#1142)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 9fca8d848 BIGTOP-3964: Add isal installation in bigtop toolchain and use isal library when compiling hadoop (#1142)
9fca8d848 is described below
commit 9fca8d8480593af6b78e7719c0e7e68728f8ccef
Author: lvkaihua <21...@qq.com>
AuthorDate: Thu Aug 3 11:46:35 2023 +0800
BIGTOP-3964: Add isal installation in bigtop toolchain and use isal library when compiling hadoop (#1142)
---
.../src/common/hadoop/do-component-build | 9 ++++--
.../src/common/hadoop/install_hadoop.sh | 2 +-
bigtop_toolchain/manifests/installer.pp | 1 +
bigtop_toolchain/manifests/isal.pp | 32 ++++++++++++++++++++++
bigtop_toolchain/manifests/packages.pp | 16 ++++++++---
5 files changed, 53 insertions(+), 7 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build
index 6616e9dee..f0d2bd081 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -119,7 +119,12 @@ if [ -z "$BUNDLE_SNAPPY" ] ; then
[ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
[ -f /usr/lib/${HOSTTYPE}-linux-gnu/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib/${HOSTTYPE}-linux-gnu"
fi
-
+if [ -z "$BUNDLE_ISAL" ] ; then
+ [ -f /usr/lib/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib"
+ [ -f /usr/lib64/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib64"
+ [ -f /usr/local/lib/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/local/lib/"
+ [ -f /usr/lib/${HOSTTYPE}-linux-gnu/libisal.so ] && BUNDLE_ISAL="-Dbundle.isal=true -Disal.lib=/usr/lib/${HOSTTYPE}-linux-gnu"
+fi
mkdir build
mkdir build/src
@@ -133,7 +138,7 @@ MAVEN_OPTS+="-DskipTests -DskipITs "
. $(dirname ${0})/maven_deploy.sh
# Build artifacts
-mvn $BUNDLE_SNAPPY -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@"
+mvn $BUNDLE_SNAPPY $BUNDLE_ISAL -Pdist -Pnative -Psrc -Pyarn-ui -Dtar ${MAVEN_OPTS} install ${EXTRA_GOALS} "$@"
mvn site site:stage ${MAVEN_OPTS} $@
(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index cabfa01dc..843b6768d 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -275,7 +275,7 @@ cp ${BUILD_DIR}/include/hdfs.h $PREFIX/$SYSTEM_INCLUDE_DIR/
cp -r ${BUILD_DIR}/include/hdfspp $PREFIX/$SYSTEM_INCLUDE_DIR/
cp ${BUILD_DIR}/lib/native/*.a $PREFIX/$HADOOP_NATIVE_LIB_DIR/
-for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do
+for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null; ls libisal.so.2.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do
cp ${BUILD_DIR}/lib/native/${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/
ldconfig -vlN $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library}
ln -s ${library} $PREFIX/$HADOOP_NATIVE_LIB_DIR/${library/.so.*/}.so
diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp
index 5c037ab8f..9d1a641b7 100644
--- a/bigtop_toolchain/manifests/installer.pp
+++ b/bigtop_toolchain/manifests/installer.pp
@@ -25,6 +25,7 @@ class bigtop_toolchain::installer {
include bigtop_toolchain::user
include bigtop_toolchain::renv
include bigtop_toolchain::grpc
+ include bigtop_toolchain::isal
Class['bigtop_toolchain::jdk11']->Class['bigtop_toolchain::jdk']
case $::operatingsystem {
diff --git a/bigtop_toolchain/manifests/isal.pp b/bigtop_toolchain/manifests/isal.pp
new file mode 100644
index 000000000..583222f3a
--- /dev/null
+++ b/bigtop_toolchain/manifests/isal.pp
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+class bigtop_toolchain::isal {
+ require bigtop_toolchain::packages
+ $url = "https://github.com/intel/isa-l/archive/refs/tags/v2.29.0.tar.gz"
+ $isal = "v2.29.0.tar.gz"
+ $isaldir = "isa-l-2.29.0"
+ exec { "download isal":
+ cwd => "/usr/src",
+ command => "/usr/bin/wget $url && mkdir -p $isaldir && /bin/tar -xvzf $isal -C $isaldir --strip-components=1",
+ creates => "/usr/src/$isaldir",
+ }
+ exec { "install isal":
+ cwd => "/usr/src/$isaldir",
+ command => "/usr/src/$isaldir/autogen.sh && /usr/src/$isaldir/configure --prefix=/usr/local && /usr/bin/make && /usr/bin/make install",
+ creates => "/usr/local/bin/$isaldir",
+ require => EXEC["download isal"],
+ timeout => 3000
+ }
+}
diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp
index 7c558ee2a..6d703f9d9 100644
--- a/bigtop_toolchain/manifests/packages.pp
+++ b/bigtop_toolchain/manifests/packages.pp
@@ -63,7 +63,9 @@ class bigtop_toolchain::packages {
"libffi-devel",
"krb5-devel",
"net-tools",
- "perl-Digest-SHA"
+ "perl-Digest-SHA",
+ "nasm",
+ "yasm"
]
if ($operatingsystem == 'Fedora' or $operatingsystemmajrelease !~ /^[0-7]$/) {
@@ -118,7 +120,9 @@ class bigtop_toolchain::packages {
"libffi48-devel",
"texlive-latex-bin-bin",
"libapr1",
- "libapr1-devel"
+ "libapr1-devel",
+ "nasm",
+ "yasm"
]
# fix package dependencies: BIGTOP-2120 and BIGTOP-2152 and BIGTOP-2471
exec { '/usr/bin/zypper -n install --force-resolution krb5 libopenssl-devel libxml2-devel libxslt-devel boost-devel':
@@ -149,7 +153,9 @@ class bigtop_toolchain::packages {
"snappy-devel",
"libzstd-devel",
"bzip2-devel",
- "libffi-devel"
+ "libffi-devel",
+ "nasm",
+ "yasm"
] }
/(Ubuntu|Debian)/: {
$_pkgs = [
@@ -209,7 +215,9 @@ class bigtop_toolchain::packages {
"python-setuptools",
"libffi-dev",
"python3-dev",
- "python2.7-dev"
+ "python2.7-dev",
+ "nasm",
+ "yasm"
]
if (($operatingsystem == 'Ubuntu' and 0 <= versioncmp($operatingsystemmajrelease, '22.04'))) {
file { '/usr/bin/python':