You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2016/02/05 17:48:28 UTC
[1/2] incubator-trafodion git commit: [TRAFODION-1614] Improve time
to build external dependencies
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 83983c7c3 -> 7f33541a4
[TRAFODION-1614] Improve time to build external dependencies
This script is for one-time set-up of build environment (aka TOOLSDIR),
but an hour or more is excessive. So to shrink the time, we kick off all
the builds in background. All the builds are independent, except
that LLVM depends on udis86.
LLVM is the long pole in build time. Biggest gain is building the
two flavors of LLVM in parallel. All the other tools can be
downloaded and built while we are waiting for LLVM.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/bb58c066
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/bb58c066
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/bb58c066
Branch: refs/heads/master
Commit: bb58c066fee9087dc81a06a99a0bd19043342a72
Parents: 40c697d
Author: Steve Varnau <st...@esgyn.com>
Authored: Thu Feb 4 00:17:55 2016 +0000
Committer: Steve Varnau <st...@esgyn.com>
Committed: Thu Feb 4 01:12:36 2016 +0000
----------------------------------------------------------------------
install/traf_tools_setup.sh | 206 +++++++++++++++------------------------
1 file changed, 78 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/bb58c066/install/traf_tools_setup.sh
----------------------------------------------------------------------
diff --git a/install/traf_tools_setup.sh b/install/traf_tools_setup.sh
index 00cecb2..b7bd86d 100755
--- a/install/traf_tools_setup.sh
+++ b/install/traf_tools_setup.sh
@@ -93,13 +93,38 @@ function downloadSource
fi
}
+LOGDIR=$(pwd)
+LOGFILE=$LOGDIR/traf_tools_setup.log
+BLDLOG=$LOGDIR/traf_tools_bld
+
+# -----------------------------------------------------------------------------
+# execute build functions in background
+# $1 - short name of component (single word, also used in logfile name)
+# $2 - first make target (usually empty string for the default target)
+# $3 - second make target
+# $4 - dir/file to check for success
+# -----------------------------------------------------------------------------
+function bkgBuild
+{
+ LOG=${BLDLOG}-${1}.log
+ echo "INFO: Starting background make and install for $1" | tee -a $LOGFILE
+
+ (make $2 > $LOG 2>&1
+ echo "INFO: make completed" > $LOG
+ make $3 > $LOG 2>&1
+ if [[ ! -e $4 ]]; then
+ echo "ERROR: failed to install $1" | tee -a $LOGFILE
+ echo " see details in $LOG" | tee -a $LOGFILE
+ exit 2;
+ fi
+ echo "INFO: $1 install complete at $(date), files placed in $TOOLSDIR" | tee -a $LOGFILE
+ ) &
+}
# main code
TOOLSDIR=
BASEDIR=
-LOGDIR=`pwd`
-LOGFILE=$LOGDIR/traf_tools_setup.log
rm $LOGFILE 2>/dev/null
echo
echo "INFO: Starting tools build on $(date)" | tee -a $LOGFILE
@@ -173,58 +198,6 @@ echo " *********************************************************** " | tee -a $L
# -----------------------------------------------------------------------------
-# install mpi
-cd $BASEDIR
-echo
-echo "INFO: Installing MPI on $(date)" | tee -a $LOGFILE
-if [ -d $TOOLSDIR/dest-mpich-3.0.4/bin ]; then
- echo "INFO: MPI is already installed, skipping to next tool" | tee -a $LOGFILE
-else
- downloadSource http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz mpich-3.0.4
- cd mpich-3.0.4
- ./configure --prefix=$TOOLSDIR/dest-mpich-3.0.4 --with-device=ch3:sock --disable-f77 --disable-fc >>$LOGFILE 2>&1
- echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/dest-mpich-3.0.4/bin ]; then
- echo "ERROR: failed to install MPI" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
-fi
-echo "INFO: MPI installation complete" | tee -a $LOGFILE
-echo " *********************************************************** " | tee -a $LOGFILE
-
-
-# -----------------------------------------------------------------------------
-# install bison
-cd $BASEDIR
-echo
-echo "INFO: Installing Bison on $(date)" | tee -a $LOGFILE
-if [ -d $TOOLSDIR/bison_3_linux/bin ]; then
- echo "INFO: Bison is already installed, skipping to next tool" | tee -a $LOGFILE
-else
- downloadSource http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz bison-3.0
- cd bison-3.0
- ./configure --prefix=$TOOLSDIR/bison_3_linux >>$LOGFILE 2>&1
- echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/bison_3_linux/bin ]; then
- echo "ERROR: failed to install Bison" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
-fi
-echo "INFO: Bison installation complete" | tee -a $LOGFILE
-echo " *********************************************************** " | tee -a $LOGFILE
-
-
-# -----------------------------------------------------------------------------
# install udis
cd $BASEDIR
echo
@@ -236,17 +209,8 @@ else
cd udis86-1.7.2
./configure --prefix=$TOOLSDIR/udis86-1.7.2 --enable-shared >>$LOGFILE 2>&1
echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/udis86-1.7.2/bin ]; then
- echo "ERROR: failed to install UDIS" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
+ bkgBuild "udis86" "" "install" "$TOOLSDIR/udis86-1.7.2/bin"
fi
-echo "INFO: UDIS installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
@@ -260,6 +224,9 @@ if [ -d $TOOLSDIR/dest-llvm-3.2/release/bin -a -d $TOOLSDIR/dest-llvm-3.2/debug/
else
downloadSource http://llvm.org/releases/3.2/llvm-3.2.src.tar.gz llvm-3.2.src
+ # Depends on UDIS, so make sure it is done
+ wait
+
export MY_UDIS_INSTALL_DIR=$TOOLSDIR/udis86-1.7.2
export MY_LLVM_INSTALL_DIR=$TOOLSDIR/dest-llvm-3.2/
export MY_LLVM_SRC_DIR=$BASEDIR/llvm-3.2.src
@@ -279,20 +246,11 @@ else
CFLAGS=-fgnu89-inline >>$LOGFILE 2>&1
echo "INFO: release configure complete" | tee -a $LOGFILE
- echo "INFO: building release - this will take some time"
- make libs-only >>$LOGFILE 2>&1
- echo "INFO: release make completed" | tee -a $LOGFILE
- make install-libs >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/dest-llvm-3.2/release/bin ]; then
- echo "ERROR: failed to install release LLVM" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
+ bkgBuild "llvm-release" "libs-only" "install-libs" "$TOOLSDIR/dest-llvm-3.2/release/bin"
fi
- echo "INFO: release make install complete, files placed in $TOOLSDIR"
# Build debug version
- if [ ! -d $TOOLSDIR/dest-llvm-3.2/debut/bin ]; then
+ if [ ! -d $TOOLSDIR/dest-llvm-3.2/debug/bin ]; then
mkdir -p $MY_LLVM_OBJ_DIR/debug
cd $MY_LLVM_OBJ_DIR/debug
@@ -303,22 +261,47 @@ else
--with-udis86=$MY_UDIS_INSTALL_DIR/lib \
CFLAGS=-fgnu89-inline >>$LOGFILE 2>&1
echo "INFO: debug configure complete" | tee -a $LOGFILE
- echo "INFO: building debug - this will take some time"
- make libs-only >>$LOGFILE 2>&1
- echo "INFO: debug make completed" | tee -a $LOGFILE
- make install-libs >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/dest-llvm-3.2/debug/bin ]; then
- echo "ERROR: failed to install debug LLVM" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
+ bkgBuild "llvm-debug" "libs-only" "install-libs" "$TOOLSDIR/dest-llvm-3.2/debug/bin"
fi
- echo "INFO: debug make install complete, files placed in $TOOLSDIR"
fi
-echo "INFO: LLVM installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
+# -----------------------------------------------------------------------------
+# install mpi
+cd $BASEDIR
+echo
+echo "INFO: Installing MPI on $(date)" | tee -a $LOGFILE
+if [ -d $TOOLSDIR/dest-mpich-3.0.4/bin ]; then
+ echo "INFO: MPI is already installed, skipping to next tool" | tee -a $LOGFILE
+else
+ downloadSource http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz mpich-3.0.4
+ cd mpich-3.0.4
+ ./configure --prefix=$TOOLSDIR/dest-mpich-3.0.4 --with-device=ch3:sock --disable-f77 --disable-fc >>$LOGFILE 2>&1
+ echo "INFO: configure complete" | tee -a $LOGFILE
+ bkgBuild "MPI" "" "install" "$TOOLSDIR/dest-mpich-3.0.4/bin"
+fi
+echo " *********************************************************** " | tee -a $LOGFILE
+
+
+# -----------------------------------------------------------------------------
+# install bison
+cd $BASEDIR
+echo
+echo "INFO: Installing Bison on $(date)" | tee -a $LOGFILE
+if [ -d $TOOLSDIR/bison_3_linux/bin ]; then
+ echo "INFO: Bison is already installed, skipping to next tool" | tee -a $LOGFILE
+else
+ downloadSource http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz bison-3.0
+ cd bison-3.0
+ ./configure --prefix=$TOOLSDIR/bison_3_linux >>$LOGFILE 2>&1
+ echo "INFO: configure complete" | tee -a $LOGFILE
+ bkgBuild "bison" "" "install" "$TOOLSDIR/bison_3_linux/bin"
+fi
+echo " *********************************************************** " | tee -a $LOGFILE
+
+
+# -----------------------------------------------------------------------------
# Build ICU
cd $BASEDIR
echo
@@ -330,17 +313,8 @@ else
cd icu/source
./configure --with-library-suffix=Nv44 --prefix=$TOOLSDIR/icu4.4/linux64 >>$LOGFILE 2>&1
echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/icu4.4/linux64/bin ]; then
- echo "ERROR: failed to install ICU" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
+ bkgBuild "ICU" "" "install" "$TOOLSDIR/icu4.4/linux64/bin"
fi
-echo "INFO: ICU installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
@@ -356,17 +330,8 @@ else
cd zookeeper-3.4.5/src/c
./configure --prefix=$TOOLSDIR/zookeeper-3.4.5 >>$LOGFILE 2>&1
echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/zookeeper-3.4.5/bin ]; then
- echo "ERROR: failed to install ZooKeeper" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
+ bkgBuild "zookeeper" "" "install" "$TOOLSDIR/zookeeper-3.4.5/bin"
fi
-echo "INFO: ZooKeeper installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
@@ -382,17 +347,8 @@ else
cd thrift-0.9.0
./configure --prefix=$TOOLSDIR/thrift-0.9.0 -without-qt >>$LOGFILE 2>&1
echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/thrift-0.9.0/bin ]; then
- echo "ERROR: failed to install Thrift" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
+ bkgBuild "thrift" "" "install" "$TOOLSDIR/thrift-0.9.0/bin"
fi
-echo "INFO: Thrift installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
@@ -427,7 +383,7 @@ if [[ ! -e /usr/lib64/liblog4cxx.so ]]; then
if [ -d $TOOLSDIR/apache-log4cxx-0.10.0/lib ]; then
echo "INFO: log4cxx is already installed, skipping to next tool" | tee -a $LOGFILE
else
- downloadSource https://dist.apache.org/repos/dist/release/logging/log4cxx/0.10.0/apache-log4cxx-0.10.0.tar.gz apache-log4cxx
+ downloadSource https://dist.apache.org/repos/dist/release/logging/log4cxx/0.10.0/apache-log4cxx-0.10.0.tar.gz apache-log4cxx-0.10.0
cd apache-log4cxx-0.10.0
echo "INFO: headerfile patch - per LOG4CXX-360" | tee -a $LOGFILE
sed -i '1 i#include <string.h>' src/main/cpp/inputstreamreader.cpp
@@ -436,17 +392,9 @@ if [[ ! -e /usr/lib64/liblog4cxx.so ]]; then
sed -i '2 i#include <stdio.h>' src/examples/cpp/console.cpp
./configure --prefix=$TOOLSDIR/apache-log4cxx-0.10.0 >>$LOGFILE 2>&1
echo "INFO: configure complete" | tee -a $LOGFILE
- make >>$LOGFILE 2>&1
- echo "INFO: make completed" | tee -a $LOGFILE
- make install >>$LOGFILE 2>&1
- if [ ! -d $TOOLSDIR/apache-log4cxx-0.10.0/lib ]; then
- echo "ERROR: failed to install log4cxx" | tee -a $LOGFILE
- echo " see details in $LOGFILE" | tee -a $LOGFILE
- exit 2;
- fi
- echo "INFO: make install complete, files placed in $TOOLSDIR"
+ echo "INFO: Be sure dependencies apr-devel and apr-util-devel are installed" | tee -a $LOGFILE
+ bkgBuild "log4cxx" "" "install" "$TOOLSDIR/apache-log4cxx-0.10.0/lib"
fi
- echo "INFO: log4cxx installation complete" | tee -a $LOGFILE
echo " *********************************************************** " | tee -a $LOGFILE
else
echo "INFO: log4cxx is already installed, skipping to next tool" | tee -a $LOGFILE
@@ -455,7 +403,9 @@ fi
# -----------------------------------------------------------------------------
echo
-echo "INFO: Completed tools build on $(date)" | tee -a $LOGFILE
+echo "INFO: Waiting for all background builds. This might take a while." | tee -a $LOGFILE
+wait
+echo "INFO: Completed tools builds on $(date)" | tee -a $LOGFILE
echo "INFO: List of tools directory: " | tee -a $LOGFILE
ls $TOOLSDIR | echo | tee -a $LOGFILE
echo "`ls $TOOLSDIR`" | tee -a $LOGFILE
[2/2] incubator-trafodion git commit: Merge [TRAFODION-1614] PR 306
Speed up build for external dependencies
Posted by db...@apache.org.
Merge [TRAFODION-1614] PR 306 Speed up build for external dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/7f33541a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/7f33541a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/7f33541a
Branch: refs/heads/master
Commit: 7f33541a4c5ef665ef8774ff8002db25989bbc3b
Parents: 83983c7 bb58c06
Author: Dave Birdsall <db...@apache.org>
Authored: Fri Feb 5 16:47:25 2016 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Fri Feb 5 16:47:25 2016 +0000
----------------------------------------------------------------------
install/traf_tools_setup.sh | 206 +++++++++++++++------------------------
1 file changed, 78 insertions(+), 128 deletions(-)
----------------------------------------------------------------------