You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2015/10/09 21:00:56 UTC
[1/3] incubator-trafodion git commit: Trafodion-1513 -- build fixes
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 471a561b2 -> ff6d16188
Trafodion-1513 -- build fixes
The build instructions on the web site were not correctly downloading dependent
tools. To help in the process, a convenience script called traf_tools_setup.sh
located in the install directory was created to download the correct versions
into a user specified directory.
Removed some of the dependencies from the dependent tool list
- QT is not longer built, this is needed by the compiler debugging GUI.
- MAVEN, PROTOBUFs are required to be installed before building Trafodion.
This removes them from the Trafodion configuration file. Plus variables
that point to HADOOP, HBASE, and HIVE were added to the configuration file
to support standalone Apache builds.
Issues with TOOLSDIR not working correctly in sqenvcom.sh setup script.
Incorrect version of Hadoop was downloaded in install_local_hadoop which caused
build error - missing hdfs library.
install_local_hadoop tried to install an non-existent test package - made this
optional. Also report what versions of Hadoop, HBase, Hive, and MySql were downloaded for testing. Trafodion JIRA 1512 was created to fix an additional issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e40f8f05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e40f8f05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e40f8f05
Branch: refs/heads/master
Commit: e40f8f05149315e97c8f601cf14cf7080c22fc5e
Parents: 87b2827
Author: Roberta Marton <ro...@esgyn.com>
Authored: Thu Oct 8 00:41:17 2015 +0000
Committer: Roberta Marton <ro...@esgyn.com>
Committed: Thu Oct 8 00:41:17 2015 +0000
----------------------------------------------------------------------
core/sqf/LocalSettingsTemplate.sh | 33 ++-
core/sqf/sqenvcom.sh | 27 +-
.../sql/scripts/install_hadoop_regr_test_env | 12 +-
core/sqf/sql/scripts/install_local_hadoop | 52 ++--
install/traf_tools_setup.sh | 249 +++++++++++++++++++
5 files changed, 330 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e40f8f05/core/sqf/LocalSettingsTemplate.sh
----------------------------------------------------------------------
diff --git a/core/sqf/LocalSettingsTemplate.sh b/core/sqf/LocalSettingsTemplate.sh
index 1793a39..cf64fb2 100644
--- a/core/sqf/LocalSettingsTemplate.sh
+++ b/core/sqf/LocalSettingsTemplate.sh
@@ -28,19 +28,32 @@
#######################
# Build Dependencies
+#
+# This file needs to be copied to ~/.trafodion and updated to override
+# build tool locations
+# .trafodion is sourced in when running $MY_SQROOT/sqenvcom.sh
+# sqenvcom.sh is sourced in by $MY_SQROOT/sqenv.sh
+# sqenv.sh is sourced in by:
+# $MY_SQROOT/../../env.sh or
+# $MY_SQROOT/sqenvd.sh (debug build) or
+# $MY_SQROOT/sqenvr.sh (release build)
+#
+# sqenvcom.sh sets up the environment in preparation for building and
+# running Trafodion.
+########################
# Standard tools expected to be installed and found in PATH
# Change to explicit path as needed
ANT=ant
+
AR=ar
FLEX=flex
CXX=g++
# Non-standard or newer version tools
-TOOLSDIR="/opt/home/tools" # convenient to put them all in the same place
+TOOLSDIR="/opt/home/tools" # convenient to put dependent tools in the same place
BISON="${TOOLSDIR}/bison_3_linux/bin/bison"
-MAVEN="${TOOLSDIR}/apache-maven-3.0.5/bin/mvn"
LLVM="${TOOLSDIR}/dest-llvm-3.2"
UDIS86="${TOOLSDIR}/udis86-1.7.2"
ICU="${TOOLSDIR}/icu4.4"
@@ -48,10 +61,14 @@ MPICH_ROOT="$TOOLSDIR/dest-mpich-3.0.4"
ZOOKEEPER_DIR="$TOOLSDIR/zookeeper-3.4.5"
THRIFT_LIB_DIR="$TOOLSDIR/thrift-0.9.0/lib"
THRIFT_INC_DIR="$TOOLSDIR/thrift-0.9.0/include"
-PROTOBUFS="/usr"
-QT_TOOLKIT="$TOOLSDIR/Qt-4.8.5-64"
-# Explicitly unset QT_TOOLKIT here if Qt is not installed and you don't want to build the SqlCompilerDebugger
-# unset QT_TOOLKIT
-LOG4CXX_DIR="$TOOLSDIR/apache-log4cxx-0.10.0"
-# HBASE*, HIVE*, HADOOP* locations may be overridden here - see sqf/sqenvcom.sh
+# Explicitly set QT_TOOLKIT here if Qt is installed and you want to build the SqlCompilerDebugger
+# QT_TOOLKIT="$TOOLSDIR/Qt-4.8.5-64"
+
+# HBASE*, HIVE*, HADOOP* locations may be overridden here
+# uncomment the following lines to describe where your Hadoop locations exist.
+# This is needed if you are building with an existing installation
+#HADOOP_PREFIX=/usr/hadoop-2.5.2
+#HBASE_HOME=/usr/hbase-0.98.6-hadoop2
+#HIVE_HOME=/usr/apache-hive-0.13.1-bin
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e40f8f05/core/sqf/sqenvcom.sh
----------------------------------------------------------------------
diff --git a/core/sqf/sqenvcom.sh b/core/sqf/sqenvcom.sh
index a9b2d00..481c403 100644
--- a/core/sqf/sqenvcom.sh
+++ b/core/sqf/sqenvcom.sh
@@ -95,17 +95,20 @@ cpucnt=$(grep processor /proc/cpuinfo | wc -l)
# no number means unlimited, and will swamp the system
export MAKEFLAGS="-j$cpucnt"
+# For now set up the TOOLSDIR, it may be overwritten later when the
+# .trafodion configuration file is loaded.
if [[ -n "$CLUSTERNAME" ]]; then
- export MY_ROOT=/opt/hp
- export TOOLSDIR=${TOOLSDIR:-/home/tools}
+ if [ -z "$TOOLSDIR" ]; then
+ export TOOLSDIR=${TOOLSDIR:-/home/tools}
+ fi
export MY_UDR_ROOT=/home/udr
else
- export MY_ROOT=/opt/home
- export TOOLSDIR=${TOOLSDIR:-/opt/home/tools}
+ if [ -z "$TOOLSDIR" ]; then
+ export TOOLSDIR=${TOOLSDIR:-/opt/home/tools}
+ fi
export MY_UDR_ROOT=$PWD
fi
-export MY_MPI_ROOT=$MY_ROOT
# Use JAVA_HOME if set, else look for installed openjdk, finally toolsdir
REQ_JDK_VER="1.7.0_67"
@@ -267,6 +270,7 @@ elif [[ -f $MY_SQROOT/Makefile && -d $TOOLSDIR ]]; then
# ----------------------------------------------------------------
# native library directories and include directories
+ # Trafodion needs native libs and include file for C++ code to build
export HADOOP_LIB_DIR=$TOOLSDIR/hadoop-2.4.0/lib/native
export HADOOP_INC_DIR=$TOOLSDIR/hadoop-2.4.0/include
export THRIFT_LIB_DIR=$TOOLSDIR/thrift-0.9.0/lib
@@ -537,10 +541,9 @@ EOF
APACHE_HBASE_HOME=$HBASE_HOME
export HBASE_CNF_DIR=$HBASE_HOME/conf
fi
- if [ -f $HIVE_HOME/conf/hive-site.xml ]; then
- APACHE_HIVE_HOME=$HIVE_HOME
- export HIVE_CNF_DIR=$HIVE_HOME/conf
- fi
+
+ APACHE_HIVE_HOME=$HIVE_HOME
+ export HIVE_CNF_DIR=$HIVE_HOME/conf
for cp in `echo $CLASSPATH | sed 's/:/ /g'`
do
@@ -619,6 +622,7 @@ EOF
$APACHE_HBASE_HOME/lib/hbase-client-*.jar
$APACHE_HBASE_HOME/lib/hbase-server-*.jar
$APACHE_HBASE_HOME/lib/hbase-protocol-*.jar
+ $APACHE_HBASE_HOME/lib/hbase-examples-*.jar
$APACHE_HBASE_HOME/lib/htrace-core-*.jar
$APACHE_HBASE_HOME/lib/zookeeper-*.jar
$APACHE_HBASE_HOME/lib/protobuf-*.jar
@@ -795,13 +799,9 @@ fi
# Non-standard or newer version tools
export BISON="${TOOLSDIR}/bison_3_linux/bin/bison" # Need 1.3 version or later
-
-
export LLVM="${TOOLSDIR}/dest-llvm-3.2"
export UDIS86="${TOOLSDIR}/udis86-1.7.2"
export ICU="${TOOLSDIR}/icu4.4"
-export QT_TOOLKIT="${TOOLSDIR}/Qt-4.8.5-64"
-
#######################
# Developer Local over-rides (see sqf/LocalSettingsTemplate.sh)
@@ -810,6 +810,7 @@ if [[ -r ~/.trafodion ]]; then
[[ $SQ_VERBOSE == 1 ]] && echo "Sourcing local settings file ~/.trafodion"
source ~/.trafodion
fi
+
# PROTOBUFS may include local over-rides
export PROTOBUFS_LIB=$PROTOBUFS/lib
export PROTOBUFS_INC=$PROTOBUFS/include
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e40f8f05/core/sqf/sql/scripts/install_hadoop_regr_test_env
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/install_hadoop_regr_test_env b/core/sqf/sql/scripts/install_hadoop_regr_test_env
index ee1bac9..0abfb74 100755
--- a/core/sqf/sql/scripts/install_hadoop_regr_test_env
+++ b/core/sqf/sql/scripts/install_hadoop_regr_test_env
@@ -134,12 +134,12 @@ fi
if [ ! -f $MY_LOCAL_SW_DIST/${TPCDS_ZIP} ]; then
# Right now there is no URL to download this tool automatically
- echo '*********************** ERROR *************************'
- echo "Please download the tpcds_kit.zip file from"
- echo "http://www.tpc.org/tpcds/dsgen-download-request.asp"
- echo "and place it into a directory pointed to by the MY_LOCAL_SW_DIST"
- echo "environment variable. Then retry installing. Sorry, this is due"
- echo "to the TPC wanting your email address."
+ # Please download the tpcds_kit.zip file from"
+ # "http://www.tpc.org/tpcds/dsgen-download-request.asp"
+ # "and place it into a directory pointed to by the MY_LOCAL_SW_DIST"
+ # "environment variable. Then retry installing. Sorry, this is due"
+ # "to the TPC wanting your email address."
+ echo "The testware tpcds_kit.zip does not exist and will not be installed"
exit 99
fi
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e40f8f05/core/sqf/sql/scripts/install_local_hadoop
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/install_local_hadoop b/core/sqf/sql/scripts/install_local_hadoop
index 490901f..c558023 100755
--- a/core/sqf/sql/scripts/install_local_hadoop
+++ b/core/sqf/sql/scripts/install_local_hadoop
@@ -521,8 +521,16 @@ fi
# Specify mirrors and versions of needed components
#####################################################
-HADOOP_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
-HADOOP_TAR=hadoop-2.5.0-cdh5.3.0.tar.gz
+# See JIRA TRAFODION-1512 that will eliminate this check
+if [ -d $MY_LOCAL_SW_DIST ]; then
+
+ HADOOP_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
+ HADOOP_TAR=hadoop-2.5.0-cdh5.3.0.tar.gz
+else
+ HADOOP_MIRROR_URL=http://download.nextag.com/apache/hadoop/common/hadoop-2.5.2
+ HADOOP_TAR=hadoop-2.5.2.tar.gz
+fi
+
if [[ "$SQ_HBASE_DISTRO" = "HDP" ]]; then
HADOOP_TAR=hadoop-2.6.0.2.2.0.0-2041.tar.gz
fi
@@ -541,7 +549,6 @@ HIVE_MIRROR_URL=https://archive.apache.org/dist/hive/hive-0.13.1
HIVE_PREFIX=apache-hive-0.13.1-bin
HIVE_TAR=${HIVE_PREFIX}.tar.gz
-#HBASE_MIRROR_URL=http://psg.mtu.edu/pub/apache/hbase/hbase-0.98.3
HBASE_MIRROR_URL=http://archive.cloudera.com/cdh5/cdh/5
HBASE_TAR=hbase-0.98.6-cdh5.3.0.tar.gz
@@ -565,8 +572,9 @@ fi
# check for missing tpcds_kit.zip file
install_hadoop_regr_test_env --check
+INSTALL_TPCDS=0
if [ $? -ne 0 ]; then
- exit 1
+ INSTALL_TPCDS=1
fi
if [ -d "$MY_SW_ROOT" ]; then
@@ -925,8 +933,10 @@ else
if [ -f $MY_LOCAL_SW_DIST/${HADOOP_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HADOOP_TAR} .
+ echo "Downloaded Hadoop tar file: $MY_LOCAL_SW_DIST/${HADOOP_TAR}"
else
curl -O ${HADOOP_MIRROR_URL}/${HADOOP_TAR}
+ echo "Downloaded Hadoop tar file: ${HADOOP_MIRROR_URL}/${HADOOP_TAR}"
fi
echo "Unpacking Hadoop tar file..."
@@ -1143,8 +1153,10 @@ else
if [ -f $MY_LOCAL_SW_DIST/${MYSQL_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${MYSQL_TAR} .
+ echo "Downloaded MySql tar file: $MY_LOCAL_SW_DIST/${MYSQL_TAR}"
else
curl ${MYSQL_MIRROR_URL}/${MYSQL_TAR} -o ${MYSQL_TAR}
+ echo "Downloaded MySql tar file: ${MYSQL_MIRROR_URL}/${MYSQL_TAR}"
fi
echo "Unpacking MySQL tar file ${MYSQL_TAR} ..."
@@ -1220,8 +1232,10 @@ else
if [ -f $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} .
+ echo "Downloaded MySql JDBC tar file: $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}"
else
curl -O ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}
+ echo "Downloaded MySql JDBC tar file: ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}"
fi
echo "Unpacking MySQL JDBC tar file ${MYSQL_JDBC_TAR} ..."
@@ -1241,8 +1255,10 @@ else
if [ -f $MY_LOCAL_SW_DIST/${HIVE_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HIVE_TAR} .
+ echo "Downloaded Hive tar file: $MY_LOCAL_SW_DIST/${HIVE_TAR}"
else
curl -O ${HIVE_MIRROR_URL}/${HIVE_TAR}
+ echo "Downloaded Hive tar file: ${HIVE_MIRROR_URL}/${HIVE_TAR}"
fi
echo "Unpacking Hive tar file ${HIVE_TAR} ..."
@@ -1367,8 +1383,10 @@ else
if [ -f $MY_LOCAL_SW_DIST/${HBASE_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HBASE_TAR} .
+ echo "Downloaded HBase tar file: $MY_LOCAL_SW_DIST/${HBASE_TAR}"
else
curl -O ${HBASE_MIRROR_URL}/${HBASE_TAR}
+ echo "Downloaded HBase tar file: ${HBASE_MIRROR_URL}/${HBASE_TAR}"
fi
echo "Unpacking HBase tar file ${HBASE_TAR} ..."
@@ -1496,18 +1514,20 @@ fi
cd $MY_SW_ROOT
-if [ -d tpcds/tools ]; then
- echo "TPC-DS files already exist, skipping TPC-DS setup"
-else
- install_hadoop_regr_test_env \
- --unpackDir=$MY_SW_ROOT/tpcds \
- --dataDir=${MY_DATA_DIR}/tpcds \
- --logFile=$MY_LOG_FILE \
- --hdfsCmd=${MY_SW_SCRIPTS_DIR}/swhdfs \
- --hiveCmd=${MY_SW_SCRIPTS_DIR}/swhive
- if [ $? -ne 0 ]; then
- echo "Error installing TPC-DS and ORC files, exiting..."
- exit 1
+if [ $INSTALL_TPCDS -eq 1 ]; then
+ if [ -d tpcds/tools ]; then
+ echo "TPC-DS files already exist, skipping TPC-DS setup"
+ else
+ install_hadoop_regr_test_env \
+ --unpackDir=$MY_SW_ROOT/tpcds \
+ --dataDir=${MY_DATA_DIR}/tpcds \
+ --logFile=$MY_LOG_FILE \
+ --hdfsCmd=${MY_SW_SCRIPTS_DIR}/swhdfs \
+ --hiveCmd=${MY_SW_SCRIPTS_DIR}/swhive
+ if [ $? -ne 0 ]; then
+ echo "Error installing TPC-DS and ORC files, exiting..."
+ exit 1
+ fi
fi
fi
# end of TPC-DS setup
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e40f8f05/install/traf_tools_setup.sh
----------------------------------------------------------------------
diff --git a/install/traf_tools_setup.sh b/install/traf_tools_setup.sh
new file mode 100755
index 0000000..9cc56a2
--- /dev/null
+++ b/install/traf_tools_setup.sh
@@ -0,0 +1,249 @@
+#!/bin/bash
+#
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+#
+
+# -----------------------------------------------------------------------------
+# script: traf_tools_setup
+#
+# Helper script that downloads and installs dependent tools required by
+# Apache Trafodion to build from a source distribution
+#
+# Tools installed:
+#
+# MPICH: Implementation of the Message Passing Interface (MPI) standard. For use
+# in Trafodion, MPICH must be built to force sockets to be used in both
+# internode and intranode message passing.
+# Bison: General-purpose parser generator.
+# Udis86: Minimalistic disassembler library (libudis86) for the x86 class of
+# instruction set architectures.
+# LLVM: Collection of modular and reusable compiler and toolchain technologies.
+# ICU: C/C++ and Java libraries providing Unicode and Globalization support for
+# software applications.
+# Zookeeper: Coordination service for distributed applications. It exposes
+# common services such as naming, configuration management, synchronization,
+# and group services.
+# Thrift: Communications and data serialization tool
+#
+# Script can be modified to meet the needs of your environment
+# May need root or SUDO access to install tools in desired location
+# -----------------------------------------------------------------------------
+
+function Usage {
+ echo
+ echo "Usage: $0 -d <downloaddir> -i <installdir>"
+ echo
+ echo " -d <downloaddir> - location of download directory"
+ echo " -i <installdir> - location of install directory"
+ echo " -v - verbose mode"
+ echo
+ echo " -h - help"
+ echo
+ echo "example: traf_tools_setup.sh -d /home/userx/download -i /home/userx/tools"
+ echo
+}
+
+ VERBOSE=0
+ TOOLSDIR=
+ BASEDIR=
+
+ while getopts "d:i:hv" arg
+ do
+ case $arg in
+ d)
+ BASEDIR=${OPTARG};
+ ;;
+ h)
+ Usage;
+ exit 1;
+ ;;
+ i)
+ TOOLSDIR=${OPTARG};
+ ;;
+ v)
+ VERBOSE=1;
+ ;;
+ *)
+ Usage;
+ exit 1;
+ ;;
+ esac
+ done
+
+
+if [ "$BASEDIR" == "" ]; then
+ echo
+ echo "ERROR: download directory (-d) is not specified"
+ Usage;
+ exit 1;
+fi
+
+if [ ! -d "$BASEDIR" ]; then
+ echo
+ echo "ERROR: download directory ($BASEDIR) does not exist"
+ echo
+ exit 1;
+fi
+
+if [ "$TOOLSDIR" == "" ]; then
+ echo
+ echo "ERROR: install directory (-t) is not specified"
+ Usage;
+ exit 1;
+fi
+
+if [ ! -d "$TOOLSDIR" ]; then
+ echo
+ echo "ERROR: install directory ($TOOLSDIR) does not exist"
+ echo
+ exit 1;
+fi
+
+if [ $VERBOSE -eq 1 ]; then
+ echo "basedir is: $BASDIR";
+ echo "toolsdir is: $TOOLSDIR":
+fi
+
+#install mpi
+cd $BASEDIR
+wget http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz
+tar -xzf mpich-3.0.4.tar.gz
+cd mpich-3.0.4
+./configure --prefix=$TOOLSDIR/dest-mpich-3.0.4 --with-device=ch3:sock --disable-f77 --disable-fc
+make
+make check
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of MPI"
+fi
+
+# See if need to install protobufs tested with version 2.3.0
+protoc --version
+
+# install bison
+cd $BASEDIR
+wget http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz
+tar -xzf bison-3.0.tar.gz
+cd bison-3.0
+./configure --prefix=$TOOLSDIR/bison_3_linux
+make
+make check
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of BISON"
+fi
+
+# install udis
+cd $BASEDIR
+wget http://sourceforge.net/projects/udis86/files/udis86/1.7/udis86-1.7.2.tar.gz
+tar -xzf udis86-1.7.2.tar.gz
+cd udis86-1.7.2
+./configure --prefix=$TOOLSDIR/udis86-1.7.2 --enable-shared
+make
+make check
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of UDIS"
+fi
+
+#install LLVM
+
+cd $BASEDIR
+wget http://llvm.org/releases/3.2/llvm-3.2.src.tar.gz
+tar xzf llvm-3.2.src.tar.gz
+
+export MY_UDIS_INSTALL_DIR=$TOOLSDIR/udis86-1.7.2
+export MY_LLVM_INSTALL_DIR=$TOOLSDIR/dest-llvm-3.2/
+export MY_LLVM_SRC_DIR=$BASEDIR/llvm-3.2.src
+export MY_LLVM_OBJ_DIR=$BASEDIR/llvm-3.2.obj/
+export LD_LIBRARY_PATH=$MY_UDIS_INSTALL_DIR/lib:$LD_LIBRARY_PATH
+export C_INCLUDE_PATH=$MY_UDIS_INSTALL_DIR/include
+export CPATH=$MY_UDIS_INSTALL_DIR/include
+
+# Build release version
+mkdir -p $MY_LLVM_OBJ_DIR/release
+cd $MY_LLVM_OBJ_DIR/release
+
+$MY_LLVM_SRC_DIR/configure --prefix=$MY_LLVM_INSTALL_DIR/release \
+ --enable-shared --enable-targets=x86,x86_64,cpp \
+ --with-udis86=$MY_UDIS_INSTALL_DIR/lib \
+ CFLAGS=-fgnu89-inline
+
+make libs-only
+make install-libs
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of LLVM release"
+fi
+
+# Build debug version
+mkdir -p $MY_LLVM_OBJ_DIR/debug
+cd $MY_LLVM_OBJ_DIR/debug
+
+$MY_LLVM_SRC_DIR/configure --prefix=$MY_LLVM_INSTALL_DIR/debug \
+ --enable-optimized --enable-jit \
+ --enable-debug-runtime --enable-debug-symbols \
+ --enable-shared --enable-targets=x86,x86_64,cpp \
+ --with-udis86=$MY_UDIS_INSTALL_DIR/lib \
+ CFLAGS=-fgnu89-inline
+
+make libs-only
+make install-libs
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of LLVM debug"
+fi
+
+# Build ICU
+cd $BASEDIR
+wget http://download.icu-project.org/files/icu4c/4.4/icu4c-4_4-src.tgz
+tar -xzf icu4c-4_4-src.tgz
+cd icu/source
+./configure --with-library-suffix=Nv44 --prefix=$TOOLSDIR/icu4.4/linux64
+make
+make check
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of ICU"
+fi
+
+# make zookeeper
+cd $BASEDIR
+wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz
+tar -zxf zookeeper-3.4.5.tar.gz
+cd zookeeper-3.4.5/src/c
+./configure --prefix=$TOOLSDIR/zookeeper-3.4.5
+make
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of ZOOKEEPER"
+fi
+
+# make thrift
+cd $BASEDIR
+wget http://archive.apache.org/dist/thrift/0.9.0/thrift-0.9.0.tar.gz
+tar -xzf thrift-0.9.0.tar.gz
+cd thrift-0.9.0
+./configure --prefix=$TOOLSDIR/thrift-0.9.0 -without-qt
+make
+make install
+if [ $VERBOSE -eq 1 ]; then
+ echo "STEP completed: installation of THRIFT"
+fi
[2/3] incubator-trafodion git commit: Trafodion-1513 -- build fixes
Posted by db...@apache.org.
Trafodion-1513 -- build fixes
Fixed issues from previous delivery:
- simplified the TOOLSDIR check in sqenvcom.sh
- added back in support for QT_TOOLKIT, builds if package exists
- mentioned Hive regression will fail if TPC is not installed
- Fixed the INSTALL_TPC check
- If the tools directory does not exit in traf_tools_setup.sh, will ask
if it should be created
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/a7319df3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/a7319df3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/a7319df3
Branch: refs/heads/master
Commit: a7319df3648f4d93ce794660e5d21159dec4c229
Parents: e40f8f0
Author: Roberta Marton <ro...@esgyn.com>
Authored: Fri Oct 9 00:55:58 2015 +0000
Committer: Roberta Marton <ro...@esgyn.com>
Committed: Fri Oct 9 00:55:58 2015 +0000
----------------------------------------------------------------------
core/sqf/sqenvcom.sh | 19 ++++++++++-------
.../sql/scripts/install_hadoop_regr_test_env | 1 +
core/sqf/sql/scripts/install_local_hadoop | 22 ++++++++++----------
install/traf_tools_setup.sh | 21 +++++++++++++++----
4 files changed, 41 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a7319df3/core/sqf/sqenvcom.sh
----------------------------------------------------------------------
diff --git a/core/sqf/sqenvcom.sh b/core/sqf/sqenvcom.sh
index 481c403..b9ff2d9 100644
--- a/core/sqf/sqenvcom.sh
+++ b/core/sqf/sqenvcom.sh
@@ -97,16 +97,14 @@ export MAKEFLAGS="-j$cpucnt"
# For now set up the TOOLSDIR, it may be overwritten later when the
# .trafodion configuration file is loaded.
-if [[ -n "$CLUSTERNAME" ]]; then
- if [ -z "$TOOLSDIR" ]; then
+if [ -z "$TOOLSDIR" ]; then
+ if [[ -n "$CLUSTERNAME" ]]; then
export TOOLSDIR=${TOOLSDIR:-/home/tools}
- fi
- export MY_UDR_ROOT=/home/udr
-else
- if [ -z "$TOOLSDIR" ]; then
+ export MY_UDR_ROOT=/home/udr
+ else
export TOOLSDIR=${TOOLSDIR:-/opt/home/tools}
+ export MY_UDR_ROOT=$PWD
fi
- export MY_UDR_ROOT=$PWD
fi
@@ -661,6 +659,13 @@ else
export LOG4CXX_INC_DIR=/usr/include/log4cxx
fi
+# For now, set the QT_TOOLKIT envvar if the required version exists in the
+# download location
+if [[ -d $TOOLSDIR/Qt-4.8.5-64 ]];
+then
+ export QT_TOOLKIT="$TOOLSDIR/Qt-4.8.5-64"
+fi
+
# for debugging
export LD_BIND_NOW=true
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a7319df3/core/sqf/sql/scripts/install_hadoop_regr_test_env
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/install_hadoop_regr_test_env b/core/sqf/sql/scripts/install_hadoop_regr_test_env
index 0abfb74..b3eacc0 100755
--- a/core/sqf/sql/scripts/install_hadoop_regr_test_env
+++ b/core/sqf/sql/scripts/install_hadoop_regr_test_env
@@ -140,6 +140,7 @@ if [ ! -f $MY_LOCAL_SW_DIST/${TPCDS_ZIP} ]; then
# "environment variable. Then retry installing. Sorry, this is due"
# "to the TPC wanting your email address."
echo "The testware tpcds_kit.zip does not exist and will not be installed"
+ echo "This testware is needed to run developer HIVE regression tests"
exit 99
fi
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a7319df3/core/sqf/sql/scripts/install_local_hadoop
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/install_local_hadoop b/core/sqf/sql/scripts/install_local_hadoop
index c558023..23503c8 100755
--- a/core/sqf/sql/scripts/install_local_hadoop
+++ b/core/sqf/sql/scripts/install_local_hadoop
@@ -571,10 +571,10 @@ if [ `netstat -anl | grep ${MY_HADOOP_JOB_TRACKER_HTTP_PORT_NUM} | grep LISTEN |
fi
# check for missing tpcds_kit.zip file
+INSTALL_TPCDS=1
install_hadoop_regr_test_env --check
-INSTALL_TPCDS=0
if [ $? -ne 0 ]; then
- INSTALL_TPCDS=1
+ INSTALL_TPCDS=0
fi
if [ -d "$MY_SW_ROOT" ]; then
@@ -933,7 +933,7 @@ else
if [ -f $MY_LOCAL_SW_DIST/${HADOOP_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HADOOP_TAR} .
- echo "Downloaded Hadoop tar file: $MY_LOCAL_SW_DIST/${HADOOP_TAR}"
+ echo "Copied Hadoop tar file from: $MY_LOCAL_SW_DIST/${HADOOP_TAR}"
else
curl -O ${HADOOP_MIRROR_URL}/${HADOOP_TAR}
echo "Downloaded Hadoop tar file: ${HADOOP_MIRROR_URL}/${HADOOP_TAR}"
@@ -1149,11 +1149,11 @@ if [ -d mysql/bin ]; then
echo "MySQL files already exist, skipping MySQL setup"
else
#####################################################
- echo "Downloading MySQL..."
+ echo "Setting up MySQL..."
if [ -f $MY_LOCAL_SW_DIST/${MYSQL_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${MYSQL_TAR} .
- echo "Downloaded MySql tar file: $MY_LOCAL_SW_DIST/${MYSQL_TAR}"
+ echo "Copied MySql tar file from: $MY_LOCAL_SW_DIST/${MYSQL_TAR}"
else
curl ${MYSQL_MIRROR_URL}/${MYSQL_TAR} -o ${MYSQL_TAR}
echo "Downloaded MySql tar file: ${MYSQL_MIRROR_URL}/${MYSQL_TAR}"
@@ -1228,11 +1228,11 @@ if [ -f mysql-connector-java-*/mysql-connector-java-*.jar ]; then
echo "MySQL JDBC driver file already exists, skipping JDBC setup"
else
#####################################################
- echo "Downloading MySQL JDBC driver..."
+ echo "Setting up MySQL JDBC driver..."
if [ -f $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR} .
- echo "Downloaded MySql JDBC tar file: $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}"
+ echo "Copied MySql JDBC tar file from: $MY_LOCAL_SW_DIST/${MYSQL_JDBC_TAR}"
else
curl -O ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}
echo "Downloaded MySql JDBC tar file: ${MYSQL_JDBC_URL}/${MYSQL_JDBC_TAR}"
@@ -1251,11 +1251,11 @@ if [ -d hive/bin ]; then
echo "Hive files already exist, skipping Hive setup"
else
#####################################################
- echo "Downloading Hive..."
+ echo "Setting up Hive..."
if [ -f $MY_LOCAL_SW_DIST/${HIVE_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HIVE_TAR} .
- echo "Downloaded Hive tar file: $MY_LOCAL_SW_DIST/${HIVE_TAR}"
+ echo "Copied Hive tar file from: $MY_LOCAL_SW_DIST/${HIVE_TAR}"
else
curl -O ${HIVE_MIRROR_URL}/${HIVE_TAR}
echo "Downloaded Hive tar file: ${HIVE_MIRROR_URL}/${HIVE_TAR}"
@@ -1379,11 +1379,11 @@ if [ -d hbase/bin ]; then
echo "HBase files already exist, skipping HBase setup"
else
#####################################################
- echo "Downloading HBase..."
+ echo "Setting up HBase..."
if [ -f $MY_LOCAL_SW_DIST/${HBASE_TAR} ]; then
cp $MY_LOCAL_SW_DIST/${HBASE_TAR} .
- echo "Downloaded HBase tar file: $MY_LOCAL_SW_DIST/${HBASE_TAR}"
+ echo "Copied HBase tar file from: $MY_LOCAL_SW_DIST/${HBASE_TAR}"
else
curl -O ${HBASE_MIRROR_URL}/${HBASE_TAR}
echo "Downloaded HBase tar file: ${HBASE_MIRROR_URL}/${HBASE_TAR}"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a7319df3/install/traf_tools_setup.sh
----------------------------------------------------------------------
diff --git a/install/traf_tools_setup.sh b/install/traf_tools_setup.sh
index 9cc56a2..a2290bc 100755
--- a/install/traf_tools_setup.sh
+++ b/install/traf_tools_setup.sh
@@ -112,10 +112,23 @@ if [ "$TOOLSDIR" == "" ]; then
fi
if [ ! -d "$TOOLSDIR" ]; then
- echo
- echo "ERROR: install directory ($TOOLSDIR) does not exist"
- echo
- exit 1;
+ read -p "Tools install directory $TOOLSDIR does not exist, do you want to to create it? y/n : " CREATEDIR
+ if [ "$CREATEDIR" == "y" ]; then
+ mkdir $TOOLSDIR
+ if [ ! -d "$TOOLSDIR" ]; then
+ echo
+ echo "ERROR: unable to create directory $TOOLSDIR"
+ echo
+ exit 1
+ else
+ echo "Created directory $TOOLSDIR"
+ fi
+ else
+ echo
+ echo "ERROR: install directory ($TOOLSDIR) does not exist"
+ echo
+ exit 1;
+ fi
fi
if [ $VERBOSE -eq 1 ]; then
[3/3] incubator-trafodion git commit: Merge [TRAFODION-1513] PR 109
Trafodion build from source tar failing
Posted by db...@apache.org.
Merge [TRAFODION-1513] PR 109 Trafodion build from source tar failing
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/ff6d1618
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/ff6d1618
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/ff6d1618
Branch: refs/heads/master
Commit: ff6d1618839fae941cddad1d38351b000b356736
Parents: 471a561 a7319df
Author: Dave Birdsall <db...@apache.org>
Authored: Fri Oct 9 18:59:58 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Fri Oct 9 18:59:58 2015 +0000
----------------------------------------------------------------------
core/sqf/LocalSettingsTemplate.sh | 33 ++-
core/sqf/sqenvcom.sh | 40 +--
.../sql/scripts/install_hadoop_regr_test_env | 13 +-
core/sqf/sql/scripts/install_local_hadoop | 60 +++--
install/traf_tools_setup.sh | 262 +++++++++++++++++++
5 files changed, 357 insertions(+), 51 deletions(-)
----------------------------------------------------------------------