You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/09/01 21:36:42 UTC
[3/3] incubator-impala git commit: IMPALA-4006: dangerous rm -rf
statements in scripts
IMPALA-4006: dangerous rm -rf statements in scripts
Quoted variable substitutions in rm -rf commands and in many other
places. This prevents disasters if those variables contain whitespace.
Redirected output of the cd commands to /dev/null. This prevents
polluting the target variable with the directory name when the CDPATH
environment variable is set.
Change-Id: I7503794180dee99eeb979e67f34e3b2edade70fe
Reviewed-on: http://gerrit.cloudera.org:8080/4078
Tested-by: Internal Jenkins
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a60ba6d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a60ba6d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a60ba6d2
Branch: refs/heads/master
Commit: a60ba6d2743eee5f2e5c5c3f1baa375ede0195e5
Parents: 2cd7524
Author: Zoltan Ivanfi <zi...@cloudera.com>
Authored: Mon Aug 22 17:29:44 2016 +0200
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Thu Sep 1 21:26:52 2016 +0000
----------------------------------------------------------------------
bin/clean.sh | 20 +--
bin/impala-config.sh | 200 ++++++++++++++-------------
bin/impala-python | 2 +-
bin/impala-python-common.sh | 4 +-
bin/run-all-tests.sh | 44 +++---
bin/set-classpath.sh | 26 ++--
buildall.sh | 104 +++++++-------
infra/python/deps/download_requirements | 12 +-
8 files changed, 209 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/clean.sh
----------------------------------------------------------------------
diff --git a/bin/clean.sh b/bin/clean.sh
index 00d02b3..9ac73cf 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -29,8 +29,8 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
make clean || :
# Stop the minikdc if needed.
-if ${CLUSTER_DIR}/admin is_kerberized; then
- ${IMPALA_HOME}/testdata/bin/minikdc.sh stop
+if "${CLUSTER_DIR}/admin" is_kerberized; then
+ "${IMPALA_HOME}/testdata/bin/minikdc.sh" stop
fi
# clean the external data source project
@@ -45,35 +45,35 @@ pushd $IMPALA_FE_DIR
rm -rf target
rm -f src/test/resources/{core,hbase,hive}-site.xml
rm -rf generated-sources/*
-rm -rf ${IMPALA_LOGS_DIR}/*
+[ -z "$IMPALA_LOGS_DIR" ] || rm -rf "${IMPALA_LOGS_DIR}"/*
mkdir -p $IMPALA_ALL_LOGS_DIRS
popd
# clean be
-pushd $IMPALA_HOME/be
+pushd "$IMPALA_HOME/be"
# remove everything listed in .gitignore
git clean -Xdfq
popd
# clean shell build artifacts
-pushd $IMPALA_HOME/shell
+pushd "$IMPALA_HOME/shell"
# remove everything listed in .gitignore
git clean -Xdfq
popd
# Clean stale .pyc, .pyo files and __pycache__ directories.
-pushd ${IMPALA_HOME}
+pushd "${IMPALA_HOME}"
find . -type f -name "*.py[co]" -delete
find . -type d -name "__pycache__" -delete
popd
# clean llvm
-rm -f $IMPALA_HOME/llvm-ir/impala*.ll
-rm -f $IMPALA_HOME/be/generated-sources/impala-ir/*
+rm -f "$IMPALA_HOME/llvm-ir/"impala*.ll
+rm -f "$IMPALA_HOME/be/generated-sources/impala-ir/"*
# Cleanup Impala-lzo
-if [ -e $IMPALA_LZO ]; then
- pushd $IMPALA_LZO; git clean -fdx .; popd
+if [ -e "$IMPALA_LZO" ]; then
+ pushd "$IMPALA_LZO"; git clean -fdx .; popd
fi
# When switching to and from toolchain, make sure to remove all CMake generated files
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f79d8f8..7f4df7f 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -34,16 +34,16 @@ if [[ ! -e "$JAVA" ]]; then
return 1
fi
-if [ -z $IMPALA_HOME ]; then
- if [[ ! -z $ZSH_NAME ]]; then
- export IMPALA_HOME=$(dirname $(cd $(dirname ${(%):-%x}) && pwd))
+if [ -z "$IMPALA_HOME" ]; then
+ if [[ ! -z "$ZSH_NAME" ]]; then
+ export IMPALA_HOME=$(dirname "$(cd $(dirname ${(%):-%x}) >/dev/null && pwd)")
else
- export IMPALA_HOME=$(dirname $(cd $(dirname "${BASH_SOURCE[0]}") && pwd))
+ export IMPALA_HOME=$(dirname "$(cd $(dirname "${BASH_SOURCE[0]}") >/dev/null && pwd)")
fi
fi
-: ${IMPALA_TOOLCHAIN=$IMPALA_HOME/toolchain}
-if [ -z $IMPALA_TOOLCHAIN ]; then
+: ${IMPALA_TOOLCHAIN="$IMPALA_HOME/toolchain"}
+if [ -z "$IMPALA_TOOLCHAIN" ]; then
echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\
"leave it unset."
return 1
@@ -69,14 +69,14 @@ fi
# If enabled, debug symbols are added to cross-compiled IR.
: ${ENABLE_IMPALA_IR_DEBUG_INFO=false}
-if [ -d $IMPALA_HOME/thirdparty ]; then
+if [ -d "$IMPALA_HOME/thirdparty" ]; then
NO_THIRDPARTY=false
else
NO_THIRDPARTY=true
fi
# If true, download and use the CDH components from S3 instead of the ones
# in $IMPALA_HOME/thirdparty.
-: ${DOWNLOAD_CDH_COMPONENTS=$NO_THIRDPARTY}
+: ${DOWNLOAD_CDH_COMPONENTS="$NO_THIRDPARTY"}
export IMPALA_TOOLCHAIN
export SKIP_TOOLCHAIN_BOOTSTRAP
@@ -85,7 +85,7 @@ export USE_GOLD_LINKER
export IMPALA_CXX_COMPILER
export ENABLE_IMPALA_IR_DEBUG_INFO
export DOWNLOAD_CDH_COMPONENTS
-export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)
+export IS_OSX="$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)"
# To use a local build of Kudu, set KUDU_BUILD_DIR to the path Kudu was built in and
# set KUDU_CLIENT_DIR to the path KUDU was installed in.
@@ -101,11 +101,11 @@ export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false
: ${KUDU_CLIENT_DIR=}
export KUDU_BUILD_DIR
export KUDU_CLIENT_DIR
-if [[ -n $KUDU_BUILD_DIR && -z $KUDU_CLIENT_DIR ]]; then
+if [[ -n "$KUDU_BUILD_DIR" && -z "$KUDU_CLIENT_DIR" ]]; then
echo When KUDU_BUILD_DIR is set KUDU_CLIENT_DIR must also be set. 1>&2
return 1
fi
-if [[ -z $KUDU_BUILD_DIR && -n $KUDU_CLIENT_DIR ]]; then
+if [[ -z "$KUDU_BUILD_DIR" && -n "$KUDU_CLIENT_DIR" ]]; then
echo When KUDU_CLIENT_DIR is set KUDU_BUILD_DIR must also be set. 1>&2
return 1
fi
@@ -117,20 +117,20 @@ export USE_KUDU_DEBUG_BUILD
# into the backend. The frontend build is OS independent since it is Java.
if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then
KUDU_IS_SUPPORTED=true
- if [[ -z $KUDU_BUILD_DIR ]]; then
+ if [[ -z "$KUDU_BUILD_DIR" ]]; then
if ! $IS_OSX; then
if ! which lsb_release &>/dev/null; then
echo Unable to find the 'lsb_release' command. \
Please ensure it is available in your PATH. 1>&2
return 1
fi
- DISTRO_VERSION=$(lsb_release -sir 2>&1)
+ DISTRO_VERSION="$(lsb_release -sir 2>&1)"
if [[ $? -ne 0 ]]; then
echo lsb_release cammond failed, output was: "$DISTRO_VERSION" 1>&2
return 1
fi
# Remove spaces, trim minor versions, and convert to lowercase.
- DISTRO_VERSION=$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")
+ DISTRO_VERSION="$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")"
case "$DISTRO_VERSION" in
# "enterprise" is Oracle
centos5 | debian* | enterprise*5 | redhat*5 | suse* | ubuntu*12)
@@ -142,18 +142,18 @@ fi
export KUDU_IS_SUPPORTED
export CDH_MAJOR_VERSION=5
-export HADOOP_LZO=${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}
-export IMPALA_LZO=${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}
-export IMPALA_AUX_TEST_HOME=${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}
-export TARGET_FILESYSTEM=${TARGET_FILESYSTEM-"hdfs"}
-export FILESYSTEM_PREFIX=${FILESYSTEM_PREFIX-""}
-export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-"DummySecretAccessKey"}
-export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-"DummyAccessKeyId"}
-export S3_BUCKET=${S3_BUCKET-""}
-export HDFS_REPLICATION=${HDFS_REPLICATION-3}
-export ISILON_NAMENODE=${ISILON_NAMENODE-""}
-export DEFAULT_FS=${DEFAULT_FS-"hdfs://localhost:20500"}
-export WAREHOUSE_LOCATION_PREFIX=${WAREHOUSE_LOCATION_PREFIX-""}
+export HADOOP_LZO="${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}"
+export IMPALA_LZO="${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}"
+export IMPALA_AUX_TEST_HOME="${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}"
+export TARGET_FILESYSTEM="${TARGET_FILESYSTEM-hdfs}"
+export FILESYSTEM_PREFIX="${FILESYSTEM_PREFIX-}"
+export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY-DummySecretAccessKey}"
+export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID-DummyAccessKeyId}"
+export S3_BUCKET="${S3_BUCKET-}"
+export HDFS_REPLICATION="${HDFS_REPLICATION-3}"
+export ISILON_NAMENODE="${ISILON_NAMENODE-}"
+export DEFAULT_FS="${DEFAULT_FS-hdfs://localhost:20500}"
+export WAREHOUSE_LOCATION_PREFIX="${WAREHOUSE_LOCATION_PREFIX-}"
export LOCAL_FS="file:${WAREHOUSE_LOCATION_PREFIX}"
export METASTORE_DB="hive_impala"
@@ -198,8 +198,8 @@ elif [ "${TARGET_FILESYSTEM}" = "local" ]; then
"'$WAREHOUSE_LOCATION_PREFIX'"
return 1
fi
- export DEFAULT_FS=${LOCAL_FS}
- export FILESYSTEM_PREFIX=${LOCAL_FS}
+ export DEFAULT_FS="${LOCAL_FS}"
+ export FILESYSTEM_PREFIX="${LOCAL_FS}"
elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then
echo "Unsupported filesystem '$TARGET_FILESYSTEM'"
echo "Valid values are: hdfs, isilon, s3, local"
@@ -207,15 +207,15 @@ elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then
fi
# Directories where local cluster logs will go when running tests or loading data
-export IMPALA_LOGS_DIR=${IMPALA_HOME}/logs
-export IMPALA_CLUSTER_LOGS_DIR=${IMPALA_LOGS_DIR}/cluster
-export IMPALA_DATA_LOADING_LOGS_DIR=${IMPALA_LOGS_DIR}/data_loading
-export IMPALA_DATA_LOADING_SQL_DIR=${IMPALA_DATA_LOADING_LOGS_DIR}/sql
-export IMPALA_FE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/fe_tests
-export IMPALA_FE_TEST_COVERAGE_DIR=${IMPALA_FE_TEST_LOGS_DIR}/coverage
-export IMPALA_BE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/be_tests
-export IMPALA_EE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/ee_tests
-export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/custom_cluster_tests
+export IMPALA_LOGS_DIR="${IMPALA_HOME}/logs"
+export IMPALA_CLUSTER_LOGS_DIR="${IMPALA_LOGS_DIR}/cluster"
+export IMPALA_DATA_LOADING_LOGS_DIR="${IMPALA_LOGS_DIR}/data_loading"
+export IMPALA_DATA_LOADING_SQL_DIR="${IMPALA_DATA_LOADING_LOGS_DIR}/sql"
+export IMPALA_FE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/fe_tests"
+export IMPALA_FE_TEST_COVERAGE_DIR="${IMPALA_FE_TEST_LOGS_DIR}/coverage"
+export IMPALA_BE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/be_tests"
+export IMPALA_EE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/ee_tests"
+export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/custom_cluster_tests"
# List of all Impala log dirs and create them.
export IMPALA_ALL_LOGS_DIRS="${IMPALA_CLUSTER_LOGS_DIR}
${IMPALA_DATA_LOADING_LOGS_DIR} ${IMPALA_DATA_LOADING_SQL_DIR}
@@ -229,21 +229,21 @@ mkdir -p $IMPALA_ALL_LOGS_DIRS
# hardwired in its code, so we cannot change the output dir by configuration.
# We create two symlinks to capture the logs when running ctest either from
# ${IMPALA_HOME} or ${IMPALA_HOME}/be.
-rm -rf ${IMPALA_HOME}/Testing
-mkdir -p ${IMPALA_HOME}/Testing
-ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/Testing/Temporary
-rm -rf ${IMPALA_HOME}/be/Testing
-mkdir -p ${IMPALA_HOME}/be/Testing
-ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/be/Testing/Temporary
+rm -rf "${IMPALA_HOME}/Testing"
+mkdir -p "${IMPALA_HOME}/Testing"
+ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
+rm -rf "${IMPALA_HOME}/be/Testing"
+mkdir -p "${IMPALA_HOME}/be/Testing"
+ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
# Reduce the concurrency for local tests to half the number of cores in the system.
# Note than nproc may not be available on older distributions (centos5.5)
if type nproc >/dev/null 2>&1; then
CORES=$(($(nproc) / 2))
else
- CORES='4'
+ CORES=4
fi
-export NUM_CONCURRENT_TESTS=${NUM_CONCURRENT_TESTS-${CORES}}
+export NUM_CONCURRENT_TESTS="${NUM_CONCURRENT_TESTS-${CORES}}"
# Versions of toolchain dependencies (or if toolchain is not used of dependencies in
# thirdparty)
@@ -280,8 +280,8 @@ export IMPALA_THRIFT_VERSION=0.9.0-p8
export IMPALA_THRIFT_JAVA_VERSION=0.9.0
export IMPALA_ZLIB_VERSION=1.2.8
-export KUDU_MASTER=${KUDU_MASTER:-"127.0.0.1"}
-export KUDU_MASTER_PORT=${KUDU_MASTER_PORT:-"7051"}
+export KUDU_MASTER="${KUDU_MASTER:-127.0.0.1}"
+export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}"
# TODO: Figure out a way to use a snapshot version without causing a lot of breakage due
# to nightly changes from Kudu. The version below is the last released version but
# before release this needs to be updated to the version about to be released.
@@ -303,83 +303,85 @@ export IMPALA_LLAMA_VERSION=1.0.0-cdh5.10.0-SNAPSHOT
export IMPALA_PARQUET_VERSION=1.5.0-cdh5.10.0-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
-export IMPALA_FE_DIR=$IMPALA_HOME/fe
-export IMPALA_BE_DIR=$IMPALA_HOME/be
-export IMPALA_WORKLOAD_DIR=$IMPALA_HOME/testdata/workloads
-export IMPALA_AUX_WORKLOAD_DIR=$IMPALA_AUX_TEST_HOME/testdata/workloads
-export IMPALA_DATASET_DIR=$IMPALA_HOME/testdata/datasets
-export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets
-export IMPALA_COMMON_DIR=$IMPALA_HOME/common
-export PATH=$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH
+export IMPALA_FE_DIR="$IMPALA_HOME/fe"
+export IMPALA_BE_DIR="$IMPALA_HOME/be"
+export IMPALA_WORKLOAD_DIR="$IMPALA_HOME/testdata/workloads"
+export IMPALA_AUX_WORKLOAD_DIR="$IMPALA_AUX_TEST_HOME/testdata/workloads"
+export IMPALA_DATASET_DIR="$IMPALA_HOME/testdata/datasets"
+export IMPALA_AUX_DATASET_DIR="$IMPALA_AUX_TEST_HOME/testdata/datasets"
+export IMPALA_COMMON_DIR="$IMPALA_HOME/common"
+export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH"
# The directory in which all the thirdparty CDH components live.
if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then
- export CDH_COMPONENTS_HOME=$IMPALA_TOOLCHAIN/cdh_components
+ export CDH_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdh_components"
else
- export CDH_COMPONENTS_HOME=$IMPALA_HOME/thirdparty
+ export CDH_COMPONENTS_HOME="$IMPALA_HOME/thirdparty"
fi
# Hadoop dependencies are snapshots in the Impala tree
-export HADOOP_HOME=$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/
-export HADOOP_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
+export HADOOP_HOME="$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/"
+export HADOOP_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
: ${HADOOP_CLASSPATH=}
-export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:"${HADOOP_HOME}/share/hadoop/tools/lib/*"
+# Please note that the * is inside quotes, thus it won't get exanded by bash but
+# by java, see "Understanding class path wildcards" at http://goo.gl/f0cfft
+export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HADOOP_HOME}/share/hadoop/tools/lib/*"
# YARN is configured to use LZO so the LZO jar needs to be in the hadoop classpath.
export LZO_JAR_PATH="$HADOOP_LZO/build/hadoop-lzo-0.4.15.jar"
HADOOP_CLASSPATH+=":$LZO_JAR_PATH"
-export MINI_DFS_BASE_DATA_DIR=$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data
-export PATH=$HADOOP_HOME/bin:$PATH
+export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data"
+export PATH="$HADOOP_HOME/bin:$PATH"
-export LLAMA_HOME=$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/
-export MINIKDC_HOME=$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}
-export SENTRY_HOME=$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}
-export SENTRY_CONF_DIR=$IMPALA_HOME/fe/src/test/resources
+export LLAMA_HOME="$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/"
+export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}"
+export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}"
+export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources"
-export HIVE_HOME=$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/
-export PATH=$HIVE_HOME/bin:$PATH
-export HIVE_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
+export HIVE_HOME="$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/"
+export PATH="$HIVE_HOME/bin:$PATH"
+export HIVE_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
# Hive looks for jar files in a single directory from HIVE_AUX_JARS_PATH plus
# any jars in AUX_CLASSPATH. (Or a list of jars in HIVE_AUX_JARS_PATH.)
# The Postgres JDBC driver is downloaded by maven when building the frontend.
# Export the location of Postgres JDBC driver so Sentry can pick it up.
-export POSTGRES_JDBC_DRIVER=${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar
+export POSTGRES_JDBC_DRIVER="${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar"
export HIVE_AUX_JARS_PATH="$POSTGRES_JDBC_DRIVER"
export AUX_CLASSPATH="${LZO_JAR_PATH}"
### Tell hive not to use jline
export HADOOP_USER_CLASSPATH_FIRST=true
-export HBASE_HOME=$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/
-export PATH=$HBASE_HOME/bin:$PATH
+export HBASE_HOME="$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/"
+export PATH="$HBASE_HOME/bin:$PATH"
# Add the jars so hive can create hbase tables.
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar"
-export HBASE_CONF_DIR=$HIVE_CONF_DIR
+export HBASE_CONF_DIR="$HIVE_CONF_DIR"
# Set $THRIFT_HOME to the Thrift directory in toolchain.
-export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}
+export THRIFT_HOME="${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}"
# ASAN needs a matching version of llvm-symbolizer to symbolize stack traces.
-export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer
+export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer"
-export CLUSTER_DIR=${IMPALA_HOME}/testdata/cluster
+export CLUSTER_DIR="${IMPALA_HOME}/testdata/cluster"
-: ${IMPALA_BUILD_THREADS:=$(nproc)}
+: ${IMPALA_BUILD_THREADS:="$(nproc)"}
export IMPALA_BUILD_THREADS
# Some environments (like the packaging build) might not have $USER set. Fix that here.
-export USER=${USER-`id -un`}
+export USER="${USER-`id -un`}"
# Configure python path
-. $IMPALA_HOME/bin/set-pythonpath.sh
+. "$IMPALA_HOME/bin/set-pythonpath.sh"
# These arguments are, despite the name, passed to every JVM created
# by an impalad.
@@ -396,17 +398,17 @@ LIBHDFS_OPTS="${LIBHDFS_OPTS} -Djava.library.path=${HADOOP_HOME}/lib/native/"
# the build type.
export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service"
-export ARTISTIC_STYLE_OPTIONS=$IMPALA_BE_DIR/.astylerc
+export ARTISTIC_STYLE_OPTIONS="$IMPALA_BE_DIR/.astylerc"
-export IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib
+export IMPALA_SNAPPY_PATH="${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib"
-export JAVA_LIBRARY_PATH=${IMPALA_SNAPPY_PATH}
+export JAVA_LIBRARY_PATH="${IMPALA_SNAPPY_PATH}"
# So that the frontend tests and PlanService can pick up libbackend.so
# and other required libraries
-LIB_JAVA=`find ${JAVA_HOME}/ -name libjava.so | head -1`
-LIB_JSIG=`find ${JAVA_HOME}/ -name libjsig.so | head -1`
-LIB_JVM=` find ${JAVA_HOME}/ -name libjvm.so | head -1`
+LIB_JAVA=`find "${JAVA_HOME}/" -name libjava.so | head -1`
+LIB_JSIG=`find "${JAVA_HOME}/" -name libjsig.so | head -1`
+LIB_JVM=` find "${JAVA_HOME}/" -name libjvm.so | head -1`
LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`"
@@ -416,7 +418,7 @@ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_LZO}/build"
if [ $USE_SYSTEM_GCC -eq 0 ]; then
- IMPALA_TOOLCHAIN_GCC_LIB=${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64
+ IMPALA_TOOLCHAIN_GCC_LIB="${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}"
fi
@@ -425,10 +427,10 @@ LD_PRELOAD="${LD_PRELOAD-}"
export LD_PRELOAD="${LD_PRELOAD}:${LIB_JSIG}"
CLASSPATH="${CLASSPATH-}"
-CLASSPATH=$IMPALA_FE_DIR/target/dependency:$CLASSPATH
-CLASSPATH=$IMPALA_FE_DIR/target/classes:$CLASSPATH
-CLASSPATH=$IMPALA_FE_DIR/src/test/resources:$CLASSPATH
-CLASSPATH=$LZO_JAR_PATH:$CLASSPATH
+CLASSPATH="$IMPALA_FE_DIR/target/dependency:$CLASSPATH"
+CLASSPATH="$IMPALA_FE_DIR/target/classes:$CLASSPATH"
+CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH"
+CLASSPATH="$LZO_JAR_PATH:$CLASSPATH"
export CLASSPATH
# Setup aliases
@@ -465,9 +467,9 @@ echo "DOWNLOAD_CDH_COMPONENTS = $DOWNLOAD_CDH_COMPONENTS"
# work. Note that if impala-config.sh is sourced before the
# kerberized cluster is created, it will have to be sourced again
# *after* the cluster is created in order to pick up these settings.
-export MINIKDC_ENV=${IMPALA_HOME}/testdata/bin/minikdc_env.sh
-if ${CLUSTER_DIR}/admin is_kerberized; then
- . ${MINIKDC_ENV}
+export MINIKDC_ENV="${IMPALA_HOME}/testdata/bin/minikdc_env.sh"
+if "${CLUSTER_DIR}/admin" is_kerberized; then
+ . "${MINIKDC_ENV}"
echo " *** This cluster is kerberized ***"
echo "KRB5_KTNAME = $KRB5_KTNAME"
echo "KRB5_CONFIG = $KRB5_CONFIG"
@@ -478,6 +480,6 @@ else
# If the cluster *isn't* kerberized, ensure that the environment isn't
# polluted with kerberos items that might screw us up. We go through
# everything set in the minikdc environment and explicitly unset it.
- unset `grep export ${MINIKDC_ENV} | sed "s/.*export \([^=]*\)=.*/\1/" \
+ unset `grep export "${MINIKDC_ENV}" | sed "s/.*export \([^=]*\)=.*/\1/" \
| sort | uniq`
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python
----------------------------------------------------------------------
diff --git a/bin/impala-python b/bin/impala-python
index 5ff4190..29f9406 100755
--- a/bin/impala-python
+++ b/bin/impala-python
@@ -1,3 +1,3 @@
#!/bin/bash
-source $(dirname "$0")/impala-python-common.sh
+source "$(dirname "$0")/impala-python-common.sh"
exec "$PY_DIR/env/bin/python" "$@"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python-common.sh
----------------------------------------------------------------------
diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh
index 2084429..29a36a2 100644
--- a/bin/impala-python-common.sh
+++ b/bin/impala-python-common.sh
@@ -21,8 +21,8 @@
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
-LD_LIBRARY_PATH+=":$(python $IMPALA_HOME/infra/python/bootstrap_virtualenv.py \
+LD_LIBRARY_PATH+=":$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
--print-ld-library-path)"
-PY_DIR=$(dirname "$0")/../infra/python
+PY_DIR="$(dirname "$0")/../infra/python"
python "$PY_DIR/bootstrap_virtualenv.py"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/run-all-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh
index 4322efc..7d68216 100755
--- a/bin/run-all-tests.sh
+++ b/bin/run-all-tests.sh
@@ -24,7 +24,7 @@
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
-. $IMPALA_HOME/bin/set-pythonpath.sh
+. "$IMPALA_HOME/bin/set-pythonpath.sh"
# Allow picking up strategy from environment
: ${EXPLORATION_STRATEGY:=core}
@@ -32,9 +32,9 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
: ${MAX_PYTEST_FAILURES:=10}
KERB_ARGS=""
-. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
-. ${IMPALA_HOME}/testdata/bin/run-step.sh
-if ${CLUSTER_DIR}/admin is_kerberized; then
+. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1
+. "${IMPALA_HOME}/testdata/bin/run-step.sh"
+if "${CLUSTER_DIR}/admin" is_kerberized; then
KERB_ARGS="--use_kerberos"
fi
@@ -69,10 +69,10 @@ while getopts "e:n:c" OPTION
do
case "$OPTION" in
e)
- EXPLORATION_STRATEGY=$OPTARG
+ EXPLORATION_STRATEGY="$OPTARG"
;;
n)
- NUM_TEST_ITERATIONS=$OPTARG
+ NUM_TEST_ITERATIONS="$OPTARG"
;;
c)
CODE_COVERAGE=true
@@ -98,7 +98,7 @@ if [[ "${TARGET_FILESYSTEM}" == "local" ]]; then
fi
# For logging when using run-step.
-LOG_DIR=${IMPALA_EE_TEST_LOGS_DIR}
+LOG_DIR="${IMPALA_EE_TEST_LOGS_DIR}"
# Enable core dumps
ulimit -c unlimited
@@ -108,7 +108,7 @@ if [[ "${TARGET_FILESYSTEM}" == "hdfs" ]]; then
# script. Restarting HBase will change the region server assignment. Run split-hbase.sh
# before running any test.
run-step "Split and assign HBase regions" split-hbase.log \
- ${IMPALA_HOME}/testdata/bin/split-hbase.sh
+ "${IMPALA_HOME}/testdata/bin/split-hbase.sh"
fi
for i in $(seq 1 $NUM_TEST_ITERATIONS)
@@ -116,7 +116,7 @@ do
TEST_RET_CODE=0
run-step "Starting Impala cluster" start-impala-cluster.log \
- ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \
+ "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \
${TEST_START_CLUSTER_ARGS}
if [[ "$BE_TEST" == true ]]; then
@@ -127,14 +127,14 @@ do
export SKIP_BE_TEST_PATTERN="session*"
fi
# Run backend tests.
- if ! ${IMPALA_HOME}/bin/run-backend-tests.sh; then
+ if ! "${IMPALA_HOME}/bin/run-backend-tests.sh"; then
TEST_RET_CODE=1
fi
fi
# Run some queries using run-workload to verify run-workload has not been broken.
if ! run-step "Run test run-workload" test-run-workload.log \
- ${IMPALA_HOME}/bin/run-workload.py -w tpch --num_clients=2 --query_names=TPCH-Q1 \
+ "${IMPALA_HOME}/bin/run-workload.py" -w tpch --num_clients=2 --query_names=TPCH-Q1 \
--table_format=text/none --exec_options="disable_codegen:False" ${KERB_ARGS}; then
TEST_RET_CODE=1
fi
@@ -143,7 +143,7 @@ do
# Run JUnit frontend tests
# Requires a running impalad cluster because some tests (such as DataErrorTest and
# JdbcTest) queries against an impala cluster.
- pushd ${IMPALA_FE_DIR}
+ pushd "${IMPALA_FE_DIR}"
MVN_ARGS=""
if [[ "${TARGET_FILESYSTEM}" == "s3" ]]; then
# When running against S3, only run the S3 frontend tests.
@@ -152,7 +152,7 @@ do
if [[ "$CODE_COVERAGE" == true ]]; then
MVN_ARGS+="-DcodeCoverage"
fi
- if ! ${IMPALA_HOME}/bin/mvn-quiet.sh -fae test ${MVN_ARGS}; then
+ if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" -fae test ${MVN_ARGS}; then
TEST_RET_CODE=1
fi
popd
@@ -161,7 +161,7 @@ do
if [[ "$EE_TEST" == true ]]; then
# Run end-to-end tests.
# KERBEROS TODO - this will need to deal with ${KERB_ARGS}
- if ! ${IMPALA_HOME}/tests/run-tests.py ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then
+ if ! "${IMPALA_HOME}/tests/run-tests.py" ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then
#${KERB_ARGS};
TEST_RET_CODE=1
fi
@@ -170,11 +170,11 @@ do
if [[ "$JDBC_TEST" == true ]]; then
# Run the JDBC tests with background loading disabled. This is interesting because
# it requires loading missing table metadata.
- ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \
+ "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \
--catalogd_args=--load_catalog_in_background=false \
${TEST_START_CLUSTER_ARGS}
- pushd ${IMPALA_FE_DIR}
- if ! ${IMPALA_HOME}/bin/mvn-quiet.sh test -Dtest=JdbcTest; then
+ pushd "${IMPALA_FE_DIR}"
+ if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" test -Dtest=JdbcTest; then
TEST_RET_CODE=1
fi
popd
@@ -186,17 +186,17 @@ do
# pollute the directory with too many files, remove what was there
# before. Also, save the IMPALA_MAX_LOG_FILES value for re-set
# later.
- rm -rf ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}
- mkdir -p ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}
- IMPALA_MAX_LOG_FILES_SAVE=${IMPALA_MAX_LOG_FILES:-10}
+ rm -rf "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
+ mkdir -p "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
+ IMPALA_MAX_LOG_FILES_SAVE="${IMPALA_MAX_LOG_FILES:-10}"
export IMPALA_MAX_LOG_FILES=0
# Run the custom-cluster tests after all other tests, since they will restart the
# cluster repeatedly and lose state.
# TODO: Consider moving in to run-tests.py.
- if ! ${IMPALA_HOME}/tests/run-custom-cluster-tests.sh ${COMMON_PYTEST_ARGS}; then
+ if ! "${IMPALA_HOME}/tests/run-custom-cluster-tests.sh" ${COMMON_PYTEST_ARGS}; then
TEST_RET_CODE=1
fi
- export IMPALA_MAX_LOG_FILES=${IMPALA_MAX_LOG_FILES_SAVE}
+ export IMPALA_MAX_LOG_FILES="${IMPALA_MAX_LOG_FILES_SAVE}"
fi
# Finally, run the process failure tests.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/set-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh
old mode 100644
new mode 100755
index 873f6ec..68aec2b
--- a/bin/set-classpath.sh
+++ b/bin/set-classpath.sh
@@ -23,20 +23,24 @@
# explicitly to the CLASSPATH.
CLASSPATH=\
-$IMPALA_HOME/fe/src/test/resources:\
-$IMPALA_HOME/fe/target/classes:\
-$IMPALA_HOME/fe/target/dependency:\
-$IMPALA_HOME/fe/target/test-classes:\
-${HIVE_HOME}/lib/datanucleus-api-jdo-3.2.1.jar:\
-${HIVE_HOME}/lib/datanucleus-core-3.2.2.jar:\
-${HIVE_HOME}/lib/datanucleus-rdbms-3.2.1.jar:
+"$IMPALA_HOME"/fe/src/test/resources:\
+"$IMPALA_HOME"/fe/target/classes:\
+"$IMPALA_HOME"/fe/target/dependency:\
+"$IMPALA_HOME"/fe/target/test-classes:\
+"${HIVE_HOME}"/lib/datanucleus-api-jdo-3.2.1.jar:\
+"${HIVE_HOME}"/lib/datanucleus-core-3.2.2.jar:\
+"${HIVE_HOME}"/lib/datanucleus-rdbms-3.2.1.jar:
-for jar in `ls ${IMPALA_HOME}/fe/target/dependency/*.jar`; do
- CLASSPATH=${CLASSPATH}:$jar
+for jar in "${IMPALA_HOME}"/fe/target/dependency/*.jar; do
+ if [ -e "$jar" ] ; then
+ CLASSPATH="${CLASSPATH}:$jar"
+ fi
done
-for jar in `ls ${IMPALA_HOME}/testdata/target/dependency/*.jar`; do
- CLASSPATH=${CLASSPATH}:$jar
+for jar in "${IMPALA_HOME}"/testdata/target/dependency/*.jar; do
+ if [ -e "$jar" ] ; then
+ CLASSPATH="${CLASSPATH}:$jar"
+ fi
done
export CLASSPATH
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index ec79420..1845081 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -23,7 +23,7 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
# run buildall.sh -help to see options
ROOT=`dirname "$0"`
-ROOT=`cd "$ROOT"; pwd`
+ROOT=`cd "$ROOT" >/dev/null; pwd`
# Grab this *before* we source impala-config.sh to see if the caller has
# kerberized environment variables already or not.
@@ -33,7 +33,7 @@ if [ ! -z "${MINIKDC_REALM}" ]; then
NEEDS_RE_SOURCE_NOTE=0
fi
-export IMPALA_HOME=$ROOT
+export IMPALA_HOME="$ROOT"
if ! . "$ROOT"/bin/impala-config.sh; then
echo "Bad configuration, aborting buildall."
exit 1
@@ -112,25 +112,25 @@ do
EXPLORATION_STRATEGY=exhaustive
;;
-snapshot_file)
- SNAPSHOT_FILE=${2-}
- if [ ! -f $SNAPSHOT_FILE ]; then
+ SNAPSHOT_FILE="${2-}"
+ if [ ! -f "$SNAPSHOT_FILE" ]; then
echo "-snapshot_file does not exist: $SNAPSHOT_FILE"
exit 1;
fi
TESTDATA_ACTION=1
# Get the full path.
- SNAPSHOT_FILE=$(readlink -f $SNAPSHOT_FILE)
+ SNAPSHOT_FILE="$(readlink -f "$SNAPSHOT_FILE")"
shift;
;;
-metastore_snapshot_file)
- METASTORE_SNAPSHOT_FILE=${2-}
- if [ ! -f $METASTORE_SNAPSHOT_FILE ]; then
+ METASTORE_SNAPSHOT_FILE="${2-}"
+ if [ ! -f "$METASTORE_SNAPSHOT_FILE" ]; then
echo "-metastore_snapshot_file does not exist: $METASTORE_SNAPSHOT_FILE"
exit 1;
fi
TESTDATA_ACTION=1
# Get the full path.
- METASTORE_SNAPSHOT_FILE=$(readlink -f $METASTORE_SNAPSHOT_FILE)
+ METASTORE_SNAPSHOT_FILE="$(readlink -f "$METASTORE_SNAPSHOT_FILE")"
shift;
;;
-k|-kerberize|-kerberos|-kerb)
@@ -236,16 +236,16 @@ if [ ${IMPALA_KERBERIZE} -eq 0 ]; then
fi
# Loading data on a filesystem other than fs.defaultFS is not supported.
-if [[ -z $METASTORE_SNAPSHOT_FILE && "${TARGET_FILESYSTEM}" != "hdfs" &&
- $TESTDATA_ACTION -eq 1 ]]; then
+if [[ -z "$METASTORE_SNAPSHOT_FILE" && "${TARGET_FILESYSTEM}" != "hdfs" &&
+ "$TESTDATA_ACTION" -eq 1 ]]; then
echo "The metastore snapshot is required for loading data into ${TARGET_FILESYSTEM}"
echo "Use the -metastore_snapshot_file command line paramater."
exit 1
fi
# option to clean everything first
-if [ $CLEAN_ACTION -eq 1 ]; then
- $IMPALA_HOME/bin/clean.sh
+if [ "$CLEAN_ACTION" -eq 1 ]; then
+ "$IMPALA_HOME/bin/clean.sh"
fi
# Populate necessary thirdparty components unless it's set to be skipped.
@@ -256,7 +256,7 @@ else
echo "Downloading Python dependencies"
# Download all the Python dependencies we need before doing anything
# of substance. Does not re-download anything that is already present.
- if ! $IMPALA_HOME/infra/python/deps/download_requirements; then
+ if ! "$IMPALA_HOME/infra/python/deps/download_requirements"; then
echo "Warning: Unable to download Python requirements."
echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail."
else
@@ -264,88 +264,88 @@ else
fi
echo "Downloading and extracting toolchain dependencies."
- $IMPALA_HOME/bin/bootstrap_toolchain.py
+ "$IMPALA_HOME/bin/bootstrap_toolchain.py"
echo "Toolchain bootstrap complete."
fi
MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}"
-if [ $BUILD_FE_ONLY -eq 1 ]; then
- $IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} -cmake_only
- ${MAKE_CMD} fe
+if [ "$BUILD_FE_ONLY" -eq 1 ]; then
+ "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only
+ "${MAKE_CMD}" fe
exit 0
fi
-if [ -e $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.so ]
+if [ -e "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.so ]
then
- cp $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.* $HADOOP_HOME/lib/native
+ cp "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* "$HADOOP_HOME/lib/native"
else
echo "No hadoop-lzo found"
fi
# Stop any running Impala services.
-${IMPALA_HOME}/bin/start-impala-cluster.py --kill --force
+"${IMPALA_HOME}/bin/start-impala-cluster.py" --kill --force
-if [[ $CLEAN_ACTION -eq 1 || $FORMAT_METASTORE -eq 1 || $FORMAT_CLUSTER -eq 1 ||
- $FORMAT_SENTRY_POLICY_DB -eq 1 || -n $METASTORE_SNAPSHOT_FILE ]]
+if [[ "$CLEAN_ACTION" -eq 1 || "$FORMAT_METASTORE" -eq 1 || "$FORMAT_CLUSTER" -eq 1 ||
+ "$FORMAT_SENTRY_POLICY_DB" -eq 1 || -n "$METASTORE_SNAPSHOT_FILE" ]]
then
# Kill any processes that may be accessing postgres metastore. To be safe, this is done
# before we make any changes to the config files.
set +e
- ${IMPALA_HOME}/testdata/bin/kill-all.sh
+ "${IMPALA_HOME}/testdata/bin/kill-all.sh"
set -e
fi
CREATE_TEST_CONFIG_ARGS=""
-if [[ $FORMAT_SENTRY_POLICY_DB -eq 1 ]]; then
+if [[ "$FORMAT_SENTRY_POLICY_DB" -eq 1 ]]; then
CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db"
fi
-if [[ $FORMAT_METASTORE -eq 1 && -z $METASTORE_SNAPSHOT_FILE ]]; then
+if [[ "$FORMAT_METASTORE" -eq 1 && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_TEST_CONFIG_ARGS+=" -create_metastore"
fi
# Generate the Hadoop configs needed by Impala
-${IMPALA_HOME}/bin/create-test-configuration.sh ${CREATE_TEST_CONFIG_ARGS}
+"${IMPALA_HOME}/bin/create-test-configuration.sh" ${CREATE_TEST_CONFIG_ARGS}
# If a metastore snapshot exists, load it.
-if [ $METASTORE_SNAPSHOT_FILE ]; then
+if [ "$METASTORE_SNAPSHOT_FILE" ]; then
echo "Loading metastore snapshot"
- ${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh $METASTORE_SNAPSHOT_FILE
+ "${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh" "$METASTORE_SNAPSHOT_FILE"
fi
# build common and backend
echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
-$IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS}
+"$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
-if [ -e $IMPALA_LZO ]
+if [ -e "$IMPALA_LZO" ]
then
- pushd $IMPALA_LZO
+ pushd "$IMPALA_LZO"
LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
rm -f CMakeCache.txt
cmake ${LZO_CMAKE_ARGS}
- ${MAKE_CMD}
+ "${MAKE_CMD}"
popd
fi
# build the external data source API
-pushd ${IMPALA_HOME}/ext-data-source
-${IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
+pushd "${IMPALA_HOME}/ext-data-source"
+"${IMPALA_HOME}/bin/mvn-quiet.sh" install -DskipTests
popd
# build frontend and copy dependencies
-pushd ${IMPALA_FE_DIR}
-${IMPALA_HOME}/bin/mvn-quiet.sh package -DskipTests
+pushd "${IMPALA_FE_DIR}"
+"${IMPALA_HOME}/bin/mvn-quiet.sh" package -DskipTests
popd
# Build the shell tarball
echo "Creating shell tarball"
-${IMPALA_HOME}/shell/make_shell_tarball.sh
+"${IMPALA_HOME}/shell/make_shell_tarball.sh"
-if [ $FORMAT_CLUSTER -eq 1 ]; then
- $IMPALA_HOME/testdata/bin/run-all.sh -format
-elif [ $TESTDATA_ACTION -eq 1 ] || [ $TESTS_ACTION -eq 1 ]; then
- $IMPALA_HOME/testdata/bin/run-all.sh
+if [ "$FORMAT_CLUSTER" -eq 1 ]; then
+ "$IMPALA_HOME/testdata/bin/run-all.sh" -format
+elif [ "$TESTDATA_ACTION" -eq 1 ] || [ "$TESTS_ACTION "-eq 1 ]; then
+ "$IMPALA_HOME/testdata/bin/run-all.sh"
fi
#
@@ -368,7 +368,7 @@ if [ ${IMPALA_KERBERIZE} -eq 1 ]; then
echo "won't be run. The impala daemons will be started."
TESTDATA_ACTION=0
TESTS_ACTION=0
- ${IMPALA_HOME}/bin/start-impala-cluster.py
+ "${IMPALA_HOME}/bin/start-impala-cluster.py"
fi
fi
# END KERBEROS TODO
@@ -389,34 +389,34 @@ fi
if [ $TESTDATA_ACTION -eq 1 ]; then
# Create testdata.
- $IMPALA_HOME/bin/create_testdata.sh
- cd $ROOT
+ "$IMPALA_HOME/bin/create_testdata.sh"
+ cd "$ROOT"
# We have 4 cases:
# - test-warehouse and metastore snapshots exists.
# - Only the test-warehouse snapshot exists.
# - Only the metastore snapshot exists.
# - Neither of them exist.
CREATE_LOAD_DATA_ARGS=""
- if [[ $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then
+ if [[ "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE} -skip_metadata_load"
- elif [[ $SNAPSHOT_FILE && -z $METASTORE_SNAPSHOT_FILE ]]; then
+ elif [[ "$SNAPSHOT_FILE" && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE}"
- elif [[ -z $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then
+ elif [[ -z "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-skip_metadata_load -skip_snapshot_load"
fi
- ${IMPALA_HOME}/testdata/bin/create-load-data.sh ${CREATE_LOAD_DATA_ARGS} <<< Y
+ "${IMPALA_HOME}/testdata/bin/create-load-data.sh" ${CREATE_LOAD_DATA_ARGS} <<< Y
fi
if [ $TESTS_ACTION -eq 1 ]; then
if [ $CODE_COVERAGE -eq 0 ]; then
- ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY
+ "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY
else
- ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY -c
+ "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY -c
fi
fi
# Generate list of files for Cscope to index
-$IMPALA_HOME/bin/gen-cscope.sh
+"$IMPALA_HOME/bin/gen-cscope.sh"
if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then
echo
@@ -426,6 +426,6 @@ if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then
echo "environment variables weren't available before the cluster"
echo "was created. To pick them up, please source impala-config.sh:"
echo
- echo " . ${IMPALA_HOME}/bin/impala-config.sh"
+ echo " . \"${IMPALA_HOME}/bin/impala-config.sh\""
echo
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/infra/python/deps/download_requirements
----------------------------------------------------------------------
diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements
index e054653..26788be 100755
--- a/infra/python/deps/download_requirements
+++ b/infra/python/deps/download_requirements
@@ -2,15 +2,15 @@
set -euo pipefail
-DIR=$(dirname "$0")
+DIR="$(dirname "$0")"
-pushd $DIR
-PY26=$(./find_py26.py)
+pushd "$DIR"
+PY26="$(./find_py26.py)"
# Directly download packages listed in requirements.txt, but don't install them.
-$PY26 pip_download.py
+"$PY26" pip_download.py
# For virtualenv, other scripts rely on the .tar.gz package (not a .whl package).
-$PY26 pip_download.py virtualenv 13.1.0
+"$PY26" pip_download.py virtualenv 13.1.0
# kudu-python is downloaded separately because pip install attempts to execute a
# setup.py subcommand for kudu-python that can fail even if the download succeeds.
-$PY26 pip_download.py kudu-python 0.1.1
+"$PY26" pip_download.py kudu-python 0.1.1
popd