You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/09/01 21:36:40 UTC
[1/3] incubator-impala git commit: IMPALA-4020: Handle external
conflicting changes to HMS gracefully
Repository: incubator-impala
Updated Branches:
refs/heads/master 24869d40f -> a60ba6d27
IMPALA-4020: Handle external conflicting changes to HMS gracefully
Currently Catalog can't handle conflicting changes to HMS' databases
from external clients while running invalidate metadata operation.
For example, if a database is dropped by a client external to Impala,
while the invalidate metadata is in process, Catalog aborts the
metadata load. This commit fixes this issue by handling appropriate
exceptions when HMS operations fail and only ignores the load for that
particular database.
Change-Id: Ic228efbcceb9ef6c165d0d9aeef7202581e3e46a
Reviewed-on: http://gerrit.cloudera.org:8080/4161
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Reviewed-by: Marcel Kornacker <ma...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/5f3ee38b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/5f3ee38b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/5f3ee38b
Branch: refs/heads/master
Commit: 5f3ee38bf4106fe4beb24e4f5237047decead3fe
Parents: 24869d4
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Mon Aug 29 12:13:37 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 1 04:15:04 2016 +0000
----------------------------------------------------------------------
.../impala/catalog/CatalogServiceCatalog.java | 92 +++++++++++++-------
1 file changed, 61 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5f3ee38b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
index f0993bb..e8f5577 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java
@@ -536,6 +536,60 @@ public class CatalogServiceCatalog extends Catalog {
}
}
}
+
+ /**
+ * Invalidates the database 'db'. This method can have potential race
+ * conditions with external changes to the Hive metastore and hence any
+ * conflicting changes to the objects can manifest in the form of exceptions
+ * from the HMS calls which are appropriately handled. Returns the invalidated
+ * 'Db' object along with list of tables to be loaded by the TableLoadingMgr.
+ * Returns null if the method encounters an exception during invalidation.
+ */
+ private Pair<Db, List<TTableName>> invalidateDb(
+ MetaStoreClient msClient, String dbName, Db existingDb) {
+ try {
+ List<org.apache.hadoop.hive.metastore.api.Function> javaFns =
+ Lists.newArrayList();
+ for (String javaFn: msClient.getHiveClient().getFunctions(dbName, "*")) {
+ javaFns.add(msClient.getHiveClient().getFunction(dbName, javaFn));
+ }
+ org.apache.hadoop.hive.metastore.api.Database msDb =
+ msClient.getHiveClient().getDatabase(dbName);
+ Db newDb = new Db(dbName, this, msDb);
+ // existingDb is usually null when the Catalog loads for the first time.
+ // In that case we needn't restore any transient functions.
+ if (existingDb != null) {
+ // Restore UDFs that aren't persisted. They are only cleaned up on
+ // Catalog restart.
+ for (Function fn: existingDb.getTransientFunctions()) {
+ newDb.addFunction(fn);
+ fn.setCatalogVersion(incrementAndGetCatalogVersion());
+ }
+ }
+ // Reload native UDFs.
+ loadFunctionsFromDbParams(newDb, msDb);
+ // Reload Java UDFs from HMS.
+ loadJavaFunctions(newDb, javaFns);
+ newDb.setCatalogVersion(incrementAndGetCatalogVersion());
+
+ List<TTableName> tblsToBackgroundLoad = Lists.newArrayList();
+ for (String tableName: msClient.getHiveClient().getAllTables(dbName)) {
+ Table incompleteTbl = IncompleteTable.createUninitializedTable(
+ getNextTableId(), newDb, tableName);
+ incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion());
+ newDb.addTable(incompleteTbl);
+ if (loadInBackground_) {
+ tblsToBackgroundLoad.add(new TTableName(dbName, tableName.toLowerCase()));
+ }
+ }
+ return Pair.create(newDb, tblsToBackgroundLoad);
+ } catch (Exception e) {
+ LOG.warn("Encountered an exception while invalidating database: " + dbName +
+ ". Ignoring further load of this db.", e);
+ }
+ return null;
+ }
+
/**
* Resets this catalog instance by clearing all cached table and database metadata.
*/
@@ -565,37 +619,13 @@ public class CatalogServiceCatalog extends Catalog {
List<TTableName> tblsToBackgroundLoad = Lists.newArrayList();
try (MetaStoreClient msClient = getMetaStoreClient()) {
for (String dbName: msClient.getHiveClient().getAllDatabases()) {
- List<org.apache.hadoop.hive.metastore.api.Function> javaFns =
- Lists.newArrayList();
- for (String javaFn: msClient.getHiveClient().getFunctions(dbName, "*")) {
- javaFns.add(msClient.getHiveClient().getFunction(dbName, javaFn));
- }
- org.apache.hadoop.hive.metastore.api.Database msDb =
- msClient.getHiveClient().getDatabase(dbName);
- Db db = new Db(dbName, this, msDb);
- // Restore UDFs that aren't persisted.
- Db oldDb = oldDbCache.get(db.getName().toLowerCase());
- if (oldDb != null) {
- for (Function fn: oldDb.getTransientFunctions()) {
- db.addFunction(fn);
- fn.setCatalogVersion(incrementAndGetCatalogVersion());
- }
- }
- loadFunctionsFromDbParams(db, msDb);
- loadJavaFunctions(db, javaFns);
- db.setCatalogVersion(incrementAndGetCatalogVersion());
- newDbCache.put(db.getName().toLowerCase(), db);
-
- for (String tableName: msClient.getHiveClient().getAllTables(dbName)) {
- Table incompleteTbl = IncompleteTable.createUninitializedTable(
- getNextTableId(), db, tableName);
- incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion());
- db.addTable(incompleteTbl);
- if (loadInBackground_) {
- tblsToBackgroundLoad.add(
- new TTableName(dbName.toLowerCase(), tableName.toLowerCase()));
- }
- }
+ dbName = dbName.toLowerCase();
+ Db oldDb = oldDbCache.get(dbName);
+ Pair<Db, List<TTableName>> invalidatedDb = invalidateDb(msClient,
+ dbName, oldDb);
+ if (invalidatedDb == null) continue;
+ newDbCache.put(dbName, invalidatedDb.first);
+ tblsToBackgroundLoad.addAll(invalidatedDb.second);
}
}
dbCache_.set(newDbCache);
[3/3] incubator-impala git commit: IMPALA-4006: dangerous rm -rf
statements in scripts
Posted by ta...@apache.org.
IMPALA-4006: dangerous rm -rf statements in scripts
Quoted variable substitutions in rm -rf commands and in many other
places. This prevents disasters if those variables contain whitespace.
Redirected output of the cd commands to /dev/null. This prevents
polluting the target variable with the directory name when the CDPATH
environment variable is set.
Change-Id: I7503794180dee99eeb979e67f34e3b2edade70fe
Reviewed-on: http://gerrit.cloudera.org:8080/4078
Tested-by: Internal Jenkins
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a60ba6d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a60ba6d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a60ba6d2
Branch: refs/heads/master
Commit: a60ba6d2743eee5f2e5c5c3f1baa375ede0195e5
Parents: 2cd7524
Author: Zoltan Ivanfi <zi...@cloudera.com>
Authored: Mon Aug 22 17:29:44 2016 +0200
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Thu Sep 1 21:26:52 2016 +0000
----------------------------------------------------------------------
bin/clean.sh | 20 +--
bin/impala-config.sh | 200 ++++++++++++++-------------
bin/impala-python | 2 +-
bin/impala-python-common.sh | 4 +-
bin/run-all-tests.sh | 44 +++---
bin/set-classpath.sh | 26 ++--
buildall.sh | 104 +++++++-------
infra/python/deps/download_requirements | 12 +-
8 files changed, 209 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/clean.sh
----------------------------------------------------------------------
diff --git a/bin/clean.sh b/bin/clean.sh
index 00d02b3..9ac73cf 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -29,8 +29,8 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
make clean || :
# Stop the minikdc if needed.
-if ${CLUSTER_DIR}/admin is_kerberized; then
- ${IMPALA_HOME}/testdata/bin/minikdc.sh stop
+if "${CLUSTER_DIR}/admin" is_kerberized; then
+ "${IMPALA_HOME}/testdata/bin/minikdc.sh" stop
fi
# clean the external data source project
@@ -45,35 +45,35 @@ pushd $IMPALA_FE_DIR
rm -rf target
rm -f src/test/resources/{core,hbase,hive}-site.xml
rm -rf generated-sources/*
-rm -rf ${IMPALA_LOGS_DIR}/*
+[ -z "$IMPALA_LOGS_DIR" ] || rm -rf "${IMPALA_LOGS_DIR}"/*
mkdir -p $IMPALA_ALL_LOGS_DIRS
popd
# clean be
-pushd $IMPALA_HOME/be
+pushd "$IMPALA_HOME/be"
# remove everything listed in .gitignore
git clean -Xdfq
popd
# clean shell build artifacts
-pushd $IMPALA_HOME/shell
+pushd "$IMPALA_HOME/shell"
# remove everything listed in .gitignore
git clean -Xdfq
popd
# Clean stale .pyc, .pyo files and __pycache__ directories.
-pushd ${IMPALA_HOME}
+pushd "${IMPALA_HOME}"
find . -type f -name "*.py[co]" -delete
find . -type d -name "__pycache__" -delete
popd
# clean llvm
-rm -f $IMPALA_HOME/llvm-ir/impala*.ll
-rm -f $IMPALA_HOME/be/generated-sources/impala-ir/*
+rm -f "$IMPALA_HOME/llvm-ir/"impala*.ll
+rm -f "$IMPALA_HOME/be/generated-sources/impala-ir/"*
# Cleanup Impala-lzo
-if [ -e $IMPALA_LZO ]; then
- pushd $IMPALA_LZO; git clean -fdx .; popd
+if [ -e "$IMPALA_LZO" ]; then
+ pushd "$IMPALA_LZO"; git clean -fdx .; popd
fi
# When switching to and from toolchain, make sure to remove all CMake generated files
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f79d8f8..7f4df7f 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -34,16 +34,16 @@ if [[ ! -e "$JAVA" ]]; then
return 1
fi
-if [ -z $IMPALA_HOME ]; then
- if [[ ! -z $ZSH_NAME ]]; then
- export IMPALA_HOME=$(dirname $(cd $(dirname ${(%):-%x}) && pwd))
+if [ -z "$IMPALA_HOME" ]; then
+ if [[ ! -z "$ZSH_NAME" ]]; then
+ export IMPALA_HOME=$(dirname "$(cd $(dirname ${(%):-%x}) >/dev/null && pwd)")
else
- export IMPALA_HOME=$(dirname $(cd $(dirname "${BASH_SOURCE[0]}") && pwd))
+ export IMPALA_HOME=$(dirname "$(cd $(dirname "${BASH_SOURCE[0]}") >/dev/null && pwd)")
fi
fi
-: ${IMPALA_TOOLCHAIN=$IMPALA_HOME/toolchain}
-if [ -z $IMPALA_TOOLCHAIN ]; then
+: ${IMPALA_TOOLCHAIN="$IMPALA_HOME/toolchain"}
+if [ -z "$IMPALA_TOOLCHAIN" ]; then
echo "IMPALA_TOOLCHAIN must be specified. Please set it to a valid directory or"\
"leave it unset."
return 1
@@ -69,14 +69,14 @@ fi
# If enabled, debug symbols are added to cross-compiled IR.
: ${ENABLE_IMPALA_IR_DEBUG_INFO=false}
-if [ -d $IMPALA_HOME/thirdparty ]; then
+if [ -d "$IMPALA_HOME/thirdparty" ]; then
NO_THIRDPARTY=false
else
NO_THIRDPARTY=true
fi
# If true, download and use the CDH components from S3 instead of the ones
# in $IMPALA_HOME/thirdparty.
-: ${DOWNLOAD_CDH_COMPONENTS=$NO_THIRDPARTY}
+: ${DOWNLOAD_CDH_COMPONENTS="$NO_THIRDPARTY"}
export IMPALA_TOOLCHAIN
export SKIP_TOOLCHAIN_BOOTSTRAP
@@ -85,7 +85,7 @@ export USE_GOLD_LINKER
export IMPALA_CXX_COMPILER
export ENABLE_IMPALA_IR_DEBUG_INFO
export DOWNLOAD_CDH_COMPONENTS
-export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)
+export IS_OSX="$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false; fi)"
# To use a local build of Kudu, set KUDU_BUILD_DIR to the path Kudu was built in and
# set KUDU_CLIENT_DIR to the path KUDU was installed in.
@@ -101,11 +101,11 @@ export IS_OSX=$(if [[ "$OSTYPE" == "darwin"* ]]; then echo true; else echo false
: ${KUDU_CLIENT_DIR=}
export KUDU_BUILD_DIR
export KUDU_CLIENT_DIR
-if [[ -n $KUDU_BUILD_DIR && -z $KUDU_CLIENT_DIR ]]; then
+if [[ -n "$KUDU_BUILD_DIR" && -z "$KUDU_CLIENT_DIR" ]]; then
echo When KUDU_BUILD_DIR is set KUDU_CLIENT_DIR must also be set. 1>&2
return 1
fi
-if [[ -z $KUDU_BUILD_DIR && -n $KUDU_CLIENT_DIR ]]; then
+if [[ -z "$KUDU_BUILD_DIR" && -n "$KUDU_CLIENT_DIR" ]]; then
echo When KUDU_CLIENT_DIR is set KUDU_BUILD_DIR must also be set. 1>&2
return 1
fi
@@ -117,20 +117,20 @@ export USE_KUDU_DEBUG_BUILD
# into the backend. The frontend build is OS independent since it is Java.
if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then
KUDU_IS_SUPPORTED=true
- if [[ -z $KUDU_BUILD_DIR ]]; then
+ if [[ -z "$KUDU_BUILD_DIR" ]]; then
if ! $IS_OSX; then
if ! which lsb_release &>/dev/null; then
echo Unable to find the 'lsb_release' command. \
Please ensure it is available in your PATH. 1>&2
return 1
fi
- DISTRO_VERSION=$(lsb_release -sir 2>&1)
+ DISTRO_VERSION="$(lsb_release -sir 2>&1)"
if [[ $? -ne 0 ]]; then
echo lsb_release cammond failed, output was: "$DISTRO_VERSION" 1>&2
return 1
fi
# Remove spaces, trim minor versions, and convert to lowercase.
- DISTRO_VERSION=$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")
+ DISTRO_VERSION="$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")"
case "$DISTRO_VERSION" in
# "enterprise" is Oracle
centos5 | debian* | enterprise*5 | redhat*5 | suse* | ubuntu*12)
@@ -142,18 +142,18 @@ fi
export KUDU_IS_SUPPORTED
export CDH_MAJOR_VERSION=5
-export HADOOP_LZO=${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}
-export IMPALA_LZO=${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}
-export IMPALA_AUX_TEST_HOME=${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}
-export TARGET_FILESYSTEM=${TARGET_FILESYSTEM-"hdfs"}
-export FILESYSTEM_PREFIX=${FILESYSTEM_PREFIX-""}
-export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-"DummySecretAccessKey"}
-export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-"DummyAccessKeyId"}
-export S3_BUCKET=${S3_BUCKET-""}
-export HDFS_REPLICATION=${HDFS_REPLICATION-3}
-export ISILON_NAMENODE=${ISILON_NAMENODE-""}
-export DEFAULT_FS=${DEFAULT_FS-"hdfs://localhost:20500"}
-export WAREHOUSE_LOCATION_PREFIX=${WAREHOUSE_LOCATION_PREFIX-""}
+export HADOOP_LZO="${HADOOP_LZO-$IMPALA_HOME/../hadoop-lzo}"
+export IMPALA_LZO="${IMPALA_LZO-$IMPALA_HOME/../Impala-lzo}"
+export IMPALA_AUX_TEST_HOME="${IMPALA_AUX_TEST_HOME-$IMPALA_HOME/../Impala-auxiliary-tests}"
+export TARGET_FILESYSTEM="${TARGET_FILESYSTEM-hdfs}"
+export FILESYSTEM_PREFIX="${FILESYSTEM_PREFIX-}"
+export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY-DummySecretAccessKey}"
+export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID-DummyAccessKeyId}"
+export S3_BUCKET="${S3_BUCKET-}"
+export HDFS_REPLICATION="${HDFS_REPLICATION-3}"
+export ISILON_NAMENODE="${ISILON_NAMENODE-}"
+export DEFAULT_FS="${DEFAULT_FS-hdfs://localhost:20500}"
+export WAREHOUSE_LOCATION_PREFIX="${WAREHOUSE_LOCATION_PREFIX-}"
export LOCAL_FS="file:${WAREHOUSE_LOCATION_PREFIX}"
export METASTORE_DB="hive_impala"
@@ -198,8 +198,8 @@ elif [ "${TARGET_FILESYSTEM}" = "local" ]; then
"'$WAREHOUSE_LOCATION_PREFIX'"
return 1
fi
- export DEFAULT_FS=${LOCAL_FS}
- export FILESYSTEM_PREFIX=${LOCAL_FS}
+ export DEFAULT_FS="${LOCAL_FS}"
+ export FILESYSTEM_PREFIX="${LOCAL_FS}"
elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then
echo "Unsupported filesystem '$TARGET_FILESYSTEM'"
echo "Valid values are: hdfs, isilon, s3, local"
@@ -207,15 +207,15 @@ elif [ "${TARGET_FILESYSTEM}" != "hdfs" ]; then
fi
# Directories where local cluster logs will go when running tests or loading data
-export IMPALA_LOGS_DIR=${IMPALA_HOME}/logs
-export IMPALA_CLUSTER_LOGS_DIR=${IMPALA_LOGS_DIR}/cluster
-export IMPALA_DATA_LOADING_LOGS_DIR=${IMPALA_LOGS_DIR}/data_loading
-export IMPALA_DATA_LOADING_SQL_DIR=${IMPALA_DATA_LOADING_LOGS_DIR}/sql
-export IMPALA_FE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/fe_tests
-export IMPALA_FE_TEST_COVERAGE_DIR=${IMPALA_FE_TEST_LOGS_DIR}/coverage
-export IMPALA_BE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/be_tests
-export IMPALA_EE_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/ee_tests
-export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR=${IMPALA_LOGS_DIR}/custom_cluster_tests
+export IMPALA_LOGS_DIR="${IMPALA_HOME}/logs"
+export IMPALA_CLUSTER_LOGS_DIR="${IMPALA_LOGS_DIR}/cluster"
+export IMPALA_DATA_LOADING_LOGS_DIR="${IMPALA_LOGS_DIR}/data_loading"
+export IMPALA_DATA_LOADING_SQL_DIR="${IMPALA_DATA_LOADING_LOGS_DIR}/sql"
+export IMPALA_FE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/fe_tests"
+export IMPALA_FE_TEST_COVERAGE_DIR="${IMPALA_FE_TEST_LOGS_DIR}/coverage"
+export IMPALA_BE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/be_tests"
+export IMPALA_EE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/ee_tests"
+export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/custom_cluster_tests"
# List of all Impala log dirs and create them.
export IMPALA_ALL_LOGS_DIRS="${IMPALA_CLUSTER_LOGS_DIR}
${IMPALA_DATA_LOADING_LOGS_DIR} ${IMPALA_DATA_LOADING_SQL_DIR}
@@ -229,21 +229,21 @@ mkdir -p $IMPALA_ALL_LOGS_DIRS
# hardwired in its code, so we cannot change the output dir by configuration.
# We create two symlinks to capture the logs when running ctest either from
# ${IMPALA_HOME} or ${IMPALA_HOME}/be.
-rm -rf ${IMPALA_HOME}/Testing
-mkdir -p ${IMPALA_HOME}/Testing
-ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/Testing/Temporary
-rm -rf ${IMPALA_HOME}/be/Testing
-mkdir -p ${IMPALA_HOME}/be/Testing
-ln -fs ${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_HOME}/be/Testing/Temporary
+rm -rf "${IMPALA_HOME}/Testing"
+mkdir -p "${IMPALA_HOME}/Testing"
+ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
+rm -rf "${IMPALA_HOME}/be/Testing"
+mkdir -p "${IMPALA_HOME}/be/Testing"
+ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
# Reduce the concurrency for local tests to half the number of cores in the system.
# Note than nproc may not be available on older distributions (centos5.5)
if type nproc >/dev/null 2>&1; then
CORES=$(($(nproc) / 2))
else
- CORES='4'
+ CORES=4
fi
-export NUM_CONCURRENT_TESTS=${NUM_CONCURRENT_TESTS-${CORES}}
+export NUM_CONCURRENT_TESTS="${NUM_CONCURRENT_TESTS-${CORES}}"
# Versions of toolchain dependencies (or if toolchain is not used of dependencies in
# thirdparty)
@@ -280,8 +280,8 @@ export IMPALA_THRIFT_VERSION=0.9.0-p8
export IMPALA_THRIFT_JAVA_VERSION=0.9.0
export IMPALA_ZLIB_VERSION=1.2.8
-export KUDU_MASTER=${KUDU_MASTER:-"127.0.0.1"}
-export KUDU_MASTER_PORT=${KUDU_MASTER_PORT:-"7051"}
+export KUDU_MASTER="${KUDU_MASTER:-127.0.0.1}"
+export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}"
# TODO: Figure out a way to use a snapshot version without causing a lot of breakage due
# to nightly changes from Kudu. The version below is the last released version but
# before release this needs to be updated to the version about to be released.
@@ -303,83 +303,85 @@ export IMPALA_LLAMA_VERSION=1.0.0-cdh5.10.0-SNAPSHOT
export IMPALA_PARQUET_VERSION=1.5.0-cdh5.10.0-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
-export IMPALA_FE_DIR=$IMPALA_HOME/fe
-export IMPALA_BE_DIR=$IMPALA_HOME/be
-export IMPALA_WORKLOAD_DIR=$IMPALA_HOME/testdata/workloads
-export IMPALA_AUX_WORKLOAD_DIR=$IMPALA_AUX_TEST_HOME/testdata/workloads
-export IMPALA_DATASET_DIR=$IMPALA_HOME/testdata/datasets
-export IMPALA_AUX_DATASET_DIR=$IMPALA_AUX_TEST_HOME/testdata/datasets
-export IMPALA_COMMON_DIR=$IMPALA_HOME/common
-export PATH=$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH
+export IMPALA_FE_DIR="$IMPALA_HOME/fe"
+export IMPALA_BE_DIR="$IMPALA_HOME/be"
+export IMPALA_WORKLOAD_DIR="$IMPALA_HOME/testdata/workloads"
+export IMPALA_AUX_WORKLOAD_DIR="$IMPALA_AUX_TEST_HOME/testdata/workloads"
+export IMPALA_DATASET_DIR="$IMPALA_HOME/testdata/datasets"
+export IMPALA_AUX_DATASET_DIR="$IMPALA_AUX_TEST_HOME/testdata/datasets"
+export IMPALA_COMMON_DIR="$IMPALA_HOME/common"
+export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/:$PATH"
# The directory in which all the thirdparty CDH components live.
if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then
- export CDH_COMPONENTS_HOME=$IMPALA_TOOLCHAIN/cdh_components
+ export CDH_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdh_components"
else
- export CDH_COMPONENTS_HOME=$IMPALA_HOME/thirdparty
+ export CDH_COMPONENTS_HOME="$IMPALA_HOME/thirdparty"
fi
# Hadoop dependencies are snapshots in the Impala tree
-export HADOOP_HOME=$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/
-export HADOOP_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
+export HADOOP_HOME="$CDH_COMPONENTS_HOME/hadoop-${IMPALA_HADOOP_VERSION}/"
+export HADOOP_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
: ${HADOOP_CLASSPATH=}
-export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:"${HADOOP_HOME}/share/hadoop/tools/lib/*"
+# Please note that the * is inside quotes, thus it won't get exanded by bash but
+# by java, see "Understanding class path wildcards" at http://goo.gl/f0cfft
+export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HADOOP_HOME}/share/hadoop/tools/lib/*"
# YARN is configured to use LZO so the LZO jar needs to be in the hadoop classpath.
export LZO_JAR_PATH="$HADOOP_LZO/build/hadoop-lzo-0.4.15.jar"
HADOOP_CLASSPATH+=":$LZO_JAR_PATH"
-export MINI_DFS_BASE_DATA_DIR=$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data
-export PATH=$HADOOP_HOME/bin:$PATH
+export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data"
+export PATH="$HADOOP_HOME/bin:$PATH"
-export LLAMA_HOME=$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/
-export MINIKDC_HOME=$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}
-export SENTRY_HOME=$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}
-export SENTRY_CONF_DIR=$IMPALA_HOME/fe/src/test/resources
+export LLAMA_HOME="$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/"
+export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}"
+export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}"
+export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources"
-export HIVE_HOME=$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/
-export PATH=$HIVE_HOME/bin:$PATH
-export HIVE_CONF_DIR=$IMPALA_FE_DIR/src/test/resources
+export HIVE_HOME="$CDH_COMPONENTS_HOME/hive-${IMPALA_HIVE_VERSION}/"
+export PATH="$HIVE_HOME/bin:$PATH"
+export HIVE_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
# Hive looks for jar files in a single directory from HIVE_AUX_JARS_PATH plus
# any jars in AUX_CLASSPATH. (Or a list of jars in HIVE_AUX_JARS_PATH.)
# The Postgres JDBC driver is downloaded by maven when building the frontend.
# Export the location of Postgres JDBC driver so Sentry can pick it up.
-export POSTGRES_JDBC_DRIVER=${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar
+export POSTGRES_JDBC_DRIVER="${IMPALA_FE_DIR}/target/dependency/postgresql-${IMPALA_POSTGRES_JDBC_DRIVER_VERSION}.jdbc4.jar"
export HIVE_AUX_JARS_PATH="$POSTGRES_JDBC_DRIVER"
export AUX_CLASSPATH="${LZO_JAR_PATH}"
### Tell hive not to use jline
export HADOOP_USER_CLASSPATH_FIRST=true
-export HBASE_HOME=$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/
-export PATH=$HBASE_HOME/bin:$PATH
+export HBASE_HOME="$CDH_COMPONENTS_HOME/hbase-${IMPALA_HBASE_VERSION}/"
+export PATH="$HBASE_HOME/bin:$PATH"
# Add the jars so hive can create hbase tables.
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar
-export AUX_CLASSPATH=$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-common-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-client-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar"
+export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar"
-export HBASE_CONF_DIR=$HIVE_CONF_DIR
+export HBASE_CONF_DIR="$HIVE_CONF_DIR"
# Set $THRIFT_HOME to the Thrift directory in toolchain.
-export THRIFT_HOME=${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}
+export THRIFT_HOME="${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}"
# ASAN needs a matching version of llvm-symbolizer to symbolize stack traces.
-export ASAN_SYMBOLIZER_PATH=${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer
+export ASAN_SYMBOLIZER_PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_ASAN_VERSION}/bin/llvm-symbolizer"
-export CLUSTER_DIR=${IMPALA_HOME}/testdata/cluster
+export CLUSTER_DIR="${IMPALA_HOME}/testdata/cluster"
-: ${IMPALA_BUILD_THREADS:=$(nproc)}
+: ${IMPALA_BUILD_THREADS:="$(nproc)"}
export IMPALA_BUILD_THREADS
# Some environments (like the packaging build) might not have $USER set. Fix that here.
-export USER=${USER-`id -un`}
+export USER="${USER-`id -un`}"
# Configure python path
-. $IMPALA_HOME/bin/set-pythonpath.sh
+. "$IMPALA_HOME/bin/set-pythonpath.sh"
# These arguments are, despite the name, passed to every JVM created
# by an impalad.
@@ -396,17 +398,17 @@ LIBHDFS_OPTS="${LIBHDFS_OPTS} -Djava.library.path=${HADOOP_HOME}/lib/native/"
# the build type.
export LIBHDFS_OPTS="${LIBHDFS_OPTS}:${IMPALA_HOME}/be/build/debug/service"
-export ARTISTIC_STYLE_OPTIONS=$IMPALA_BE_DIR/.astylerc
+export ARTISTIC_STYLE_OPTIONS="$IMPALA_BE_DIR/.astylerc"
-export IMPALA_SNAPPY_PATH=${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib
+export IMPALA_SNAPPY_PATH="${IMPALA_TOOLCHAIN}/snappy-${IMPALA_SNAPPY_VERSION}/lib"
-export JAVA_LIBRARY_PATH=${IMPALA_SNAPPY_PATH}
+export JAVA_LIBRARY_PATH="${IMPALA_SNAPPY_PATH}"
# So that the frontend tests and PlanService can pick up libbackend.so
# and other required libraries
-LIB_JAVA=`find ${JAVA_HOME}/ -name libjava.so | head -1`
-LIB_JSIG=`find ${JAVA_HOME}/ -name libjsig.so | head -1`
-LIB_JVM=` find ${JAVA_HOME}/ -name libjvm.so | head -1`
+LIB_JAVA=`find "${JAVA_HOME}/" -name libjava.so | head -1`
+LIB_JSIG=`find "${JAVA_HOME}/" -name libjsig.so | head -1`
+LIB_JVM=` find "${JAVA_HOME}/" -name libjvm.so | head -1`
LD_LIBRARY_PATH="${LD_LIBRARY_PATH-}"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JAVA}`:`dirname ${LIB_JSIG}`"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:`dirname ${LIB_JVM}`"
@@ -416,7 +418,7 @@ LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_SNAPPY_PATH}"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_LZO}/build"
if [ $USE_SYSTEM_GCC -eq 0 ]; then
- IMPALA_TOOLCHAIN_GCC_LIB=${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64
+ IMPALA_TOOLCHAIN_GCC_LIB="${IMPALA_TOOLCHAIN}/gcc-${IMPALA_GCC_VERSION}/lib64"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${IMPALA_TOOLCHAIN_GCC_LIB}"
fi
@@ -425,10 +427,10 @@ LD_PRELOAD="${LD_PRELOAD-}"
export LD_PRELOAD="${LD_PRELOAD}:${LIB_JSIG}"
CLASSPATH="${CLASSPATH-}"
-CLASSPATH=$IMPALA_FE_DIR/target/dependency:$CLASSPATH
-CLASSPATH=$IMPALA_FE_DIR/target/classes:$CLASSPATH
-CLASSPATH=$IMPALA_FE_DIR/src/test/resources:$CLASSPATH
-CLASSPATH=$LZO_JAR_PATH:$CLASSPATH
+CLASSPATH="$IMPALA_FE_DIR/target/dependency:$CLASSPATH"
+CLASSPATH="$IMPALA_FE_DIR/target/classes:$CLASSPATH"
+CLASSPATH="$IMPALA_FE_DIR/src/test/resources:$CLASSPATH"
+CLASSPATH="$LZO_JAR_PATH:$CLASSPATH"
export CLASSPATH
# Setup aliases
@@ -465,9 +467,9 @@ echo "DOWNLOAD_CDH_COMPONENTS = $DOWNLOAD_CDH_COMPONENTS"
# work. Note that if impala-config.sh is sourced before the
# kerberized cluster is created, it will have to be sourced again
# *after* the cluster is created in order to pick up these settings.
-export MINIKDC_ENV=${IMPALA_HOME}/testdata/bin/minikdc_env.sh
-if ${CLUSTER_DIR}/admin is_kerberized; then
- . ${MINIKDC_ENV}
+export MINIKDC_ENV="${IMPALA_HOME}/testdata/bin/minikdc_env.sh"
+if "${CLUSTER_DIR}/admin" is_kerberized; then
+ . "${MINIKDC_ENV}"
echo " *** This cluster is kerberized ***"
echo "KRB5_KTNAME = $KRB5_KTNAME"
echo "KRB5_CONFIG = $KRB5_CONFIG"
@@ -478,6 +480,6 @@ else
# If the cluster *isn't* kerberized, ensure that the environment isn't
# polluted with kerberos items that might screw us up. We go through
# everything set in the minikdc environment and explicitly unset it.
- unset `grep export ${MINIKDC_ENV} | sed "s/.*export \([^=]*\)=.*/\1/" \
+ unset `grep export "${MINIKDC_ENV}" | sed "s/.*export \([^=]*\)=.*/\1/" \
| sort | uniq`
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python
----------------------------------------------------------------------
diff --git a/bin/impala-python b/bin/impala-python
index 5ff4190..29f9406 100755
--- a/bin/impala-python
+++ b/bin/impala-python
@@ -1,3 +1,3 @@
#!/bin/bash
-source $(dirname "$0")/impala-python-common.sh
+source "$(dirname "$0")/impala-python-common.sh"
exec "$PY_DIR/env/bin/python" "$@"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/impala-python-common.sh
----------------------------------------------------------------------
diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh
index 2084429..29a36a2 100644
--- a/bin/impala-python-common.sh
+++ b/bin/impala-python-common.sh
@@ -21,8 +21,8 @@
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
-LD_LIBRARY_PATH+=":$(python $IMPALA_HOME/infra/python/bootstrap_virtualenv.py \
+LD_LIBRARY_PATH+=":$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
--print-ld-library-path)"
-PY_DIR=$(dirname "$0")/../infra/python
+PY_DIR="$(dirname "$0")/../infra/python"
python "$PY_DIR/bootstrap_virtualenv.py"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/run-all-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh
index 4322efc..7d68216 100755
--- a/bin/run-all-tests.sh
+++ b/bin/run-all-tests.sh
@@ -24,7 +24,7 @@
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
-. $IMPALA_HOME/bin/set-pythonpath.sh
+. "$IMPALA_HOME/bin/set-pythonpath.sh"
# Allow picking up strategy from environment
: ${EXPLORATION_STRATEGY:=core}
@@ -32,9 +32,9 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
: ${MAX_PYTEST_FAILURES:=10}
KERB_ARGS=""
-. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
-. ${IMPALA_HOME}/testdata/bin/run-step.sh
-if ${CLUSTER_DIR}/admin is_kerberized; then
+. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1
+. "${IMPALA_HOME}/testdata/bin/run-step.sh"
+if "${CLUSTER_DIR}/admin" is_kerberized; then
KERB_ARGS="--use_kerberos"
fi
@@ -69,10 +69,10 @@ while getopts "e:n:c" OPTION
do
case "$OPTION" in
e)
- EXPLORATION_STRATEGY=$OPTARG
+ EXPLORATION_STRATEGY="$OPTARG"
;;
n)
- NUM_TEST_ITERATIONS=$OPTARG
+ NUM_TEST_ITERATIONS="$OPTARG"
;;
c)
CODE_COVERAGE=true
@@ -98,7 +98,7 @@ if [[ "${TARGET_FILESYSTEM}" == "local" ]]; then
fi
# For logging when using run-step.
-LOG_DIR=${IMPALA_EE_TEST_LOGS_DIR}
+LOG_DIR="${IMPALA_EE_TEST_LOGS_DIR}"
# Enable core dumps
ulimit -c unlimited
@@ -108,7 +108,7 @@ if [[ "${TARGET_FILESYSTEM}" == "hdfs" ]]; then
# script. Restarting HBase will change the region server assignment. Run split-hbase.sh
# before running any test.
run-step "Split and assign HBase regions" split-hbase.log \
- ${IMPALA_HOME}/testdata/bin/split-hbase.sh
+ "${IMPALA_HOME}/testdata/bin/split-hbase.sh"
fi
for i in $(seq 1 $NUM_TEST_ITERATIONS)
@@ -116,7 +116,7 @@ do
TEST_RET_CODE=0
run-step "Starting Impala cluster" start-impala-cluster.log \
- ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \
+ "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \
${TEST_START_CLUSTER_ARGS}
if [[ "$BE_TEST" == true ]]; then
@@ -127,14 +127,14 @@ do
export SKIP_BE_TEST_PATTERN="session*"
fi
# Run backend tests.
- if ! ${IMPALA_HOME}/bin/run-backend-tests.sh; then
+ if ! "${IMPALA_HOME}/bin/run-backend-tests.sh"; then
TEST_RET_CODE=1
fi
fi
# Run some queries using run-workload to verify run-workload has not been broken.
if ! run-step "Run test run-workload" test-run-workload.log \
- ${IMPALA_HOME}/bin/run-workload.py -w tpch --num_clients=2 --query_names=TPCH-Q1 \
+ "${IMPALA_HOME}/bin/run-workload.py" -w tpch --num_clients=2 --query_names=TPCH-Q1 \
--table_format=text/none --exec_options="disable_codegen:False" ${KERB_ARGS}; then
TEST_RET_CODE=1
fi
@@ -143,7 +143,7 @@ do
# Run JUnit frontend tests
# Requires a running impalad cluster because some tests (such as DataErrorTest and
# JdbcTest) queries against an impala cluster.
- pushd ${IMPALA_FE_DIR}
+ pushd "${IMPALA_FE_DIR}"
MVN_ARGS=""
if [[ "${TARGET_FILESYSTEM}" == "s3" ]]; then
# When running against S3, only run the S3 frontend tests.
@@ -152,7 +152,7 @@ do
if [[ "$CODE_COVERAGE" == true ]]; then
MVN_ARGS+="-DcodeCoverage"
fi
- if ! ${IMPALA_HOME}/bin/mvn-quiet.sh -fae test ${MVN_ARGS}; then
+ if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" -fae test ${MVN_ARGS}; then
TEST_RET_CODE=1
fi
popd
@@ -161,7 +161,7 @@ do
if [[ "$EE_TEST" == true ]]; then
# Run end-to-end tests.
# KERBEROS TODO - this will need to deal with ${KERB_ARGS}
- if ! ${IMPALA_HOME}/tests/run-tests.py ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then
+ if ! "${IMPALA_HOME}/tests/run-tests.py" ${COMMON_PYTEST_ARGS} ${EE_TEST_FILES}; then
#${KERB_ARGS};
TEST_RET_CODE=1
fi
@@ -170,11 +170,11 @@ do
if [[ "$JDBC_TEST" == true ]]; then
# Run the JDBC tests with background loading disabled. This is interesting because
# it requires loading missing table metadata.
- ${IMPALA_HOME}/bin/start-impala-cluster.py --log_dir=${IMPALA_EE_TEST_LOGS_DIR} \
+ "${IMPALA_HOME}/bin/start-impala-cluster.py" --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \
--catalogd_args=--load_catalog_in_background=false \
${TEST_START_CLUSTER_ARGS}
- pushd ${IMPALA_FE_DIR}
- if ! ${IMPALA_HOME}/bin/mvn-quiet.sh test -Dtest=JdbcTest; then
+ pushd "${IMPALA_FE_DIR}"
+ if ! "${IMPALA_HOME}/bin/mvn-quiet.sh" test -Dtest=JdbcTest; then
TEST_RET_CODE=1
fi
popd
@@ -186,17 +186,17 @@ do
# pollute the directory with too many files, remove what was there
# before. Also, save the IMPALA_MAX_LOG_FILES value for re-set
# later.
- rm -rf ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}
- mkdir -p ${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}
- IMPALA_MAX_LOG_FILES_SAVE=${IMPALA_MAX_LOG_FILES:-10}
+ rm -rf "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
+ mkdir -p "${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
+ IMPALA_MAX_LOG_FILES_SAVE="${IMPALA_MAX_LOG_FILES:-10}"
export IMPALA_MAX_LOG_FILES=0
# Run the custom-cluster tests after all other tests, since they will restart the
# cluster repeatedly and lose state.
# TODO: Consider moving in to run-tests.py.
- if ! ${IMPALA_HOME}/tests/run-custom-cluster-tests.sh ${COMMON_PYTEST_ARGS}; then
+ if ! "${IMPALA_HOME}/tests/run-custom-cluster-tests.sh" ${COMMON_PYTEST_ARGS}; then
TEST_RET_CODE=1
fi
- export IMPALA_MAX_LOG_FILES=${IMPALA_MAX_LOG_FILES_SAVE}
+ export IMPALA_MAX_LOG_FILES="${IMPALA_MAX_LOG_FILES_SAVE}"
fi
# Finally, run the process failure tests.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/bin/set-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh
old mode 100644
new mode 100755
index 873f6ec..68aec2b
--- a/bin/set-classpath.sh
+++ b/bin/set-classpath.sh
@@ -23,20 +23,24 @@
# explicitly to the CLASSPATH.
CLASSPATH=\
-$IMPALA_HOME/fe/src/test/resources:\
-$IMPALA_HOME/fe/target/classes:\
-$IMPALA_HOME/fe/target/dependency:\
-$IMPALA_HOME/fe/target/test-classes:\
-${HIVE_HOME}/lib/datanucleus-api-jdo-3.2.1.jar:\
-${HIVE_HOME}/lib/datanucleus-core-3.2.2.jar:\
-${HIVE_HOME}/lib/datanucleus-rdbms-3.2.1.jar:
+"$IMPALA_HOME"/fe/src/test/resources:\
+"$IMPALA_HOME"/fe/target/classes:\
+"$IMPALA_HOME"/fe/target/dependency:\
+"$IMPALA_HOME"/fe/target/test-classes:\
+"${HIVE_HOME}"/lib/datanucleus-api-jdo-3.2.1.jar:\
+"${HIVE_HOME}"/lib/datanucleus-core-3.2.2.jar:\
+"${HIVE_HOME}"/lib/datanucleus-rdbms-3.2.1.jar:
-for jar in `ls ${IMPALA_HOME}/fe/target/dependency/*.jar`; do
- CLASSPATH=${CLASSPATH}:$jar
+for jar in "${IMPALA_HOME}"/fe/target/dependency/*.jar; do
+ if [ -e "$jar" ] ; then
+ CLASSPATH="${CLASSPATH}:$jar"
+ fi
done
-for jar in `ls ${IMPALA_HOME}/testdata/target/dependency/*.jar`; do
- CLASSPATH=${CLASSPATH}:$jar
+for jar in "${IMPALA_HOME}"/testdata/target/dependency/*.jar; do
+ if [ -e "$jar" ] ; then
+ CLASSPATH="${CLASSPATH}:$jar"
+ fi
done
export CLASSPATH
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index ec79420..1845081 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -23,7 +23,7 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
# run buildall.sh -help to see options
ROOT=`dirname "$0"`
-ROOT=`cd "$ROOT"; pwd`
+ROOT=`cd "$ROOT" >/dev/null; pwd`
# Grab this *before* we source impala-config.sh to see if the caller has
# kerberized environment variables already or not.
@@ -33,7 +33,7 @@ if [ ! -z "${MINIKDC_REALM}" ]; then
NEEDS_RE_SOURCE_NOTE=0
fi
-export IMPALA_HOME=$ROOT
+export IMPALA_HOME="$ROOT"
if ! . "$ROOT"/bin/impala-config.sh; then
echo "Bad configuration, aborting buildall."
exit 1
@@ -112,25 +112,25 @@ do
EXPLORATION_STRATEGY=exhaustive
;;
-snapshot_file)
- SNAPSHOT_FILE=${2-}
- if [ ! -f $SNAPSHOT_FILE ]; then
+ SNAPSHOT_FILE="${2-}"
+ if [ ! -f "$SNAPSHOT_FILE" ]; then
echo "-snapshot_file does not exist: $SNAPSHOT_FILE"
exit 1;
fi
TESTDATA_ACTION=1
# Get the full path.
- SNAPSHOT_FILE=$(readlink -f $SNAPSHOT_FILE)
+ SNAPSHOT_FILE="$(readlink -f "$SNAPSHOT_FILE")"
shift;
;;
-metastore_snapshot_file)
- METASTORE_SNAPSHOT_FILE=${2-}
- if [ ! -f $METASTORE_SNAPSHOT_FILE ]; then
+ METASTORE_SNAPSHOT_FILE="${2-}"
+ if [ ! -f "$METASTORE_SNAPSHOT_FILE" ]; then
echo "-metastore_snapshot_file does not exist: $METASTORE_SNAPSHOT_FILE"
exit 1;
fi
TESTDATA_ACTION=1
# Get the full path.
- METASTORE_SNAPSHOT_FILE=$(readlink -f $METASTORE_SNAPSHOT_FILE)
+ METASTORE_SNAPSHOT_FILE="$(readlink -f "$METASTORE_SNAPSHOT_FILE")"
shift;
;;
-k|-kerberize|-kerberos|-kerb)
@@ -236,16 +236,16 @@ if [ ${IMPALA_KERBERIZE} -eq 0 ]; then
fi
# Loading data on a filesystem other than fs.defaultFS is not supported.
-if [[ -z $METASTORE_SNAPSHOT_FILE && "${TARGET_FILESYSTEM}" != "hdfs" &&
- $TESTDATA_ACTION -eq 1 ]]; then
+if [[ -z "$METASTORE_SNAPSHOT_FILE" && "${TARGET_FILESYSTEM}" != "hdfs" &&
+ "$TESTDATA_ACTION" -eq 1 ]]; then
echo "The metastore snapshot is required for loading data into ${TARGET_FILESYSTEM}"
echo "Use the -metastore_snapshot_file command line paramater."
exit 1
fi
# option to clean everything first
-if [ $CLEAN_ACTION -eq 1 ]; then
- $IMPALA_HOME/bin/clean.sh
+if [ "$CLEAN_ACTION" -eq 1 ]; then
+ "$IMPALA_HOME/bin/clean.sh"
fi
# Populate necessary thirdparty components unless it's set to be skipped.
@@ -256,7 +256,7 @@ else
echo "Downloading Python dependencies"
# Download all the Python dependencies we need before doing anything
# of substance. Does not re-download anything that is already present.
- if ! $IMPALA_HOME/infra/python/deps/download_requirements; then
+ if ! "$IMPALA_HOME/infra/python/deps/download_requirements"; then
echo "Warning: Unable to download Python requirements."
echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail."
else
@@ -264,88 +264,88 @@ else
fi
echo "Downloading and extracting toolchain dependencies."
- $IMPALA_HOME/bin/bootstrap_toolchain.py
+ "$IMPALA_HOME/bin/bootstrap_toolchain.py"
echo "Toolchain bootstrap complete."
fi
MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}"
-if [ $BUILD_FE_ONLY -eq 1 ]; then
- $IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} -cmake_only
- ${MAKE_CMD} fe
+if [ "$BUILD_FE_ONLY" -eq 1 ]; then
+ "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only
+ "${MAKE_CMD}" fe
exit 0
fi
-if [ -e $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.so ]
+if [ -e "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.so ]
then
- cp $HADOOP_LZO/build/native/Linux-*-*/lib/libgplcompression.* $HADOOP_HOME/lib/native
+ cp "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* "$HADOOP_HOME/lib/native"
else
echo "No hadoop-lzo found"
fi
# Stop any running Impala services.
-${IMPALA_HOME}/bin/start-impala-cluster.py --kill --force
+"${IMPALA_HOME}/bin/start-impala-cluster.py" --kill --force
-if [[ $CLEAN_ACTION -eq 1 || $FORMAT_METASTORE -eq 1 || $FORMAT_CLUSTER -eq 1 ||
- $FORMAT_SENTRY_POLICY_DB -eq 1 || -n $METASTORE_SNAPSHOT_FILE ]]
+if [[ "$CLEAN_ACTION" -eq 1 || "$FORMAT_METASTORE" -eq 1 || "$FORMAT_CLUSTER" -eq 1 ||
+ "$FORMAT_SENTRY_POLICY_DB" -eq 1 || -n "$METASTORE_SNAPSHOT_FILE" ]]
then
# Kill any processes that may be accessing postgres metastore. To be safe, this is done
# before we make any changes to the config files.
set +e
- ${IMPALA_HOME}/testdata/bin/kill-all.sh
+ "${IMPALA_HOME}/testdata/bin/kill-all.sh"
set -e
fi
CREATE_TEST_CONFIG_ARGS=""
-if [[ $FORMAT_SENTRY_POLICY_DB -eq 1 ]]; then
+if [[ "$FORMAT_SENTRY_POLICY_DB" -eq 1 ]]; then
CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db"
fi
-if [[ $FORMAT_METASTORE -eq 1 && -z $METASTORE_SNAPSHOT_FILE ]]; then
+if [[ "$FORMAT_METASTORE" -eq 1 && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_TEST_CONFIG_ARGS+=" -create_metastore"
fi
# Generate the Hadoop configs needed by Impala
-${IMPALA_HOME}/bin/create-test-configuration.sh ${CREATE_TEST_CONFIG_ARGS}
+"${IMPALA_HOME}/bin/create-test-configuration.sh" ${CREATE_TEST_CONFIG_ARGS}
# If a metastore snapshot exists, load it.
-if [ $METASTORE_SNAPSHOT_FILE ]; then
+if [ "$METASTORE_SNAPSHOT_FILE" ]; then
echo "Loading metastore snapshot"
- ${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh $METASTORE_SNAPSHOT_FILE
+ "${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh" "$METASTORE_SNAPSHOT_FILE"
fi
# build common and backend
echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
-$IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS}
+"$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
-if [ -e $IMPALA_LZO ]
+if [ -e "$IMPALA_LZO" ]
then
- pushd $IMPALA_LZO
+ pushd "$IMPALA_LZO"
LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
rm -f CMakeCache.txt
cmake ${LZO_CMAKE_ARGS}
- ${MAKE_CMD}
+ "${MAKE_CMD}"
popd
fi
# build the external data source API
-pushd ${IMPALA_HOME}/ext-data-source
-${IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
+pushd "${IMPALA_HOME}/ext-data-source"
+"${IMPALA_HOME}/bin/mvn-quiet.sh" install -DskipTests
popd
# build frontend and copy dependencies
-pushd ${IMPALA_FE_DIR}
-${IMPALA_HOME}/bin/mvn-quiet.sh package -DskipTests
+pushd "${IMPALA_FE_DIR}"
+"${IMPALA_HOME}/bin/mvn-quiet.sh" package -DskipTests
popd
# Build the shell tarball
echo "Creating shell tarball"
-${IMPALA_HOME}/shell/make_shell_tarball.sh
+"${IMPALA_HOME}/shell/make_shell_tarball.sh"
-if [ $FORMAT_CLUSTER -eq 1 ]; then
- $IMPALA_HOME/testdata/bin/run-all.sh -format
-elif [ $TESTDATA_ACTION -eq 1 ] || [ $TESTS_ACTION -eq 1 ]; then
- $IMPALA_HOME/testdata/bin/run-all.sh
+if [ "$FORMAT_CLUSTER" -eq 1 ]; then
+ "$IMPALA_HOME/testdata/bin/run-all.sh" -format
+elif [ "$TESTDATA_ACTION" -eq 1 ] || [ "$TESTS_ACTION "-eq 1 ]; then
+ "$IMPALA_HOME/testdata/bin/run-all.sh"
fi
#
@@ -368,7 +368,7 @@ if [ ${IMPALA_KERBERIZE} -eq 1 ]; then
echo "won't be run. The impala daemons will be started."
TESTDATA_ACTION=0
TESTS_ACTION=0
- ${IMPALA_HOME}/bin/start-impala-cluster.py
+ "${IMPALA_HOME}/bin/start-impala-cluster.py"
fi
fi
# END KERBEROS TODO
@@ -389,34 +389,34 @@ fi
if [ $TESTDATA_ACTION -eq 1 ]; then
# Create testdata.
- $IMPALA_HOME/bin/create_testdata.sh
- cd $ROOT
+ "$IMPALA_HOME/bin/create_testdata.sh"
+ cd "$ROOT"
# We have 4 cases:
# - test-warehouse and metastore snapshots exists.
# - Only the test-warehouse snapshot exists.
# - Only the metastore snapshot exists.
# - Neither of them exist.
CREATE_LOAD_DATA_ARGS=""
- if [[ $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then
+ if [[ "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE} -skip_metadata_load"
- elif [[ $SNAPSHOT_FILE && -z $METASTORE_SNAPSHOT_FILE ]]; then
+ elif [[ "$SNAPSHOT_FILE" && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE}"
- elif [[ -z $SNAPSHOT_FILE && $METASTORE_SNAPSHOT_FILE ]]; then
+ elif [[ -z "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-skip_metadata_load -skip_snapshot_load"
fi
- ${IMPALA_HOME}/testdata/bin/create-load-data.sh ${CREATE_LOAD_DATA_ARGS} <<< Y
+ "${IMPALA_HOME}/testdata/bin/create-load-data.sh" ${CREATE_LOAD_DATA_ARGS} <<< Y
fi
if [ $TESTS_ACTION -eq 1 ]; then
if [ $CODE_COVERAGE -eq 0 ]; then
- ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY
+ "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY
else
- ${IMPALA_HOME}/bin/run-all-tests.sh -e $EXPLORATION_STRATEGY -c
+ "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY -c
fi
fi
# Generate list of files for Cscope to index
-$IMPALA_HOME/bin/gen-cscope.sh
+"$IMPALA_HOME/bin/gen-cscope.sh"
if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then
echo
@@ -426,6 +426,6 @@ if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then
echo "environment variables weren't available before the cluster"
echo "was created. To pick them up, please source impala-config.sh:"
echo
- echo " . ${IMPALA_HOME}/bin/impala-config.sh"
+ echo " . \"${IMPALA_HOME}/bin/impala-config.sh\""
echo
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a60ba6d2/infra/python/deps/download_requirements
----------------------------------------------------------------------
diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements
index e054653..26788be 100755
--- a/infra/python/deps/download_requirements
+++ b/infra/python/deps/download_requirements
@@ -2,15 +2,15 @@
set -euo pipefail
-DIR=$(dirname "$0")
+DIR="$(dirname "$0")"
-pushd $DIR
-PY26=$(./find_py26.py)
+pushd "$DIR"
+PY26="$(./find_py26.py)"
# Directly download packages listed in requirements.txt, but don't install them.
-$PY26 pip_download.py
+"$PY26" pip_download.py
# For virtualenv, other scripts rely on the .tar.gz package (not a .whl package).
-$PY26 pip_download.py virtualenv 13.1.0
+"$PY26" pip_download.py virtualenv 13.1.0
# kudu-python is downloaded separately because pip install attempts to execute a
# setup.py subcommand for kudu-python that can fail even if the download succeeds.
-$PY26 pip_download.py kudu-python 0.1.1
+"$PY26" pip_download.py kudu-python 0.1.1
popd
[2/3] incubator-impala git commit: Bump Impala version to 2.8
Posted by ta...@apache.org.
Bump Impala version to 2.8
Change-Id: I5b147dee9bb7fcb159e98a008c01659a4cb7473b
Reviewed-on: http://gerrit.cloudera.org:8080/4170
Tested-by: Internal Jenkins
Reviewed-by: Alex Behm <al...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/2cd75246
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/2cd75246
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/2cd75246
Branch: refs/heads/master
Commit: 2cd752461f69578694bd8be2bde137b5595b8bed
Parents: 5f3ee38
Author: Harrison Sheinblatt <hs...@cloudera.com>
Authored: Tue Aug 30 10:15:14 2016 -0700
Committer: Harrison Sheinblatt <hs...@hotmail.com>
Committed: Thu Sep 1 05:01:03 2016 +0000
----------------------------------------------------------------------
bin/save-version.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/2cd75246/bin/save-version.sh
----------------------------------------------------------------------
diff --git a/bin/save-version.sh b/bin/save-version.sh
index 6b93f41..01caf77 100755
--- a/bin/save-version.sh
+++ b/bin/save-version.sh
@@ -21,7 +21,7 @@
# Note: for internal (aka pre-release) versions, the version should have
# "-INTERNAL" appended. Parts of the code will look for this to distinguish
# between released and internal versions.
-VERSION=2.7.0-cdh5-INTERNAL
+VERSION=2.8.0-cdh5-INTERNAL
GIT_HASH=$(git rev-parse HEAD)
BUILD_TIME=`date`
HEADER="# Generated version information from save-version.sh"