You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ab...@apache.org on 2018/08/24 09:08:03 UTC
[1/3] kudu git commit: [hms] update the HIVE/HADOOP package scripts
Repository: kudu
Updated Branches:
refs/heads/master 36187ef62 -> d3ef58b71
[hms] update the HIVE/HADOOP package scripts
This patch updates the HIVE/HADOOP package scripts to be more flexible,
so that users can choose whether to do jar downloading or repackaging.
It also adjusts the jars that need to be stripped for universal HIVE
and HADOOP distributions.
Change-Id: Ie66c4e132800fbf5ea76ad4b1fec8212757c84c2
Reviewed-on: http://gerrit.cloudera.org:8080/11274
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Hao Hao <ha...@cloudera.com>
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4fe8967d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4fe8967d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4fe8967d
Branch: refs/heads/master
Commit: 4fe8967dd9aaca37f835a0199324e5f7158e8210
Parents: 36187ef
Author: Hao Hao <ha...@cloudera.com>
Authored: Thu Aug 16 12:43:23 2018 -0700
Committer: Hao Hao <ha...@cloudera.com>
Committed: Thu Aug 23 23:18:16 2018 +0000
----------------------------------------------------------------------
thirdparty/package-hadoop.sh | 79 +++++++++++++++++++++++++----
thirdparty/package-hive.sh | 104 +++++++++++++++++++++++++++++++-------
2 files changed, 155 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/4fe8967d/thirdparty/package-hadoop.sh
----------------------------------------------------------------------
diff --git a/thirdparty/package-hadoop.sh b/thirdparty/package-hadoop.sh
index 65983ef..ea5ec33 100755
--- a/thirdparty/package-hadoop.sh
+++ b/thirdparty/package-hadoop.sh
@@ -23,24 +23,81 @@
# improves HMS startup time.
#
# Summary:
-# 1. Download and unpack the Hadoop binary tarball
+# 1. Download and unpack the Hadoop binary tarball (optional)
# 2. Strip out unnecessary jars
-# 3. Repackage the tarball
-#
-# Usage:
-# $ env VERSION=2.8.2 thirdparty/package-hadoop.sh
+# 3. Repackage the tarball (optional)
+
+set -ex
+
+function usage() {
+ cat <<EOF
+ Usage:
+ ${0} -h | --help
+ ${0} [-d] [-r] <ARTIFACT>
+ -h | --help - This message
+ -d | --download - Optional
+ Download and unpack the binary tarball.
+ -r | --repackage - Optional
+ Repackage the tarball.
+ <ARTIFACT> - Required
+ The artifact name.
+EOF
+}
-set -eux
+OPTS=`getopt -o hdr -l "help,download,repackage" -n 'parse-options' -- "$@"`
+OPTS_RESULT=$?
+if [ ${OPTS_RESULT} != 0 ]; then
+ echo "Failed parsing options." >&2
+ exit ${OPTS_RESULT}
+fi
+
+while true; do
+ case "$1" in
+ -d | --download) DOWNLOAD=True; shift;;
+ -r | --repackage) REPACKAGE=True; shift;;
+ -h | --help) usage; exit;;
+ --) shift; break;;
+ *) break ;;
+ esac
+done
-ARTIFACT=hadoop-$VERSION
+ARTIFACT="${1}"
-wget https://archive.apache.org/dist/hadoop/common/$ARTIFACT/$ARTIFACT.tar.gz
-tar xf $ARTIFACT.tar.gz
+if [ -z "${ARTIFACT}" ]; then
+ usage
+ exit
+fi
-for DIR in common/jdiff httpfs kms tools yarn; do
+if [ -n "${DOWNLOAD}" ]; then
+ curl --retry 3 -L -O https://archive.apache.org/dist/hadoop/common/$ARTIFACT/$ARTIFACT.tar.gz
+ tar xf $ARTIFACT.tar.gz
+fi
+
+DIRS="client"
+DIRS="$DIRS common/jdiff"
+DIRS="$DIRS common/sources"
+DIRS="$DIRS hdfs/sources"
+DIRS="$DIRS httpfs"
+DIRS="$DIRS kms"
+DIRS="$DIRS mapreduce/lib-examples"
+DIRS="$DIRS mapreduce/sources"
+DIRS="$DIRS tools"
+
+for DIR in $DIRS; do
rm -rf $ARTIFACT/share/hadoop/$DIR
done
+for PROJECT in avro commons-math3 snappy; do
+ rm -f $ARTIFACT/share/hadoop/common/lib/*$PROJECT*.jar
+ rm -f $ARTIFACT/share/hadoop/hdfs/lib/*$PROJECT*.jar
+done
+
+rm -f $ARTIFACT/share/hadoop/yarn/*.jar
+rm -rf $ARTIFACT/share/hadoop/yarn/lib
+rm -rf $ARTIFACT/share/hadoop/yarn/sources
+rm -rf $ARTIFACT/share/hadoop/yarn/timelineservice
rm -rf $ARTIFACT/share/doc
-tar czf $ARTIFACT-stripped.tar.gz $ARTIFACT
+if [ -n "${REPACKAGE}" ]; then
+ tar czf $ARTIFACT-stripped.tar.gz $ARTIFACT
+fi
http://git-wip-us.apache.org/repos/asf/kudu/blob/4fe8967d/thirdparty/package-hive.sh
----------------------------------------------------------------------
diff --git a/thirdparty/package-hive.sh b/thirdparty/package-hive.sh
index 5eedde5..4a27134 100755
--- a/thirdparty/package-hive.sh
+++ b/thirdparty/package-hive.sh
@@ -19,35 +19,105 @@
# Packages Hive for inclusion in the Kudu thirdparty build.
#
-# Strips jars unecessary for running the Hive Metastore. This makes the download
+# Strips jars unnecessary for running the Hive Metastore. This makes the download
# smaller, and improves HMS startup time.
#
# Summary:
-# 1. Download and unpack the Hive binary tarball
+# 1. Download and unpack the Hive binary tarball (optional)
# 2. Strip out unnecessary jars
-# 3. Repackage the tarball
-#
-# Usage:
-# $ env VERSION=2.3.1 thirdparty/package-hive.sh
+# 3. Repackage the tarball (optional)
+
+set -ex
+
+function usage() {
+ cat <<EOF
+ Usage:
+ ${0} -h | --help
+ ${0} [-d] [-r] <ARTIFACT>
+ -h | --help - This message
+ -d | --download - Optional
+ Download and unpack the binary tarball.
+ -r | --repackage - Optional
+ Repackage the tarball.
+ -v | --version VERSION - Optional
+ The Hive version.
+ <ARTIFACT> - Required
+ The artifact name.
+EOF
+}
+
+OPTS=`getopt -o hdrv: -l "help,download,repackage,version:" -n 'parse-options' -- "$@"`
+OPTS_RESULT=$?
+if [ ${OPTS_RESULT} != 0 ]; then
+ echo "Failed parsing options." >&2
+ exit ${OPTS_RESULT}
+fi
+
+while true; do
+ case "$1" in
+ -d | --download) DOWNLOAD=True; shift;;
+ -r | --repackage) REPACKAGE=True; shift;;
+ -v | --version) VERSION=$2; shift; shift;;
+ -h | --help) usage; exit;;
+ --) shift; break;;
+ *) break ;;
+ esac
+done
+
+ARTIFACT="${1}"
-set -eux
+if [ -z "${ARTIFACT}" ]; then
+ usage
+ exit
+fi
-ARTIFACT=apache-hive-$VERSION-bin
+if [ -n "${DOWNLOAD}" ]; then
+ curl --retry 3 -L -O https://archive.apache.org/dist/hive/hive-$VERSION/$ARTIFACT.tar.gz
+ tar xf $ARTIFACT.tar.gz
+fi
-curl --retry 3 -L -O https://archive.apache.org/dist/hive/hive-$VERSION/$ARTIFACT.tar.gz
-tar xf $ARTIFACT.tar.gz
+PROJECTS="accumulo"
+PROJECTS="$PROJECTS aether"
+PROJECTS="$PROJECTS avatica"
+PROJECTS="$PROJECTS avro"
+PROJECTS="$PROJECTS calcite"
+PROJECTS="$PROJECTS curator"
+PROJECTS="$PROJECTS druid"
+PROJECTS="$PROJECTS ecj"
+PROJECTS="$PROJECTS fastutil"
+PROJECTS="$PROJECTS groovy"
+PROJECTS="$PROJECTS hbase"
+PROJECTS="$PROJECTS htrace"
+PROJECTS="$PROJECTS icu4j"
+PROJECTS="$PROJECTS jcodings"
+PROJECTS="$PROJECTS jersey"
+PROJECTS="$PROJECTS jetty"
+PROJECTS="$PROJECTS jsp"
+PROJECTS="$PROJECTS maven"
+PROJECTS="$PROJECTS parquet"
+PROJECTS="$PROJECTS slider"
+PROJECTS="$PROJECTS snappy"
+PROJECTS="$PROJECTS zookeeper"
-for PROJECT in accumulo aether avatica calcite curator druid groovy hbase icu4j jetty jsp maven parquet zookeeper; do
- rm $ARTIFACT/lib/*$PROJECT*.jar
+for PROJECT in $PROJECTS; do
+ rm -f $ARTIFACT/lib/*$PROJECT*.jar
done
-rm -rf $ARTIFACT/hcatalog
+rm -rf $ARTIFACT/auxlib
+rm -rf $ARTIFACT/hcatalog/bin
+rm -rf $ARTIFACT/hcatalog/etc
+rm -rf $ARTIFACT/hcatalog/libexec
+rm -rf $ARTIFACT/hcatalog/sbin
+rm -rf $ARTIFACT/hcatalog/share/doc
+rm -rf $ARTIFACT/hcatalog/share/webhcat
rm -rf $ARTIFACT/jdbc
rm -rf $ARTIFACT/lib/php
rm -rf $ARTIFACT/lib/python
rm -rf $ARTIFACT/lib/py
-# Remove the 'apache-' prefix and '-bin' suffix in order to normalize the
-# tarball with Hadoop and the rest of our thirdparty dependencies.
-mv $ARTIFACT hive-$VERSION
-tar czf hive-$VERSION-stripped.tar.gz hive-$VERSION
+if [ -n "${REPACKAGE}" ]; then
+ # Remove the 'apache-' prefix and '-bin' suffix in order to normalize the
+ # tarball with Hadoop and the rest of our thirdparty dependencies.
+ mv $ARTIFACT hive-$VERSION
+ tar czf hive-$VERSION-stripped.tar.gz hive-$VERSION
+fi
[2/3] kudu git commit: [thirdparty] allow different URL prefix
Posted by ab...@apache.org.
[thirdparty] allow different URL prefix
This patch adds the option to allow different URL prefix when
downloading third party dependencies.
Change-Id: I004828785d9b0d484806d743d7a1084a3ffd576b
Reviewed-on: http://gerrit.cloudera.org:8080/11275
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/51519e65
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/51519e65
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/51519e65
Branch: refs/heads/master
Commit: 51519e65bd3701b7549bc780c3d0fad45d799854
Parents: 4fe8967
Author: Hao Hao <ha...@cloudera.com>
Authored: Thu Aug 16 12:50:43 2018 -0700
Committer: Hao Hao <ha...@cloudera.com>
Committed: Thu Aug 23 23:18:21 2018 +0000
----------------------------------------------------------------------
thirdparty/download-thirdparty.sh | 62 +++++++++++++++++++++++-----------
thirdparty/vars.sh | 4 +++
2 files changed, 47 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/51519e65/thirdparty/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index e1389c1..7992065 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -55,17 +55,25 @@ unzip_to_source() {
fetch_and_expand() {
local FILENAME=$1
local SOURCE=$2
+ local URL_PREFIX=$3
+
if [ -z "$FILENAME" ]; then
echo "Error: Must specify file to fetch"
exit 1
fi
+ if [ -z "$URL_PREFIX" ]; then
+ echo "Error: Must specify url prefix to fetch"
+ exit 1
+ fi
+
TAR_CMD=tar
if [[ "$OSTYPE" == "darwin"* ]] && which gtar &>/dev/null; then
TAR_CMD=gtar
fi
- FULL_URL="${CLOUDFRONT_URL_PREFIX}/${FILENAME}"
+ FULL_URL="${URL_PREFIX}/${FILENAME}"
+
SUCCESS=0
# Loop in case we encounter an error.
for attempt in 1 2 3; do
@@ -119,25 +127,41 @@ fetch_and_expand() {
echo
}
+fetch_with_url_and_patch() {
+ local FILENAME=$1
+ local SOURCE=$2
+ local PATCH_LEVEL=$3
+ local URL_PREFIX=$4
+ # Remaining args are expected to be a list of patch commands
+
+ delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL
+ if [ ! -d $SOURCE ]; then
+ fetch_and_expand $FILENAME $SOURCE $URL_PREFIX
+ pushd $SOURCE
+ shift 4
+ # Run the patch commands
+ for f in "$@"; do
+ eval "$f"
+ done
+ touch patchlevel-$PATCH_LEVEL
+ popd
+ echo
+ fi
+}
+
+# Call fetch_with_url_and_patch with the default dependency URL source.
fetch_and_patch() {
- local FILENAME=$1
- local SOURCE=$2
- local PATCH_LEVEL=$3
- # Remaining args are expected to be a list of patch commands
-
- delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL
- if [ ! -d $SOURCE ]; then
- fetch_and_expand $FILENAME $SOURCE
- pushd $SOURCE
- shift 3
- # Run the patch commands
- for f in "$@"; do
- eval "$f"
- done
- touch patchlevel-$PATCH_LEVEL
- popd
- echo
- fi
+ local FILENAME=$1
+ local SOURCE=$2
+ local PATCH_LEVEL=$3
+
+ shift 3
+ fetch_with_url_and_patch \
+ $FILENAME \
+ $SOURCE \
+ $PATCH_LEVEL \
+ $DEPENDENCY_URL \
+ "$@"
}
mkdir -p $TP_SOURCE_DIR
http://git-wip-us.apache.org/repos/asf/kudu/blob/51519e65/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 2804bb3..71beea6 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -30,6 +30,10 @@ TP_BUILD_DIR="$TP_DIR/build"
# http://cloudera-thirdparty-libs.s3.amazonaws.com/
CLOUDFRONT_URL_PREFIX=http://d3dr9sfxru4sde.cloudfront.net
+# Third party dependency downloading URL, default to the CloudFront
+# Distribution URL.
+DEPENDENCY_URL=${DEPENDENCY_URL:-$CLOUDFRONT_URL_PREFIX}
+
PREFIX_COMMON=$TP_DIR/installed/common
PREFIX_DEPS=$TP_DIR/installed/uninstrumented
PREFIX_DEPS_TSAN=$TP_DIR/installed/tsan
[3/3] kudu git commit: [cmake] add OpenSSL include path into -isystem
Posted by ab...@apache.org.
[cmake] add OpenSSL include path into -isystem
Prior to this change, when OpenSSL package is installed under the same
prefix as boost package (e.g. /opt/local on OS X if using MacPorts),
running cmake will choose the thirdparty-built boost, but due to the
intermix of '-isystem' and '-I' options, the '-I' path wins when running
the compiler. I.e., the path specified with the '-I' option is chosen
over the path specified with the '-isystem' option if the latter comes
first in the command line. That could lead to mysterious compilation
errors if the version and features of the OS-packaged boost library
differ from those compiled under $KUDU_HOME/thirdparty.
This patch puts the OpenSSL's include path into the -isystem
category along with other third-party libraries used by Kudu. Aside
from resolving the inconsistency mentioned above, this makes the usage
of the angle brackets vs quotes in #include statement to be in sync with
the way how OpenSSL headers are included into the Kudu source files.
Change-Id: I345bf5f72fb9c899340339452ba1464becd6ada3
Reviewed-on: http://gerrit.cloudera.org:8080/11311
Reviewed-by: Adar Dembo <ad...@cloudera.com>
Tested-by: Alexey Serbin <as...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d3ef58b7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d3ef58b7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d3ef58b7
Branch: refs/heads/master
Commit: d3ef58b71c83f5e6afa7c9f6cc5d42be82e5f7d8
Parents: 51519e6
Author: Alexey Serbin <as...@cloudera.com>
Authored: Thu Aug 23 15:36:05 2018 -0700
Committer: Alexey Serbin <as...@cloudera.com>
Committed: Fri Aug 24 00:41:58 2018 +0000
----------------------------------------------------------------------
CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/d3ef58b7/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1458bc8..559e3e2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1106,7 +1106,7 @@ if (NOT OPENSSL_ROOT_DIR AND EXISTS "${CENTOS_6_4_OPENSSL_DIR}")
set(OPENSSL_ROOT_DIR "${CENTOS_6_4_OPENSSL_DIR}")
endif()
find_package(OpenSSL 1.0.0 REQUIRED)
-include_directories(${OPENSSL_INCLUDE_DIR})
+include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR})
ADD_THIRDPARTY_LIB(openssl_ssl
SHARED_LIB "${OPENSSL_SSL_LIBRARY}")
ADD_THIRDPARTY_LIB(openssl_crypto