You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2019/11/27 15:15:26 UTC
[hbase] 01/01: HBASE-23162 backport nightly pseudo-distributed
hbase test to branches-1.
This is an automated email from the ASF dual-hosted git repository.
busbey pushed a commit to branch HBASE-23162-branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ac52e3503fa3cb661cacb05650f581aee96132c8
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Thu Oct 10 17:01:01 2019 -0500
HBASE-23162 backport nightly pseudo-distributed hbase test to branches-1.
* Includes backport of HBASE-20334 and current state of updates to master branch for test script
* Changes specific to branch-1 APIs and expectations noted in test file
* skips hadoop 3 since we don't do hadoop 3 for branches-1 yet.
* skips using the hbase client tarball since branches-1 don't make one.
* pass location of shaded artifacts from build area, since branches-1 don't include them in the convenience tarball
---
dev-support/Jenkinsfile | 121 ++++-
.../hbase_nightly_pseudo-distributed-test.sh | 549 +++++++++++++++++++++
dev-support/hbase_nightly_source-artifact.sh | 14 +-
.../cache-apache-project-artifact.sh | 137 +++++
4 files changed, 789 insertions(+), 32 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index ea7d1c6..6966f93 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -57,8 +57,24 @@ pipeline {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
stages {
- stage ('yetus install') {
+ stage ('scm-checkout') {
steps {
+ dir('component') {
+ checkout scm
+ }
+ }
+ }
+ stage ('thirdparty installs') {
+ parallel {
+ stage ('yetus install') {
+ steps {
+ // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
+ dir('downloads-yetus') {
+ // can't just do a simple echo or the directory won't be created. :(
+ sh '''#!/usr/bin/env bash
+ echo "Make sure we have a directory for downloading dependencies: $(pwd)"
+'''
+ }
sh '''#!/usr/bin/env bash
set -e
echo "Ensure we have a copy of Apache Yetus."
@@ -66,25 +82,13 @@ if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
if ! "${YETUS_DIR}/bin/test-patch" --version >/dev/null 2>&1 ; then
- echo "New download of Apache Yetus version ${YETUS_RELEASE}."
rm -rf "${YETUS_DIR}"
- rm -rf "${WORKSPACE}/.gpg"
- mkdir -p "${WORKSPACE}/.gpg"
- chmod -R 700 "${WORKSPACE}/.gpg"
-
- echo "install yetus project KEYS"
- curl -L --fail -o "${WORKSPACE}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS
- gpg --homedir "${WORKSPACE}/.gpg" --import "${WORKSPACE}/KEYS_YETUS"
-
- echo "download yetus release ${YETUS_RELEASE}"
- curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
- curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
- echo "verifying yetus release"
- gpg --homedir "${WORKSPACE}/.gpg" --verify "apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
- mv "apache-yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
- else
- echo "Reusing cached download of Apache Yetus version ${YETUS_RELEASE}."
- fi
+ "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --working-dir "${WORKSPACE}/downloads-yetus" \
+ --keys 'https://www.apache.org/dist/yetus/KEYS' \
+ "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
+ "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
+ mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
else
YETUS_DIR="${WORKSPACE}/yetus-git"
rm -rf "${YETUS_DIR}"
@@ -104,8 +108,33 @@ set -e
echo "Downloading Project personality."
curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
"""
+ }
+ stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
+ }
+ }
+ stage ('hadoop 2 cache') {
+ environment {
+ HADOOP2_VERSION="2.8.5"
+ }
+ steps {
+ // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :(
+ dir('downloads-hadoop-2') {
+ sh '''#!/usr/bin/env bash
+ echo "Make sure we have a directory for downloading dependencies: $(pwd)"
+'''
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}"
+ "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
+ --working-dir "${WORKSPACE}/downloads-hadoop-2" \
+ --keys 'http://www.apache.org/dist/hadoop/common/KEYS' \
+ "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \
+ "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz"
+ '''
+ stash name: 'hadoop-2', includes: "hadoop-${HADOOP2_VERSION}-bin.tar.gz"
+ }
}
- stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
}
}
stage ('init health results') {
@@ -150,6 +179,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
'''
unstash 'yetus'
+ // since we have a new node definition we need to re-do the scm checkout
dir('component') {
checkout scm
}
@@ -441,7 +471,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
// This is meant to mimic what a release manager will do to create RCs.
// See http://hbase.apache.org/book.html#maven.release
- stage ('create source tarball') {
+ stage ('packaging and integration') {
tools {
maven 'Maven (latest)'
// this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
@@ -454,14 +484,15 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
sh '''#!/bin/bash -e
echo "Setting up directories"
rm -rf "output-srctarball" && mkdir "output-srctarball"
+ rm -rf "output-integration" && mkdir "output-integration" "output-integration/hadoop-2"
rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
+ rm -rf "hbase-install" && mkdir "hbase-install"
+ rm -rf "hadoop-2" && mkdir "hadoop-2"
rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
rm -rf ".m2-for-src" && mkdir ".m2-for-src"
- echo '(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console].' >output-srctarball/commentfile
+ echo "(x) {color:red}-1 source release artifact{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-srctarball/commentfile
+ echo "(x) {color:red}-1 client integration test{color}\n-- Something went wrong with this stage, [check relevant console output|${BUILD_URL}/console]." >output-integration/commentfile
'''
- dir('component') {
- checkout scm
- }
sh '''#!/usr/bin/env bash
set -e
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
@@ -470,6 +501,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
ls -lh "output-srctarball/machine"
'''
sh """#!/bin/bash -e
+ echo "Checking the steps for an RM to make a source artifact, then a binary artifact."
if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
--intermediate-file-dir output-srctarball \
--unpack-temp-dir unpacked_src_tarball \
@@ -483,12 +515,46 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
exit 1
fi
"""
+ echo "unpacking the hbase bin tarball into 'hbase-install'"
+ sh '''#!/bin/bash -e
+ if [ 1 -ne $(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | wc -l) ]; then
+ echo '(x) {color:red}-1 testing binary artifact{color}\n-- source tarball did not produce the expected binaries.' >>output-srctarball/commentfile
+ exit 1
+ fi
+ install_artifact=$(ls -1 "${WORKSPACE}"/unpacked_src_tarball/hbase-assembly/target/hbase-*-bin.tar.gz | sort | head -n 1)
+ tar --strip-component=1 -xzf "${install_artifact}" -C "hbase-install"
+'''
+ unstash 'hadoop-2'
+ echo "Attempting to use run an instance on top of Hadoop 2."
+ sh '''#!/bin/bash -xe
+ artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
+ tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --working-dir output-integration/hadoop-2 \
+ --hbase-shaded-server "${WORKSPACE}"/unpacked_src_tarball/hbase-shaded/hbase-shaded-server/target/hbase-shaded-server-*.jar \
+ --hbase-shaded-client "${WORKSPACE}"/hbase-shaded/hbase-shaded-client/target/hbase-shaded-client-*.jar \
+ "hbase-install" \
+ "hadoop-2/bin/hadoop" \
+ hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ >output-integration/hadoop-2.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
+ exit 2
+ fi
+'''
+ sh '''#!/bin/bash -e
+ echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
+'''
+
}
post {
always {
- stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
+ stash name: 'srctarball-result', includes: "output-srctarball/commentfile,output-integration/commentfile"
archiveArtifacts 'output-srctarball/*'
archiveArtifacts 'output-srctarball/**/*'
+ archiveArtifacts 'output-integration/*'
+ archiveArtifacts 'output-integration/**/*'
}
}
}
@@ -509,7 +575,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile",
- 'output-srctarball/commentfile']
+ 'output-srctarball/commentfile',
+ 'output-integration/commentfile']
echo env.BRANCH_NAME
echo env.BUILD_URL
echo currentBuild.result
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
new file mode 100755
index 0000000..6a8f4d8
--- /dev/null
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -0,0 +1,549 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/component/bin-install /path/to/hadoop/executable /path/to/hadoop/hadoop-yarn-server-tests-tests.jar /path/to/hadoop/hadoop-mapreduce-client-jobclient-tests.jar /path/to/mapred/executable"
+ echo ""
+ echo " --zookeeper-data /path/to/use Where the embedded zookeeper instance should write its data."
+ echo " defaults to 'zk-data' in the working-dir."
+ echo " --working-dir /path/to/use Path for writing configs and logs. must exist."
+ echo " defaults to making a directory via mktemp."
+ echo " --hadoop-client-classpath /path/to/some.jar:/path/to/another.jar classpath for hadoop jars."
+ echo " defaults to 'hadoop classpath'"
+ echo " --hbase-client-install /path/to/unpacked/client/tarball if given we'll look here for hbase client jars instead of the bin-install"
+ echo " --force-data-clean Delete all data in HDFS and ZK prior to starting up hbase"
+ echo " --single-process Run as single process instead of pseudo-distributed"
+ echo ""
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 5 ]; then
+ usage
+fi
+
+# Get arguments
+declare component_install
+declare hadoop_exec
+declare working_dir
+declare zk_data_dir
+declare clean
+declare distributed="true"
+declare hadoop_jars
+declare hbase_client
+declare hbase_shaded_mapreduce
+declare hbase_shaded_client
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --working-dir) shift; working_dir=$1; shift;;
+ --force-data-clean) shift; clean="true";;
+ --zookeeper-data) shift; zk_data_dir=$1; shift;;
+ --single-process) shift; distributed="false";;
+ --hadoop-client-classpath) shift; hadoop_jars="$1"; shift;;
+ --hbase-client-install) shift; hbase_client="$1"; shift;;
+ --hbase-shaded-server) shift; hbase_shaded_mapreduce="$1"; shift;;
+ --hbase-shaded-client) shift; hbase_shaded_client="$1"; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have where component checkout is.
+if [ $# -lt 5 ]; then
+ usage
+fi
+component_install="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+hadoop_exec="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")"
+yarn_server_tests_test_jar="$(cd "$(dirname "$3")"; pwd)/$(basename "$3")"
+mapred_jobclient_test_jar="$(cd "$(dirname "$4")"; pwd)/$(basename "$4")"
+mapred_exec="$(cd "$(dirname "$5")"; pwd)/$(basename "$5")"
+
+if [ ! -x "${hadoop_exec}" ]; then
+ echo "hadoop cli does not appear to be executable." >&2
+ exit 1
+fi
+
+if [ ! -x "${mapred_exec}" ]; then
+ echo "mapred cli does not appear to be executable." >&2
+ exit 1
+fi
+
+if [ ! -d "${component_install}" ]; then
+ echo "Path to HBase binary install should be a directory." >&2
+ exit 1
+fi
+
+if [ ! -f "${yarn_server_tests_test_jar}" ]; then
+ echo "Specified YARN server tests test jar is not a file." >&2
+ exit 1
+fi
+
+if [ ! -f "${mapred_jobclient_test_jar}" ]; then
+ echo "Specified MapReduce jobclient test jar is not a file." >&2
+ exit 1
+fi
+
+if [ -z "${working_dir}" ]; then
+ if ! working_dir="$(mktemp -d -t hbase-pseudo-dist-test)" ; then
+ echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
+ exit 1
+ fi
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist." >&2
+ exit 1
+ fi
+fi
+
+if [ -z "${zk_data_dir}" ]; then
+ zk_data_dir="${working_dir}/zk-data"
+ mkdir "${zk_data_dir}"
+else
+ # absolutes please
+ zk_data_dir="$(cd "$(dirname "${zk_data_dir}")"; pwd)/$(basename "${zk_data_dir}")"
+ if [ ! -d "${zk_data_dir}" ]; then
+ echo "passed directory for unpacking the source tarball '${zk_data_dir}' must already exist."
+ exit 1
+ fi
+fi
+
+if [ -z "${hbase_client}" ]; then
+ hbase_client="${component_install}"
+else
+ echo "Using HBase client-side artifact"
+ # absolutes please
+ hbase_client="$(cd "$(dirname "${hbase_client}")"; pwd)/$(basename "${hbase_client}")"
+ if [ ! -d "${hbase_client}" ]; then
+ echo "If given hbase client install should be a directory with contents of the client tarball." >&2
+ exit 1
+ fi
+fi
+
+if [ -n "${hadoop_jars}" ]; then
+ declare -a tmp_jars
+ for entry in $(echo "${hadoop_jars}" | tr ':' '\n'); do
+ tmp_jars=("${tmp_jars[@]}" "$(cd "$(dirname "${entry}")"; pwd)/$(basename "${entry}")")
+ done
+ hadoop_jars="$(IFS=:; echo "${tmp_jars[*]}")"
+fi
+
+if [ -n "${hbase_shaded_mapreduce}" ]; then
+ echo "Using HBase shaded artifact for mapreduce"
+ # absolutes please
+ hbase_shaded_mapreduce="$(cd "$(dirname "${hbase_shaded_mapreduce}")"; pwd)/$(basename "${hbase_shaded_mapreduce}")"
+ if [ ! -f "${hbase_shaded_mapreduce}" ]; then
+ echo "If given the HBase shaded artifact for mapreduce should be a plain file." >&2
+ exit 1
+ fi
+fi
+
+if [ -n "${hbase_shaded_client}" ]; then
+ echo "Using HBase shaded artifact for downstream clients"
+ # absolutes please
+ hbase_shaded_client="$(cd "$(dirname "${hbase_shaded_client}")"; pwd)/$(basename "${hbase_shaded_client}")"
+ if [ ! -f "${hbase_shaded_client}" ]; then
+ echo "If given the HBase shaded artifact for downstream clients should be a plain file." >&2
+ exit 1
+ fi
+fi
+
+
+echo "You'll find logs and temp files in ${working_dir}"
+
+function redirect_and_run {
+ log_base=$1
+ shift
+ echo "$*" >"${log_base}.err"
+ "$@" >"${log_base}.out" 2>>"${log_base}.err"
+}
+
+(cd "${working_dir}"
+
+echo "Hadoop version information:"
+"${hadoop_exec}" version
+hadoop_version=$("${hadoop_exec}" version | head -n 1)
+hadoop_version="${hadoop_version#Hadoop }"
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ "${hadoop_exec}" envvars
+else
+ echo "JAVA_HOME: ${JAVA_HOME}"
+fi
+
+# Ensure that if some other Hadoop install happens to be present in the environment we ignore it.
+HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true"
+export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP
+
+if [ -n "${clean}" ]; then
+ echo "Cleaning out ZooKeeper..."
+ rm -rf "${zk_data_dir:?}/*"
+fi
+
+echo "HBase version information:"
+"${component_install}/bin/hbase" version 2>/dev/null
+hbase_version=$("${component_install}/bin/hbase" version | head -n 1 2>/dev/null)
+hbase_version="${hbase_version#HBase }"
+
+if [ -z "${hbase_shaded_mapreduce}" ]; then
+ if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-server-${hbase_version}.jar" ]; then
+ echo "HBase binary install doesn't appear to include a shaded mapreduce artifact." >&2
+ exit 1
+ else
+ hbase_shaded_mapreduce="${hbase_client}/lib/shaded-clients/hbase-shaded-server-${hbase_version}.jar"
+ fi
+fi
+
+if [ -z "${hbase_shaded_client}" ]; then
+ if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-${hbase_version}.jar" ]; then
+ echo "HBase binary install doesn't appear to include a shaded client artifact." >&2
+ exit 1
+ else
+ hbase_shaded_client="${hbase_client}/lib/shaded-clients/hbase-shaded-client-${hbase_version}.jar"
+ fi
+fi
+
+echo "Writing out configuration for HBase."
+rm -rf "${working_dir}/hbase-conf"
+mkdir "${working_dir}/hbase-conf"
+
+if [ -f "${component_install}/conf/log4j.properties" ]; then
+ cp "${component_install}/conf/log4j.properties" "${working_dir}/hbase-conf/log4j.properties"
+else
+ cat >"${working_dir}/hbase-conf/log4j.properties" <<EOF
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+# console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+EOF
+fi
+
+cat >"${working_dir}/hbase-conf/hbase-site.xml" <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.rootdir</name>
+ <!-- We rely on the defaultFS being set in our hadoop confs -->
+ <value>/hbase</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.dataDir</name>
+ <value>${zk_data_dir}</value>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>${distributed}</value>
+ </property>
+</configuration>
+EOF
+
+if [ "true" = "${distributed}" ]; then
+ cat >"${working_dir}/hbase-conf/regionservers" <<EOF
+localhost
+EOF
+fi
+
+function cleanup {
+
+ echo "Shutting down HBase"
+ HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/stop-hbase.sh"
+
+ if [ -f "${working_dir}/hadoop.pid" ]; then
+ echo "Shutdown: listing HDFS contents"
+ redirect_and_run "${working_dir}/hadoop_listing_at_end" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
+
+ echo "Shutting down Hadoop"
+ kill -6 "$(cat "${working_dir}/hadoop.pid")"
+ fi
+}
+
+trap cleanup EXIT SIGQUIT
+
+echo "Starting up Hadoop"
+
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ "${mapred_exec}" minicluster -format -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" &
+else
+ HADOOP_CLASSPATH="${yarn_server_tests_test_jar}" "${hadoop_exec}" jar "${mapred_jobclient_test_jar}" minicluster -format -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" &
+fi
+
+echo "$!" > "${working_dir}/hadoop.pid"
+
+sleep_time=2
+until [ -s "${working_dir}/hbase-conf/core-site.xml" ]; do
+ printf '\twaiting for Hadoop to finish starting up.\n'
+ sleep "${sleep_time}"
+ sleep_time="$((sleep_time*2))"
+done
+
+if [ "${hadoop_version%.*.*}" -gt 2 ]; then
+ echo "Verifying configs"
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest
+fi
+
+if [ -n "${clean}" ]; then
+ echo "Cleaning out HDFS..."
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r /hbase
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example/
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example-region-listing.data
+fi
+
+echo "Listing HDFS contents"
+redirect_and_run "${working_dir}/hadoop_cluster_smoke" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
+
+echo "Starting up HBase"
+HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh"
+
+sleep_time=2
+until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 <<EOF
+ count 'hbase:meta'
+EOF
+do
+ printf '\tretry waiting for hbase to come up.\n'
+ sleep "${sleep_time}"
+ sleep_time="$((sleep_time*2))"
+done
+
+echo "Setting up table 'test:example' with 1,000 regions"
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/table_create.log" 2>&1 <<EOF
+ create_namespace 'test'
+ create 'test:example', 'family1', 'family2', {NUMREGIONS => 1000, SPLITALGO => 'UniformSplit'}
+EOF
+
+echo "writing out example TSV to example.tsv"
+cat >"${working_dir}/example.tsv" <<EOF
+row1 value8 value8
+row3 value2
+row2 value9
+row10 value1
+pow1 value8 value8
+pow3 value2
+pow2 value9
+pow10 value1
+paw1 value8 value8
+paw3 value2
+paw2 value9
+paw10 value1
+raw1 value8 value8
+raw3 value2
+raw2 value9
+raw10 value1
+aow1 value8 value8
+aow3 value2
+aow2 value9
+aow10 value1
+aaw1 value8 value8
+aaw3 value2
+aaw2 value9
+aaw10 value1
+how1 value8 value8
+how3 value2
+how2 value9
+how10 value1
+zow1 value8 value8
+zow3 value2
+zow2 value9
+zow10 value1
+zaw1 value8 value8
+zaw3 value2
+zaw2 value9
+zaw10 value1
+haw1 value8 value8
+haw3 value2
+haw2 value9
+haw10 value1
+low1 value8 value8
+low3 value2
+low2 value9
+low10 value1
+law1 value8 value8
+law3 value2
+law2 value9
+law10 value1
+EOF
+
+echo "uploading example.tsv to HDFS"
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -mkdir example
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyFromLocal "${working_dir}/example.tsv" "example/"
+
+echo "Importing TSV via shaded client artifact for HBase - MapReduce integration."
+# hbase_thirdparty_jars=("${component_install}"/lib/htrace-core4*.jar \
+# "${component_install}"/lib/slf4j-api-*.jar \
+# "${component_install}"/lib/commons-logging-*.jar \
+# "${component_install}"/lib/slf4j-log4j12-*.jar \
+# "${component_install}"/lib/log4j-1.2.*.jar \
+# "${working_dir}/hbase-conf/log4j.properties")
+# hbase_dep_classpath=$(IFS=:; echo "${hbase_thirdparty_jars[*]}")
+hbase_dep_classpath="$("${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" mapredcp)"
+# Work around HBASE-23147 by specifying the Driver
+HADOOP_CLASSPATH="${hbase_dep_classpath}" redirect_and_run "${working_dir}/mr-importtsv" \
+ "${hadoop_exec}" --config "${working_dir}/hbase-conf/" jar "${hbase_shaded_mapreduce}" org.apache.hadoop.hbase.mapreduce.Driver importtsv -Dimporttsv.columns=HBASE_ROW_KEY,family1:column1,family1:column4,family1:column3 test:example example/ -libjars "${hbase_dep_classpath}"
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_import.out" 2>"${working_dir}/scan_import.err" <<EOF
+ scan 'test:example'
+EOF
+
+echo "Verifying row count from import."
+import_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
+if [ ! "${import_rowcount}" -eq 48 ]; then
+ echo "ERROR: Instead of finding 48 rows, we found ${import_rowcount}."
+ exit 2
+fi
+
+if [ -z "${hadoop_jars}" ]; then
+ echo "Hadoop client jars not given; getting them from 'hadoop classpath' for the example."
+ hadoop_jars=$("${hadoop_exec}" --config "${working_dir}/hbase-conf/" classpath)
+fi
+
+echo "Building shaded client example."
+cat >"${working_dir}/HBaseClientReadWriteExample.java" <<EOF
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.RegionLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.util.LinkedList;
+import java.util.List;
+
+
+public class HBaseClientReadWriteExample {
+ private static final byte[] FAMILY_BYTES = Bytes.toBytes("family2");
+ private static final byte[] EMPTY = new byte[] {};
+
+ public static void main(String[] args) throws Exception {
+ Configuration hbase = HBaseConfiguration.create();
+ Configuration hadoop = new Configuration();
+ try (Connection connection = ConnectionFactory.createConnection(hbase)) {
+ System.out.println("Generating list of regions");
+ final List<String> regions = new LinkedList<>();
+ try (Admin admin = connection.getAdmin()) {
+ final ClusterStatus cluster = admin.getClusterStatus();
+ System.out.println(String.format("\tCluster reports version %s, ave load %f, region count %d", cluster.getHBaseVersion(), cluster.getAverageLoad(), cluster.getRegionsCount()));
+ for (ServerName server : cluster.getServers()) {
+ for (RegionLoad region : cluster.getLoad(server).getRegionsLoad().values()) {
+ regions.add(region.getNameAsString());
+ }
+ }
+ }
+ final Path listing = new Path("example-region-listing.data");
+ System.out.println("Writing list to HDFS");
+ try (FileSystem fs = FileSystem.newInstance(hadoop)) {
+ final Path path = fs.makeQualified(listing);
+ try (FSDataOutputStream out = fs.create(path)) {
+ out.writeInt(regions.size());
+ for (String region : regions) {
+ out.writeUTF(region);
+ }
+ out.hsync();
+ }
+ }
+ final List<Put> puts = new LinkedList<>();
+ final Put marker = new Put(new byte[] { (byte)0 });
+ System.out.println("Reading list from HDFS");
+ try (FileSystem fs = FileSystem.newInstance(hadoop)) {
+ final Path path = fs.makeQualified(listing);
+ try (FSDataInputStream in = fs.open(path)) {
+ final int count = in.readInt();
+ marker.addColumn(FAMILY_BYTES, Bytes.toBytes("count"), Bytes.toBytes(count));
+ for(int i = 0; i < count; i++) {
+ final byte[] row = Bytes.toBytes(in.readUTF());
+ final Put put = new Put(row);
+ put.addColumn(FAMILY_BYTES, EMPTY, EMPTY);
+ puts.add(put);
+ }
+ }
+ }
+ System.out.println("Writing list into HBase table");
+ try (Table table = connection.getTable(TableName.valueOf("test:example"))) {
+ table.put(marker);
+ table.put(puts);
+ }
+ }
+ }
+}
+EOF
+# The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-7190.
+# However we also have to work around the branch-1 equivalent of HBASE-20333. Since YARN-7190 got fixed and we're unlikely to further clean up
+# branches-1's shaded artifacts, put hadoop classes first
+redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
+ javac -cp "${hadoop_jars}:${hbase_shaded_client}" "${working_dir}/HBaseClientReadWriteExample.java"
+echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
+redirect_and_run "${working_dir}/hbase-shaded-client-example" \
+ java -cp "${working_dir}/hbase-conf/:${hadoop_jars}:${hbase_shaded_client}:${hbase_dep_classpath}:${working_dir}" HBaseClientReadWriteExample
+
+echo "Checking on results of example program."
+"${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
+
+"${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_example.out" 2>"${working_dir}/scan_example.err" <<EOF
+ scan 'test:example'
+EOF
+
+echo "Verifying row count from example."
+example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
+if [ "${example_rowcount}" -gt "1049" ]; then
+ echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 1 for example's use of meta region, and 1 for example's count record"
+else
+ echo "ERROR: Only found ${example_rowcount} rows."
+fi
+
+)
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index f3655e0..719a70f 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -175,9 +175,13 @@ echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarba
# N.B. Older JDK7 requires the TLSv1.2 in order to talk to maven central
if mvn -Dhttps.protocols=TLSv1.2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
- echo "Building a binary tarball from the source tarball succeeded."
-else
- echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details."
- exit 1
+ for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
+ if [ -f "${artifact}" ]; then
+ # TODO check the layout of the binary artifact we just made.
+ echo "Building a binary tarball from the source tarball succeeded."
+ exit 0
+ fi
+ done
fi
-# TODO check the layout of the binary artifact we just made.
+echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details."
+exit 1
diff --git a/dev-support/jenkins-scripts/cache-apache-project-artifact.sh b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
new file mode 100755
index 0000000..5653b05
--- /dev/null
+++ b/dev-support/jenkins-scripts/cache-apache-project-artifact.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
+ echo ""
+ echo " --force for a redownload even if /path/to/download/file.tar.gz exists."
+ echo " --working-dir /path/to/use Path for writing tempfiles. must exist."
+ echo " defaults to making a directory via mktemp that we clean."
+ echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download."
+ echo ""
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+
+# Get arguments
+declare done_if_cached="true"
+declare working_dir
+declare cleanup="true"
+declare keys
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --force) shift; done_if_cached="false";;
+ --working-dir) shift; working_dir=$1; cleanup="false"; shift;;
+ --keys) shift; keys=$1; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have required args
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+target="$1"
+artifact="$2"
+
+if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
+ echo "Reusing existing download of '${artifact}'."
+ exit 0
+fi
+
+if [ -z "${working_dir}" ]; then
+ if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then
+ echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
+ exit 1
+ fi
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist." >&2
+ exit 1
+ fi
+fi
+
+function cleanup {
+ if [ -n "${keys}" ]; then
+ echo "Stopping gpg agent daemon"
+ gpgconf --homedir "${working_dir}/.gpg" --kill gpg-agent
+ echo "Stopped gpg agent daemon"
+ fi
+
+ if [ "true" = "${cleanup}" ]; then
+ echo "cleaning up temp space."
+ rm -rf "${working_dir}"
+ fi
+}
+trap cleanup EXIT SIGQUIT
+
+echo "New download of '${artifact}'"
+
+# N.B. this comes first so that if gpg falls over we skip the expensive download.
+if [ -n "${keys}" ]; then
+ if [ ! -d "${working_dir}/.gpg" ]; then
+ rm -rf "${working_dir}/.gpg"
+ mkdir -p "${working_dir}/.gpg"
+ chmod -R 700 "${working_dir}/.gpg"
+ fi
+
+ echo "installing project KEYS"
+ curl -L --fail -o "${working_dir}/KEYS" "${keys}"
+ if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then
+ echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2
+ echo " gpg: can't connect to the agent: File name too long" >&2
+ # we mean to give them the command to run, not to run it.
+ #shellcheck disable=SC2016
+ echo 'then you prolly need to create /var/run/user/$(id -u)' >&2
+ echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2
+ exit 2
+ fi
+
+ echo "downloading signature"
+ curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc"
+fi
+
+echo "downloading artifact"
+if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then
+ echo "Artifact wasn't in mirror system. falling back to archive.a.o."
+ curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}"
+fi
+
+if [ -n "${keys}" ]; then
+ echo "verifying artifact signature"
+ gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc"
+ echo "signature good."
+fi
+
+echo "moving artifact into place at '${target}'"
+# ensure we're on the same filesystem
+mv "${working_dir}/artifact" "${target}.copying"
+# attempt atomic move
+mv "${target}.copying" "${target}"
+echo "all done!"