You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/09/17 01:11:17 UTC
[hbase] branch branch-2.2 updated: HBASE-25021 Nightly job should
skip hadoop-2 integration test for master (#2409)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.2 by this push:
new b3485b4 HBASE-25021 Nightly job should skip hadoop-2 integration test for master (#2409)
b3485b4 is described below
commit b3485b42ac067c0e47f579cec940c13d31a50e92
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Thu Sep 17 08:48:11 2020 +0800
HBASE-25021 Nightly job should skip hadoop-2 integration test for master (#2409)
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
dev-support/Jenkinsfile | 104 ++++++++++++---------
.../hbase_nightly_pseudo-distributed-test.sh | 2 +-
2 files changed, 59 insertions(+), 47 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index cfc07dd..412a6cb 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -511,6 +511,7 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
+ BRANCH = "${env.BRANCH_NAME}"
}
steps {
sh '''#!/bin/bash -e
@@ -561,58 +562,69 @@ pipeline {
tar --strip-component=1 -xzf "${client_artifact}" -C "hbase-client"
'''
unstash 'hadoop-2'
- echo "Attempting to use run an instance on top of Hadoop 2."
sh '''#!/bin/bash -xe
- artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
- tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
- --single-process \
- --working-dir output-integration/hadoop-2 \
- --hbase-client-install "hbase-client" \
- "hbase-install" \
- "hadoop-2/bin/hadoop" \
- hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
- hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
- hadoop-2/bin/mapred \
- >output-integration/hadoop-2.log 2>&1 ; then
- echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
- exit 2
+ if [[ "${BRANCH}" = branch-2* ]] || [[ "${BRANCH}" = branch-1* ]]; then
+ echo "Attempting to use run an instance on top of Hadoop 2."
+ artifact=$(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | head -n 1)
+ tar --strip-components=1 -xzf "${artifact}" -C "hadoop-2"
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --working-dir output-integration/hadoop-2 \
+ --hbase-client-install "hbase-client" \
+ "hbase-install" \
+ "hadoop-2/bin/hadoop" \
+ hadoop-2/share/hadoop/yarn/timelineservice \
+ hadoop-2/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-2/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ hadoop-2/bin/mapred \
+ >output-integration/hadoop-2.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 2. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-2.log]. (note that this means we didn't run on Hadoop 3)" >output-integration/commentfile
+ exit 2
+ fi
+ else
+ echo "Skipping to run against Hadoop 2 for branch ${BRANCH}"
fi
'''
unstash 'hadoop-3'
- echo "Attempting to use run an instance on top of Hadoop 3."
sh '''#!/bin/bash -e
- artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
- tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
- --single-process \
- --working-dir output-integration/hadoop-3 \
- --hbase-client-install hbase-client \
- hbase-install \
- hadoop-3/bin/hadoop \
- hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
- hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
- hadoop-3/bin/mapred \
- >output-integration/hadoop-3.log 2>&1 ; then
- echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
- exit 2
- fi
- echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
- if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
- --single-process \
- --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
- --working-dir output-integration/hadoop-3-shaded \
- --hbase-client-install hbase-client \
- hbase-install \
- hadoop-3/bin/hadoop \
- hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
- hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
- hadoop-3/bin/mapred \
- >output-integration/hadoop-3-shaded.log 2>&1 ; then
- echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
- exit 2
+ if [[ "${BRANCH}" = branch-1* ]]; then
+ echo "Skipping to run against Hadoop 3 for branch ${BRANCH}"
+ else
+ echo "Attempting to use run an instance on top of Hadoop 3."
+ artifact=$(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | head -n 1)
+ tar --strip-components=1 -xzf "${artifact}" -C "hadoop-3"
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --working-dir output-integration/hadoop-3 \
+ --hbase-client-install hbase-client \
+ hbase-install \
+ hadoop-3/bin/hadoop \
+ hadoop-3/share/hadoop/yarn/timelineservice \
+ hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ hadoop-3/bin/mapred \
+ >output-integration/hadoop-3.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3.log]. (note that this means we didn't check the Hadoop 3 shaded client)" >output-integration/commentfile
+ exit 2
+ fi
+ echo "Attempting to use run an instance on top of Hadoop 3, relying on the Hadoop client artifacts for the example client program."
+ if ! "${BASEDIR}/dev-support/hbase_nightly_pseudo-distributed-test.sh" \
+ --single-process \
+ --hadoop-client-classpath hadoop-3/share/hadoop/client/hadoop-client-api-*.jar:hadoop-3/share/hadoop/client/hadoop-client-runtime-*.jar \
+ --working-dir output-integration/hadoop-3-shaded \
+ --hbase-client-install hbase-client \
+ hbase-install \
+ hadoop-3/bin/hadoop \
+ hadoop-3/share/hadoop/yarn/timelineservice \
+ hadoop-3/share/hadoop/yarn/test/hadoop-yarn-server-tests-*-tests.jar \
+ hadoop-3/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-*-tests.jar \
+ hadoop-3/bin/mapred \
+ >output-integration/hadoop-3-shaded.log 2>&1 ; then
+ echo "(x) {color:red}-1 client integration test{color}\n--Failed when running client tests on top of Hadoop 3 using Hadoop's shaded client. [see log for details|${BUILD_URL}/artifact/output-integration/hadoop-3-shaded.log]." >output-integration/commentfile
+ exit 2
+ fi
+ echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
fi
- echo "(/) {color:green}+1 client integration test{color}" >output-integration/commentfile
'''
diff --git a/dev-support/hbase_nightly_pseudo-distributed-test.sh b/dev-support/hbase_nightly_pseudo-distributed-test.sh
index 7324e6c..e51d272 100755
--- a/dev-support/hbase_nightly_pseudo-distributed-test.sh
+++ b/dev-support/hbase_nightly_pseudo-distributed-test.sh
@@ -175,7 +175,7 @@ fi
echo "HBase version information:"
"${component_install}/bin/hbase" version 2>/dev/null
-hbase_version=$("${component_install}/bin/hbase" version | head -n 1 2>/dev/null)
+hbase_version=$("${component_install}/bin/hbase" version 2>&1 | grep ^HBase | head -n 1)
hbase_version="${hbase_version#HBase }"
if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" ]; then