You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/11/06 20:15:35 UTC
hbase git commit: HBASE-19189 Ad-hoc test job for running a subset of
tests lots of times [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-19189 c0231a664 -> 0b66df4f5 (forced update)
HBASE-19189 Ad-hoc test job for running a subset of tests lots of times
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0b66df4f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0b66df4f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0b66df4f
Branch: refs/heads/HBASE-19189
Commit: 0b66df4f53e4b37447a5a9e7fdf0c0221f736691
Parents: 28cdf4a
Author: Sean Busbey <bu...@apache.org>
Authored: Mon Nov 6 13:48:05 2017 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Mon Nov 6 14:15:10 2017 -0600
----------------------------------------------------------------------
dev-support/adhoc_run_tests/Jenkinsfile | 78 +++++++++++++++++++
dev-support/adhoc_run_tests/adhoc_run_tests.sh | 83 +++++++++++++++++++++
dev-support/gather_machine_environment.sh | 45 +++++++++++
3 files changed, 206 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b66df4f/dev-support/adhoc_run_tests/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/adhoc_run_tests/Jenkinsfile b/dev-support/adhoc_run_tests/Jenkinsfile
new file mode 100644
index 0000000..429b03a
--- /dev/null
+++ b/dev-support/adhoc_run_tests/Jenkinsfile
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+ agent {
+ node {
+ label 'ubuntu'
+ }
+ }
+ options {
+ timeout (time: 6, unit: 'HOURS')
+ timestamps()
+ skipDefaultCheckout()
+ }
+ environment {
+ // where we check out to across stages
+ BASEDIR = "${env.WORKSPACE}/component"
+ OUTPUT_RELATIVE = 'output'
+ OUTPUTDIR = "${env.WORKSPACE}/output"
+ BRANCH_SPECIFIC_DOCKERFILE = "${env.BASEDIR}/dev-support/docker/Dockerfile"
+ }
+ stages {
+// stage ('scm checkout') {
+// steps {
+// dir('component') {
+// checkout scm
+// }
+// }
+// }
+ stage ('run tests') {
+ tools {
+ maven 'Maven (latest)'
+ // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
+ jdk "JDK 1.8 (latest)"
+ }
+ steps {
+ sh """#!/bin/bash -e
+ echo "Setting up directories"
+ rm -rf "${env.OUTPUTDIR}" && mkdir "${env.OUTPUTDIR}"
+ rm -rf ".m2-repo" && mkdir ".m2-repo"
+ mkdir "${env.OUTPUTDIR}/machine"
+"""
+ sh """#!/bin/bash -e
+ ${env.BASEDIR}/dev-support/gather_machine_environment.sh \
+ "${env.OUTPUT_RELATIVE}/machine"
+"""
+ sh """#!/bin/bash -e
+ ${env.BASEDIR}/dev-support/adhoc_run_tests/adhoc_run_tests.sh \
+ --force-timeout 1800 \
+ --maven-local-repo ".m2-repo" \
+ --log-output "${OUTPUTDIR}" \
+ --repeat 100 \
+ "${tests}"
+"""
+ }
+ post {
+ failure {
+ archive 'output/*'
+ archive 'output/**/*'
+ archive 'component/**/target/surefire-reports/*'
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b66df4f/dev-support/adhoc_run_tests/adhoc_run_tests.sh
----------------------------------------------------------------------
diff --git a/dev-support/adhoc_run_tests/adhoc_run_tests.sh b/dev-support/adhoc_run_tests/adhoc_run_tests.sh
new file mode 100755
index 0000000..3e61017
--- /dev/null
+++ b/dev-support/adhoc_run_tests/adhoc_run_tests.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] TestSomeTestName [TestOtherTest...]"
+ echo ""
+ echo " --force-timeout seconds Seconds to wait before killing a given test run."
+ echo " --maven-local-repo /path/to/use Path for maven artifacts while building"
+ echo " --repeat times number of times to repeat if successful"
+ echo " --log-output /path/to/use path to directory to hold attempt log"
+ exit 1
+}
+# Get arguments
+declare -i force_timeout=7200
+declare -i attempts=1
+declare maven_repo="${HOME}/.m2/repository"
+declare output="."
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --force-timeout) shift; force_timeout=$1; shift;;
+ --maven-local-repo) shift; maven_repo=$1; shift;;
+ --repeat) shift; attempts=$1; shift;;
+ --log-output) shift; output=$1; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+if [ "$#" -lt 1 ]; then
+ usage
+fi
+
+function find_module
+{
+ declare testname=$1
+ declare path
+ path=$(find . -name "${testname}.java")
+ while [ -n "${path}" ]; do
+ path=$(dirname "${path}")
+ if [ -f "${path}/pom.xml" ]; then
+ basename "${path}"
+ break
+ fi
+ done
+}
+
+declare -a modules
+
+for test in "${@}"; do
+ module=$(find_module "${test}")
+ if [[ ! "${modules[*]}" =~ ${module} ]]; then
+ modules+=(${module})
+ fi
+done
+
+declare -a mvn_module_arg
+
+for module in "${modules[@]}"; do
+ mvn_module_arg+=(-pl "${module}")
+done
+declare tests="${*}"
+for attempt in $(seq "${attempts}"); do
+ echo "Attempt ${attempt}" >&2
+ mvn --batch-mode -Dmaven.repo.local="${maven_repo}" -Dtest="${tests// /,}" -Dsurefire.rerunFailingTestsCount=0 -Dsurefire.parallel.forcedTimeout="${force_timeout}" -Dsurefire.shutdown=kill -DtrimStackTrace=false -am "${mvn_module_arg[@]}" package >"${output}/mvn_test.log" 2>&1
+done
http://git-wip-us.apache.org/repos/asf/hbase/blob/0b66df4f/dev-support/gather_machine_environment.sh
----------------------------------------------------------------------
diff --git a/dev-support/gather_machine_environment.sh b/dev-support/gather_machine_environment.sh
new file mode 100755
index 0000000..3bd96e3
--- /dev/null
+++ b/dev-support/gather_machine_environment.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} /path/for/output/dir"
+ echo ""
+ echo " Gather info about a build machine that test harnesses should poll before running."
+ echo " presumes you'll then archive the passed output dir."
+
+ exit 1
+}
+
+declare output=$1
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -l >"${output}/ulimit-l" 2>&1 || true