You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2023/06/16 08:15:47 UTC
[hbase-kustomize] branch main updated: HBASE-27935 Introduce Jenkins PR job for hbase-kustomize
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hbase-kustomize.git
The following commit(s) were added to refs/heads/main by this push:
new 781615f HBASE-27935 Introduce Jenkins PR job for hbase-kustomize
781615f is described below
commit 781615f230919cdcf42cc398cd1fc07a12ff2803
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Jun 15 14:29:23 2023 +0200
HBASE-27935 Introduce Jenkins PR job for hbase-kustomize
Copy over the structure and content from hbase-operator-tools. Strip out the Maven and Java
bits. Created a Jenkins job at https://ci-hbase.apache.org/job/hbase-kustomize-github-pr/
Signed-off-by: Sean Busbey <bu...@apache.org>
---
dev-support/jenkins/Dockerfile | 49 ++++++++
dev-support/jenkins/Jenkinsfile | 140 +++++++++++++++++++++
dev-support/jenkins/gather_machine_environment.sh | 57 +++++++++
.../jenkins/jenkins_precommit_github_yetus.sh | 127 +++++++++++++++++++
4 files changed, 373 insertions(+)
diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile
new file mode 100644
index 0000000..0fdc6cc
--- /dev/null
+++ b/dev-support/jenkins/Dockerfile
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for hbase-operator-tools pre-commit build.
+# https://ci-hbase.apache.org/job/HBase-Operator-Tools-PreCommit/
+
+ARG IMG_BASE='ubuntu'
+ARG IMG_BASE_TAG='22.04'
+
+FROM hadolint/hadolint:latest-debian as hadolint
+
+FROM ${IMG_BASE}:${IMG_BASE_TAG} as final
+ARG IMG_BASE
+ARG IMG_BASE_TAG
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# hadolint ignore=DL3008
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ binutils \
+ git \
+ rsync \
+ shellcheck \
+ wget && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY --from=hadolint /bin/hadolint /bin/hadolint
+
+CMD ["/bin/bash"]
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###
diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile
new file mode 100644
index 0000000..6d31cf5
--- /dev/null
+++ b/dev-support/jenkins/Jenkinsfile
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+ agent {
+ label 'hbase'
+ }
+
+ options {
+ // N.B. this is per-branch, which means per PR
+ disableConcurrentBuilds()
+ buildDiscarder(logRotator(numToKeepStr: '15'))
+ timeout (time: 1, unit: 'HOURS')
+ timestamps()
+ skipDefaultCheckout()
+ }
+
+ environment {
+ SRC_REL = 'src'
+ PATCH_REL = 'output'
+ YETUS_REL = 'yetus'
+ // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
+ YETUS_VERSION = 'rel/0.14.1'
+ DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile"
+ YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh"
+ ARCHIVE_PATTERN_LIST = 'TEST-*.xml'
+ BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+ WORKDIR_REL = 'yetus-precommit-check'
+ WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+ SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+ PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+ DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+ YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+ PLUGINS = 'all'
+ }
+
+ parameters {
+ booleanParam(name: 'DEBUG',
+ defaultValue: false,
+ description: 'Print extra outputs for debugging the jenkins job and yetus')
+ }
+
+ stages {
+ stage ('precommit checks') {
+ steps {
+ dir("${SOURCEDIR}") {
+ checkout scm
+ }
+ dir("${YETUSDIR}") {
+ checkout([
+ $class : 'GitSCM',
+ branches : [[name: "${YETUS_VERSION}"]],
+ userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
+ )
+ }
+ dir("${WORKDIR}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'apache-hbase-at-github.com',
+ passwordVariable: 'GITHUB_PASSWORD',
+ usernameVariable: 'GITHUB_USER'
+ )]) {
+ sh label: 'test-patch', script: '''#!/bin/bash -e
+ printenv 2>&1 | sort
+ echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
+ "${YETUS_DRIVER}"
+ '''
+ }
+ }
+ }
+ post {
+ always {
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+ reportFiles: 'report.html',
+ reportName: 'PR General Check Report'
+ ]
+ }
+ // Jenkins pipeline jobs fill slaves on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+ # See YETUS-764
+ if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to still be running: killing"
+ kill `cat "${PATCHDIR}/pidfile.txt"` || true
+ sleep 10
+ fi
+ if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to still be running: killing"
+ docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
+ fi
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ dir ("${WORKDIR}") {
+ deleteDir()
+ }
+ }
+ }
+ }
+ }
+ }
+
+ post {
+ // Jenkins pipeline jobs fill slaves on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ deleteDir()
+ }
+ }
+ }
+}
diff --git a/dev-support/jenkins/gather_machine_environment.sh b/dev-support/jenkins/gather_machine_environment.sh
new file mode 100644
index 0000000..cabd158
--- /dev/null
+++ b/dev-support/jenkins/gather_machine_environment.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} /path/for/output/dir"
+ echo ""
+ echo " Gather info about a build machine that test harnesses should poll before running."
+ echo " presumes you'll then archive the passed output dir."
+
+ exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+ usage
+fi
+
+
+declare output=$1
+
+if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then
+ echo "Specified output directory must exist and be writable." >&2
+ exit 1
+fi
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -a >"${output}/ulimit-a" 2>&1 || true
+uptime >"${output}/uptime" 2>&1 || true
+hostname -a >"${output}/hostname-a" 2>&1 || true
diff --git a/dev-support/jenkins/jenkins_precommit_github_yetus.sh b/dev-support/jenkins/jenkins_precommit_github_yetus.sh
new file mode 100644
index 0000000..896af25
--- /dev/null
+++ b/dev-support/jenkins/jenkins_precommit_github_yetus.sh
@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# place ourselves in the directory containing the hbase and yetus checkouts
+cd "$(dirname "$0")/../.."
+echo "executing from $(pwd)"
+
+if [[ "true" = "${DEBUG}" ]]; then
+ set -x
+ printenv 2>&1 | sort
+fi
+
+declare -i missing_env=0
+declare -a required_envs=(
+ # these ENV variables define the required API with Jenkinsfile_GitHub
+ "ARCHIVE_PATTERN_LIST"
+ "BUILD_URL_ARTIFACTS"
+ "DOCKERFILE"
+ "GITHUB_PASSWORD"
+ "GITHUB_USER"
+ "PATCHDIR"
+ "PLUGINS"
+ "SOURCEDIR"
+ "YETUSDIR"
+)
+# Validate params
+for required_env in "${required_envs[@]}"; do
+ if [ -z "${!required_env}" ]; then
+ echo "[ERROR] Required environment variable '${required_env}' is not set."
+ missing_env=${missing_env}+1
+ fi
+done
+
+if [ ${missing_env} -gt 0 ]; then
+ echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
+ "on Jenkins, then please file a JIRA about the error."
+ exit 1
+fi
+
+# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
+TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+
+# this must be clean for every run
+rm -rf "${PATCHDIR}"
+mkdir -p "${PATCHDIR}"
+
+# Gather machine information
+mkdir "${PATCHDIR}/machine"
+"${SOURCEDIR}/dev-support/jenkins/gather_machine_environment.sh" "${PATCHDIR}/machine"
+
+# If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
+# Otherwise exit, because we don't want HBase to do a
+# full build. We wouldn't normally do this check for smaller
+# projects. :)
+if [[ -z "${CHANGE_URL}" ]]; then
+ echo "Full build skipped" > "${PATCHDIR}/report.html"
+ exit 0
+fi
+# enable debug output for yetus
+if [[ "true" = "${DEBUG}" ]]; then
+ YETUS_ARGS+=("--debug")
+fi
+# If we're doing docker, make sure we don't accidentally pollute the image with a host java path
+if [ -n "${JAVA_HOME}" ]; then
+ unset JAVA_HOME
+fi
+YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+# where the source is located
+YETUS_ARGS+=("--basedir=${SOURCEDIR}")
+# lots of different output formats
+YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
+# enable writing back to Github
+YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+# auto-kill any surefire stragglers during unit test runs
+YETUS_ARGS+=("--reapermode=kill")
+# set relatively high limits for ASF machines
+# changing these to higher values may cause problems
+# with other jobs on systemd-enabled machines
+YETUS_ARGS+=("--dockermemlimit=20g")
+# -1 spotbugs issues that show up prior to the patch being applied
+YETUS_ARGS+=("--spotbugs-strict-precheck")
+# rsync these files back into the archive dir
+YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
+# URL for user-side presentation in reports and such to our artifacts
+YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}")
+# plugins to enable
+YETUS_ARGS+=("--plugins=${PLUGINS}")
+YETUS_ARGS+=("--tests-filter=test4tests")
+# run in docker mode and specifically point to our
+# Dockerfile since we don't want to use the auto-pulled version.
+YETUS_ARGS+=("--docker")
+YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+# effectively treat dev-support as a custom maven module
+YETUS_ARGS+=("--skip-dirs=dev-support")
+# help keep the ASF boxes clean
+YETUS_ARGS+=("--sentinel")
+# use emoji vote so it is easier to find the broken line
+YETUS_ARGS+=("--github-use-emoji-vote")
+YETUS_ARGS+=("--github-repo=apache/hbase-kustomize")
+# increasing proc limit to avoid OOME: unable to create native threads
+YETUS_ARGS+=("--proclimit=5000")
+
+
+echo "Launching yetus with command line:"
+echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
+
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"