You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2020/08/18 03:13:14 UTC
[hadoop] branch branch-2.10 updated: HADOOP-17205. Move personality
file from Yetus to Hadoop repository (#2226)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new c7ab319 HADOOP-17205. Move personality file from Yetus to Hadoop repository (#2226)
c7ab319 is described below
commit c7ab3195de9a49a818cd68abb9ea20497ed178c9
Author: Chao Sun <su...@apache.org>
AuthorDate: Mon Aug 17 20:08:58 2020 -0700
HADOOP-17205. Move personality file from Yetus to Hadoop repository (#2226)
Signed-off-by: Akira Ajisaka <aa...@apache.org>
(cherry picked from commit fefacf2578e58cf7bf96d920f5b01dba66909ca2)
---
Jenkinsfile | 2 +-
dev-support/bin/hadoop.sh | 563 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 564 insertions(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 87396eb..eed3fa8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -97,8 +97,8 @@ pipeline {
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
# our project defaults come from a personality file
- # which will get loaded automatically by setting the project name
YETUS_ARGS+=("--project=hadoop")
+ YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
# lots of different output formats
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh
new file mode 100755
index 0000000..88b6005
--- /dev/null
+++ b/dev-support/bin/hadoop.sh
@@ -0,0 +1,563 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# SHELLDOC-IGNORE
+#
+# Override these to match Apache Hadoop's requirements
+personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=trunk
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute"
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO_DEFAULT="apache/hadoop"
+
+ HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)}
+ if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then
+ HADOOP_HOMEBREW_DIR=/usr/local
+ fi
+}
+
+function personality_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --hadoop-isal-prefix=*)
+ delete_parameter "${i}"
+ ISAL_HOME=${i#*=}
+ ;;
+ --hadoop-openssl-prefix=*)
+ delete_parameter "${i}"
+ OPENSSL_HOME=${i#*=}
+ ;;
+ --hadoop-snappy-prefix=*)
+ delete_parameter "${i}"
+ SNAPPY_HOME=${i#*=}
+ ;;
+ esac
+ done
+}
+
+## @description Calculate the actual module ordering
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_order
+{
+ declare ordering=$1
+ declare hadoopm
+
+ if [[ ${ordering} = normal ]]; then
+ hadoopm="${CHANGED_MODULES[*]}"
+ elif [[ ${ordering} = union ]]; then
+ hadoopm="${CHANGED_UNION_MODULES}"
+ elif [[ ${ordering} = mvnsrc ]]; then
+ hadoopm="${MAVEN_SRC_MODULES[*]}"
+ elif [[ ${ordering} = mvnsrctest ]]; then
+ hadoopm="${MAVEN_SRCTEST_MODULES[*]}"
+ else
+ hadoopm="${ordering}"
+ fi
+ echo "${hadoopm}"
+}
+
+## @description Determine if it is safe to run parallel tests
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_test_parallel
+{
+ if [[ -f "${BASEDIR}/pom.xml" ]]; then
+ HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
+ | head -1 \
+ | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
+ | cut -f1 -d- )
+ export HADOOP_VERSION
+ else
+ return 1
+ fi
+
+ hmajor=${HADOOP_VERSION%%\.*}
+ hmajorminor=${HADOOP_VERSION%\.*}
+ hminor=${hmajorminor##*\.}
+ # ... and just for reference
+ #hmicro=${HADOOP_VERSION##*\.}
+
+ # Apache Hadoop v2.8.0 was the first one to really
+ # get working parallel unit tests
+ if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then
+ return 1
+ fi
+
+ return 0
+}
+
+## @description Install extra modules for unit tests
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_unittest_prereqs
+{
+ declare input=$1
+ declare mods
+ declare need_common=0
+ declare building_common=0
+ declare module
+ declare flags
+ declare fn
+
+ # prior to running unit tests, hdfs needs libhadoop.so built
+ # if we're building root, then this extra work is moot
+
+ #shellcheck disable=SC2086
+ mods=$(hadoop_order ${input})
+
+ for module in ${mods}; do
+ if [[ ${module} = hadoop-hdfs-project* ]]; then
+ need_common=1
+ elif [[ ${module} = hadoop-common-project/hadoop-common
+ || ${module} = hadoop-common-project ]]; then
+ building_common=1
+ elif [[ ${module} = . ]]; then
+ return
+ fi
+ done
+
+ # Windows builds *ALWAYS* need hadoop-common compiled
+ case ${OSTYPE} in
+ Windows_NT|CYGWIN*|MINGW*|MSYS*)
+ need_common=1
+ ;;
+ esac
+
+ if [[ ${need_common} -eq 1
+ && ${building_common} -eq 0 ]]; then
+ echo "unit test pre-reqs:"
+ module="hadoop-common-project/hadoop-common"
+ fn=$(module_file_fragment "${module}")
+ flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
+ pushd "${BASEDIR}/${module}" >/dev/null || return 1
+ # shellcheck disable=SC2086
+ echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
+ popd >/dev/null || return 1
+ fi
+}
+
+## @description Calculate the flags/settings for yarn-ui v2 build
+## @description based upon the OS
+## @audience private
+## @stability evolving
+function yarn_ui2_flag
+{
+
+ if [[ ${BUILD_NATIVE} != true ]]; then
+ return
+ fi
+
+ # Now it only tested on Linux/OSX, don't enable the profile on
+ # windows until it get verified
+ case ${OSTYPE} in
+ Linux)
+ # shellcheck disable=SC2086
+ echo -Pyarn-ui
+ ;;
+ Darwin)
+ echo -Pyarn-ui
+ ;;
+ *)
+ # Do nothing
+ ;;
+ esac
+}
+
+## @description Calculate the flags/settings for native code
+## @description based upon the OS
+## @audience private
+## @stability evolving
+function hadoop_native_flags
+{
+ if [[ ${BUILD_NATIVE} != true ]]; then
+ return
+ fi
+
+ declare -a args
+
+ # Based upon HADOOP-11937
+ #
+ # Some notes:
+ #
+ # - getting fuse to compile on anything but Linux
+ # is always tricky.
+ # - Darwin assumes homebrew is in use.
+ # - HADOOP-12027 required for bzip2 on OS X.
+ # - bzip2 is broken in lots of places
+ # (the shared library is considered experimental)
+ # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
+ #
+
+ args=("-Drequire.test.libhadoop")
+
+ if [[ -d "${ISAL_HOME}/include" ]]; then
+ args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}")
+ fi
+
+ if [[ -d "${OPENSSL_HOME}/include" ]]; then
+ args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}")
+ elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then
+ args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/")
+ fi
+
+ if [[ -d "${SNAPPY_HOME}/include" ]]; then
+ args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}")
+ elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then
+ args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy")
+ fi
+
+ case ${OSTYPE} in
+ Linux)
+ # shellcheck disable=SC2086
+ echo \
+ -Pnative \
+ -Drequire.fuse \
+ -Drequire.openssl \
+ -Drequire.snappy \
+ -Drequire.valgrind \
+ -Drequire.zstd \
+ "${args[@]}"
+ ;;
+ Darwin)
+ echo \
+ "${args[@]}" \
+ -Pnative \
+ -Drequire.snappy \
+ -Drequire.openssl
+ ;;
+ Windows_NT|CYGWIN*|MINGW*|MSYS*)
+ echo \
+ "${args[@]}" \
+ -Drequire.snappy -Drequire.openssl -Pnative-win
+ ;;
+ *)
+ echo \
+ "${args[@]}"
+ ;;
+ esac
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ declare repostatus=$1
+ declare testtype=$2
+ declare extra=""
+ declare ordering="normal"
+ declare needflags=false
+ declare foundbats=false
+ declare flags
+ declare fn
+ declare i
+ declare hadoopm
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ asflicense)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ checkstyle)
+ ordering="union"
+ extra="-DskipTests"
+ ;;
+ compile)
+ ordering="union"
+ extra="-DskipTests"
+ needflags=true
+
+ # if something in common changed, we build the whole world
+ if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then
+ yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
+ ordering="."
+ fi
+ ;;
+ distclean)
+ ordering="."
+ extra="-DskipTests"
+ ;;
+ javadoc)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+
+ if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then
+ echo "javadoc pre-reqs:"
+ for i in hadoop-project \
+ hadoop-common-project/hadoop-annotations; do
+ fn=$(module_file_fragment "${i}")
+ pushd "${BASEDIR}/${i}" >/dev/null || return 1
+ echo "cd ${i}"
+ echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install
+ popd >/dev/null || return 1
+ done
+ fi
+ extra="-Pdocs -DskipTests"
+ ;;
+ mvneclipse)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+ ;;
+ mvninstall)
+ extra="-DskipTests"
+ if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then
+ ordering=.
+ fi
+ ;;
+ mvnsite)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+ ;;
+ unit)
+ if [[ "${BUILDMODE}" = full ]]; then
+ ordering=mvnsrc
+ elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+
+ if [[ ${TEST_PARALLEL} = "true" ]] ; then
+ if hadoop_test_parallel; then
+ extra="-Pparallel-tests"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ fi
+ needflags=true
+ hadoop_unittest_prereqs "${ordering}"
+
+ if ! verify_needed_test javac; then
+ yetus_debug "hadoop: javac not requested"
+ if ! verify_needed_test native; then
+ yetus_debug "hadoop: native not requested"
+ yetus_debug "hadoop: adding -DskipTests to unit test"
+ extra="-DskipTests"
+ fi
+ fi
+
+ for i in "${CHANGED_FILES[@]}"; do
+ if [[ "${i}" =~ \.bats ]]; then
+ foundbats=true
+ fi
+ done
+
+ if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then
+ yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
+ extra="${extra} -P!shelltest"
+ else
+ extra="${extra} -Pshelltest"
+ fi
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ if [[ ${needflags} = true ]]; then
+ flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
+ extra="${extra} ${flags}"
+ fi
+
+ extra="-Ptest-patch ${extra}"
+ for module in $(hadoop_order ${ordering}); do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+## @description Add tests based upon personality needs
+## @audience private
+## @stability evolving
+## @param filename
+function personality_file_tests
+{
+ declare filename=$1
+
+ yetus_debug "Using Hadoop-specific personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ add_test shadedclient
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ || ${filename} =~ src/scripts
+ || ${filename} =~ src/test/scripts
+ || ${filename} =~ src/main/bin
+ || ${filename} =~ shellprofile\.d
+ || ${filename} =~ src/main/conf
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ add_test mvnsite
+ add_test unit
+ elif [[ ${filename} =~ \.md$
+ || ${filename} =~ \.md\.vm$
+ || ${filename} =~ src/site
+ ]]; then
+ yetus_debug "tests/site: ${filename}"
+ add_test mvnsite
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test compile
+ add_test cc
+ add_test mvnsite
+ add_test javac
+ add_test unit
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ || ${filename} =~ src/main
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test compile
+ add_test javac
+ add_test javadoc
+ add_test mvninstall
+ add_test mvnsite
+ add_test unit
+ add_test shadedclient
+ fi
+
+ # if we change anything in here, e.g. the test scripts
+ # then run the client artifact tests
+ if [[ ${filename} =~ hadoop-client-modules ]]; then
+ add_test shadedclient
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests: src/test"
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
+
+## @description Image to print on success
+## @audience private
+## @stability evolving
+function hadoop_console_success
+{
+ printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
+ printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
+ printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
+ printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
+ printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
+ printf "ICB8bXwgICB8bXwgIAo"
+}
+
+###################################################
+# Hadoop project specific check of IT for shaded artifacts
+
+add_test_type shadedclient
+
+## @description check for test modules and add test/plugins as needed
+## @audience private
+## @stability evolving
+function shadedclient_initialize
+{
+ maven_add_install shadedclient
+}
+
+## @description build client facing shaded artifacts and test them
+## @audience private
+## @stability evolving
+## @param repostatus
+function shadedclient_rebuild
+{
+ declare repostatus=$1
+ declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt"
+ declare module
+ declare -a modules=()
+
+ if [[ ${OSTYPE} = Windows_NT ||
+ ${OSTYPE} =~ ^CYGWIN.* ||
+ ${OSTYPE} =~ ^MINGW32.* ||
+ ${OSTYPE} =~ ^MSYS.* ]]; then
+ echo "hadoop personality: building on windows, skipping check of client artifacts."
+ return 0
+ fi
+
+ yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
+ for module in hadoop-client-modules/hadoop-client-check-invariants \
+ hadoop-client-modules/hadoop-client-check-test-invariants \
+ hadoop-client-modules/hadoop-client-integration-tests; do
+ if [ -d "${module}" ]; then
+ yetus_debug "hadoop personality: test module '${module}' is present."
+ modules+=(-pl "${module}")
+ fi
+ done
+ if [ ${#modules[@]} -eq 0 ]; then
+ echo "hadoop personality: no test modules present, skipping check of client artifacts."
+ return 0
+ fi
+
+ big_console_header "Checking client artifacts on ${repostatus}"
+
+ echo_and_redirect "${logfile}" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
+ "${modules[@]}" \
+ -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
+
+ count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."
+ return 1
+ fi
+
+ add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts."
+ return 0
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org