You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2016/01/13 17:57:51 UTC
yetus git commit: YETUS-280. build systems should be able to
predetermine module order
Repository: yetus
Updated Branches:
refs/heads/master 6ce890bcd -> e330e3fdf
YETUS-280. build systems should be able to predetermine module order
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e330e3fd
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e330e3fd
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e330e3fd
Branch: refs/heads/master
Commit: e330e3fdf111ce0ac22076db59cc2f34780a347c
Parents: 6ce890b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jan 12 14:06:37 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Jan 13 08:57:31 2016 -0800
----------------------------------------------------------------------
precommit/personality/hadoop.sh | 8 ---
precommit/test-patch.d/maven.sh | 130 +++++++++++++++++++++++++++++++++++
precommit/test-patch.sh | 8 ++-
3 files changed, 137 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh
index 0656491..0ccf137 100755
--- a/precommit/personality/hadoop.sh
+++ b/precommit/personality/hadoop.sh
@@ -63,15 +63,7 @@ function hadoop_module_manipulation
fi
done
- # ${startingmodules} is already sorted and uniq'd.
- # let's remove child modules if we're going to
- # touch their parent.
passed_modules=${startingmodules}
- for module in ${startingmodules}; do
- yetus_debug "Stripping ${module}"
- # shellcheck disable=SC2086
- passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ )
- done
yetus_debug "hmm pre-ordering: ${startingmodules}"
http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.d/maven.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/maven.sh b/precommit/test-patch.d/maven.sh
index 7068ea6..861c31c 100755
--- a/precommit/test-patch.d/maven.sh
+++ b/precommit/test-patch.d/maven.sh
@@ -15,6 +15,8 @@
# limitations under the License.
declare -a MAVEN_ARGS=("--batch-mode")
+declare -a MAVEN_ALREADY=()
+declare -a MAVEN_NEWLIST=()
if [[ -z "${MAVEN_HOME:-}" ]]; then
MAVEN=mvn
@@ -24,6 +26,7 @@ fi
MAVEN_CUSTOM_REPOS=false
MAVEN_CUSTOM_REPOS_DIR="${HOME}/yetus-m2"
+MAVEN_DEPENDENCY_ORDER=true
add_test_type mvnsite
add_test_type mvneclipse
@@ -34,6 +37,7 @@ function maven_usage
yetus_add_option "--mvn-cmd=<cmd>" "The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
yetus_add_option "--mvn-custom-repos" "Use per-project maven repos"
yetus_add_option "--mvn-custom-repos-dir=dir" "Location of repos, default is '${MAVEN_CUSTOM_REPOS_DIR}'"
+ yetus_add_option "--mvn-deps-order=<bool>" "Disable maven's auto-dependency module ordering (Default: '${MAVEN_DEPENDENCY_ORDER}')"
yetus_add_option "--mvn-settings=file" "File to use for settings.xml"
}
@@ -52,6 +56,9 @@ function maven_parse_args
--mvn-custom-repos-dir=*)
MAVEN_CUSTOM_REPOS_DIR=${i#*=}
;;
+ --mvn-deps-order=*)
+ MAVEN_DEPENDENCY_ORDER=${i#*=}
+ ;;
--mvn-settings=*)
MAVEN_SETTINGS=${i#*=}
if [[ -f ${MAVEN_SETTINGS} ]]; then
@@ -524,3 +531,126 @@ function maven_docker_support
>> "${PATCH_DIR}/buildtool-docker-params.txt"
fi
}
+
+## @description worker for maven reordering
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param repostatus
+## @param module
+function maven_reorder_module_process
+{
+ declare repostatus=$1
+ declare module=$2
+ declare indexd
+ declare indexm
+ declare deplist
+ declare fn
+ declare basemod
+
+ yetus_debug "Maven: check order: ${module}"
+
+ # this is a recursive function; no need to process
+ # modules that we have already seen
+ if [[ " ${MAVEN_ALREADY[@]} " =~ \ ${module}\ ]]; then
+ return
+ fi
+
+ # immediately place a module in the list to avoid
+ # loops
+ MAVEN_ALREADY=("${MAVEN_ALREADY[@]}" "${module}")
+
+ pushd "${BASEDIR}/${module}" >/dev/null
+ fn=$(module_file_fragment "${module}")
+
+ # get the dependency list from maven
+ #shellcheck disable=SC2046
+ echo_and_redirect "${PATCH_DIR}/maven-${repostatus}-dependencylist-${fn}.txt" \
+ $("${BUILDTOOL}_executor") dependency:list \
+ -DoutputFile="${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt"
+
+ # if this fails, there's really not much we can do other than tell the
+ # user that it did. So we'll press on and just hope for the best
+
+ if [[ $? != 0 ]]; then
+ add_vote_table -1 mvndep "${repostatus}'s ${module} dependency:list failed"
+ add_footer_table mvndep "${fn}: @@BASE@@/maven-${repostatus}-dependencylist-${fn}.txt"
+ fi
+
+ # take the list from maven, and pull out the artifactids. this means
+ # we have an assumption that the module names we are given map
+ # 1:1 to the artifactid
+ deplist=$(cut -f2 -d: "${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt")
+
+ # loop through the generated dependency artificatid list
+ # and try to match it to any of the other modules we are working on
+ # if it is a match, process it. dependencies that aren't being hit
+ # by the patch should be irrelevant since maven will pick them up
+ # from the repo caches. if a dependency really does matter, then
+ # the personality will have a chance to manipulate this list later
+ for indexd in ${deplist}; do
+ for indexm in ${CHANGED_MODULES}; do
+
+ # modules could be foo/bar, where bar is the artifactid
+ # so get the basename and compare that too
+ basemod=$(basename "${indexm}")
+ if [[ " ${indexd} " = " ${indexm} "
+ || " ${indexd} " = " ${basemod} " ]]; then
+ maven_reorder_module_process "${repostatus}" "${indexm}"
+ fi
+ done
+ done
+ popd >/dev/null
+
+ # at this point, any modules that we care about should have been placed
+ # before us. now add the one we've been working on. if the artificatids
+ # don't match, then we're effectively back to the old behavior of
+ # using fs globs and the personality will have to do the hard work
+ yetus_debug "Maven: placing ${module}"
+ MAVEN_NEWLIST=("${MAVEN_NEWLIST[@]}" " ${module} ")
+}
+
+## @description take a stab at reordering modules based upon
+## @description maven dependency order
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param repostatus
+## @param module
+function maven_reorder_modules
+{
+ declare repostatus=$1
+ declare index
+
+ # sidenote: this code does not get called when there is only one
+ # module, so no reason to shortcut that
+
+ if [[ "${MAVEN_DEPENDENCY_ORDER}" != "true" ]]; then
+ return
+ fi
+
+ big_console_header "Determining Maven Dependency Order"
+
+ start_clock
+
+ # this will get called again, so always make sure we start fresh
+ MAVEN_ALREADY=()
+ MAVEN_NEWLIST=()
+
+ yetus_debug "Maven: start re-ordering modules"
+ yetus_debug "Starting list: ${CHANGED_MODULES}"
+
+ # sort the list. parents will likely get us most of the
+ # dependency list sooner/more correct, esp if . is in the list
+ # shellcheck disable=SC2086
+ for index in $(echo ${CHANGED_MODULES} | tr ' ' '\n' | sort); do
+ maven_reorder_module_process "${repostatus}" "${index}"
+ done
+
+ CHANGED_MODULES="${MAVEN_NEWLIST[*]}"
+
+ yetus_debug "Maven: finish re-ordering modules"
+ yetus_debug "Finished list: ${CHANGED_MODULES}"
+
+ add_vote_table 0 mvndep "Maven dependency ordering for ${repostatus}"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index f2f9c59..a71c299 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -1026,9 +1026,11 @@ function module_skipdir
## @audience private
## @stability stable
## @replaceable no
+## @param repostatus
## @return None; sets ${CHANGED_MODULES}
function find_changed_modules
{
+ local repostatus=$1
local i
local changed_dirs
local builddirs
@@ -1115,6 +1117,10 @@ function find_changed_modules
#shellcheck disable=SC2034
CHANGED_UNION_MODULES="${builddir}"
+
+ if declare -f ${BUILDTOOL}_reorder_modules >/dev/null; then
+ "${BUILDTOOL}_reorder_modules" "${repostatus}"
+ fi
}
## @description git checkout the appropriate branch to test. Additionally, this calls
@@ -2506,7 +2512,7 @@ function compile_cycle
declare result=0
declare plugin
- find_changed_modules
+ find_changed_modules "${codebase}"
for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do
if declare -f ${plugin}_precompile >/dev/null 2>&1; then