You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/03/04 05:58:36 UTC
[2/3] hadoop git commit: HADOOP-12651. Replace dev-support with
wrappers to Yetus (aw)
HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)
(cherry picked from commit 8cecad2d564d89d0f56e98de80a6f9b0bcb39d7a)
Conflicts:
dev-support/test-patch.sh
hadoop-common-project/hadoop-common/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0d3b0c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0d3b0c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0d3b0c1
Branch: refs/heads/branch-2
Commit: b0d3b0c1f86469d1f16a79a2b06f060c3e6f1c26
Parents: f9f5726
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jan 21 12:22:03 2016 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Mar 3 20:54:08 2016 -0800
----------------------------------------------------------------------
BUILDING.txt | 2 +-
dev-support/releasedocmaker.py | 580 -----
dev-support/smart-apply-patch.sh | 187 --
dev-support/test-patch.d/checkstyle.sh | 205 --
dev-support/test-patch.d/shellcheck.sh | 178 --
dev-support/test-patch.d/whitespace.sh | 46 -
dev-support/test-patch.sh | 2808 ----------------------
hadoop-common-project/hadoop-common/pom.xml | 18 +-
8 files changed, 15 insertions(+), 4009 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 6e20a42..a63553b 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -37,7 +37,7 @@ and all required tools for testing and building have been installed and configur
Note that from within this docker environment you ONLY have access to the Hadoop source
tree from where you started. So if you need to run
- dev-support/test-patch.sh /path/to/my.patch
+ dev-support/bin/test-patch /path/to/my.patch
then the patch must be placed inside the hadoop source tree.
Known issues:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
deleted file mode 100755
index d2e5dda..0000000
--- a/dev-support/releasedocmaker.py
+++ /dev/null
@@ -1,580 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from glob import glob
-from optparse import OptionParser
-from time import gmtime, strftime
-import pprint
-import os
-import re
-import sys
-import urllib
-import urllib2
-try:
- import json
-except ImportError:
- import simplejson as json
-
-releaseVersion={}
-namePattern = re.compile(r' \([0-9]+\)')
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def clean(str):
- return tableclean(re.sub(namePattern, "", str))
-
-def formatComponents(str):
- str = re.sub(namePattern, '', str).replace("'", "")
- if str != "":
- ret = str
- else:
- # some markdown parsers don't like empty tables
- ret = "."
- return clean(ret)
-
-# convert to utf-8
-# protect some known md metachars
-# or chars that screw up doxia
-def tableclean(str):
- str=str.encode('utf-8')
- str=str.replace("_","\_")
- str=str.replace("\r","")
- str=str.rstrip()
- return str
-
-# same thing as tableclean,
-# except table metachars are also
-# escaped as well as more
-# things we don't want doxia to
-# screw up
-def notableclean(str):
- str=tableclean(str)
- str=str.replace("|","\|")
- str=str.replace("<","\<")
- str=str.replace(">","\>")
- str=str.replace("*","\*")
- str=str.rstrip()
- return str
-
-# clean output dir
-def cleanOutputDir(dir):
- files = os.listdir(dir)
- for name in files:
- os.remove(os.path.join(dir,name))
- os.rmdir(dir)
-
-def mstr(obj):
- if (obj is None):
- return ""
- return unicode(obj)
-
-def buildindex(title,license):
- versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
- with open("index.md","w") as indexfile:
- if license is True:
- indexfile.write(asflicense)
- for v in versions:
- indexfile.write("* %s v%s\n" % (title,v))
- for k in ("Changes","Release Notes"):
- indexfile.write(" * %s (%s/%s.%s.html)\n" \
- % (k,v,k.upper().replace(" ",""),v))
- indexfile.close()
-
-class GetVersions:
- """ yo """
- def __init__(self,versions, projects):
- versions = versions
- projects = projects
- self.newversions = []
- pp = pprint.PrettyPrinter(indent=4)
- at=0
- end=1
- count=100
- versions.sort()
- print "Looking for %s through %s"%(versions[0],versions[-1])
- for p in projects:
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
- data = json.loads(resp.read())
- for d in data:
- if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
- print "Adding %s to the list" % d['name']
- self.newversions.append(d['name'])
- newlist=list(set(self.newversions))
- self.newversions=newlist
-
- def getlist(self):
- pp = pprint.PrettyPrinter(indent=4)
- return(self.newversions)
-
-class Version:
- """Represents a version number"""
- def __init__(self, data):
- self.mod = False
- self.data = data
- found = re.match('^((\d+)(\.\d+)*).*$', data)
- if (found):
- self.parts = [ int(p) for p in found.group(1).split('.') ]
- else:
- self.parts = []
- # backfill version with zeroes if missing parts
- self.parts.extend((0,) * (3 - len(self.parts)))
-
- def __str__(self):
- if (self.mod):
- return '.'.join([ str(p) for p in self.parts ])
- return self.data
-
- def __cmp__(self, other):
- return cmp(self.parts, other.parts)
-
-class Jira:
- """A single JIRA"""
-
- def __init__(self, data, parent):
- self.key = data['key']
- self.fields = data['fields']
- self.parent = parent
- self.notes = None
- self.incompat = None
- self.reviewed = None
-
- def getId(self):
- return mstr(self.key)
-
- def getDescription(self):
- return mstr(self.fields['description'])
-
- def getReleaseNote(self):
- if (self.notes is None):
- field = self.parent.fieldIdMap['Release Note']
- if (self.fields.has_key(field)):
- self.notes=mstr(self.fields[field])
- else:
- self.notes=self.getDescription()
- return self.notes
-
- def getPriority(self):
- ret = ""
- pri = self.fields['priority']
- if(pri is not None):
- ret = pri['name']
- return mstr(ret)
-
- def getAssignee(self):
- ret = ""
- mid = self.fields['assignee']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getComponents(self):
- if (len(self.fields['components'])>0):
- return ", ".join([ comp['name'] for comp in self.fields['components'] ])
- else:
- return ""
-
- def getSummary(self):
- return self.fields['summary']
-
- def getType(self):
- ret = ""
- mid = self.fields['issuetype']
- if(mid is not None):
- ret = mid['name']
- return mstr(ret)
-
- def getReporter(self):
- ret = ""
- mid = self.fields['reporter']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getProject(self):
- ret = ""
- mid = self.fields['project']
- if(mid is not None):
- ret = mid['key']
- return mstr(ret)
-
- def __cmp__(self,other):
- selfsplit=self.getId().split('-')
- othersplit=other.getId().split('-')
- v1=cmp(selfsplit[0],othersplit[0])
- if (v1!=0):
- return v1
- else:
- if selfsplit[1] < othersplit[1]:
- return True
- elif selfsplit[1] > othersplit[1]:
- return False
- return False
-
- def getIncompatibleChange(self):
- if (self.incompat is None):
- field = self.parent.fieldIdMap['Hadoop Flags']
- self.reviewed=False
- self.incompat=False
- if (self.fields.has_key(field)):
- if self.fields[field]:
- for hf in self.fields[field]:
- if hf['value'] == "Incompatible change":
- self.incompat=True
- if hf['value'] == "Reviewed":
- self.reviewed=True
- return self.incompat
-
- def checkMissingComponent(self):
- if (len(self.fields['components'])>0):
- return False
- return True
-
- def checkMissingAssignee(self):
- if (self.fields['assignee'] is not None):
- return False
- return True
-
- def checkVersionString(self):
- field = self.parent.fieldIdMap['Fix Version/s']
- for h in self.fields[field]:
- found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
- if not found:
- return True
- return False
-
- def getReleaseDate(self,version):
- for j in range(len(self.fields['fixVersions'])):
- if self.fields['fixVersions'][j]==version:
- return(self.fields['fixVersions'][j]['releaseDate'])
- return None
-
-class JiraIter:
- """An Iterator of JIRAs"""
-
- def __init__(self, version, projects):
- self.version = version
- self.projects = projects
- v=str(version).replace("-SNAPSHOT","")
-
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
- data = json.loads(resp.read())
-
- self.fieldIdMap = {}
- for part in data:
- self.fieldIdMap[part['name']] = part['id']
-
- self.jiras = []
- at=0
- end=1
- count=100
- while (at < end):
- params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
- data = json.loads(resp.read())
- if (data.has_key('errorMessages')):
- raise Exception(data['errorMessages'])
- at = data['startAt'] + data['maxResults']
- end = data['total']
- self.jiras.extend(data['issues'])
-
- needaversion=False
- if v not in releaseVersion:
- needaversion=True
-
- if needaversion is True:
- for i in range(len(data['issues'])):
- for j in range(len(data['issues'][i]['fields']['fixVersions'])):
- if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
- releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
- data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
-
- self.iter = self.jiras.__iter__()
-
- def __iter__(self):
- return self
-
- def next(self):
- data = self.iter.next()
- j = Jira(data, self)
- return j
-
-class Outputs:
- """Several different files to output to at the same time"""
-
- def __init__(self, base_file_name, file_name_pattern, keys, params={}):
- self.params = params
- self.base = open(base_file_name%params, 'w')
- self.others = {}
- for key in keys:
- both = dict(params)
- both['key'] = key
- self.others[key] = open(file_name_pattern%both, 'w')
-
- def writeAll(self, pattern):
- both = dict(self.params)
- both['key'] = ''
- self.base.write(pattern%both)
- for key in self.others.keys():
- both = dict(self.params)
- both['key'] = key
- self.others[key].write(pattern%both)
-
- def writeKeyRaw(self, key, str):
- self.base.write(str)
- if (self.others.has_key(key)):
- self.others[key].write(str)
-
- def close(self):
- self.base.close()
- for fd in self.others.values():
- fd.close()
-
- def writeList(self, mylist):
- for jira in sorted(mylist):
- line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()),
- notableclean(jira.getSummary()),
- notableclean(jira.getPriority()),
- formatComponents(jira.getComponents()),
- notableclean(jira.getReporter()),
- notableclean(jira.getAssignee()))
- self.writeKeyRaw(jira.getProject(), line)
-
-def main():
- parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
- epilog=
- "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
- " named after the highest version provided.")
- parser.add_option("-i","--index", dest="index", action="store_true",
- default=False, help="build an index file")
- parser.add_option("-l","--license", dest="license", action="store_false",
- default=True, help="Add an ASF license")
- parser.add_option("-n","--lint", dest="lint", action="store_true",
- help="use lint flag to exit on failures")
- parser.add_option("-p", "--project", dest="projects",
- action="append", type="string",
- help="projects in JIRA to include in releasenotes", metavar="PROJECT")
- parser.add_option("-r", "--range", dest="range", action="store_true",
- default=False, help="Given versions are a range")
- parser.add_option("-t", "--projecttitle", dest="title",
- type="string",
- help="Title to use for the project (default is Apache PROJECT)")
- parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
- default=False, help="use current date for unreleased versions")
- parser.add_option("-v", "--version", dest="versions",
- action="append", type="string",
- help="versions in JIRA to include in releasenotes", metavar="VERSION")
- (options, args) = parser.parse_args()
-
- if (options.versions is None):
- options.versions = []
-
- if (len(args) > 2):
- options.versions.append(args[2])
-
- if (len(options.versions) <= 0):
- parser.error("At least one version needs to be supplied")
-
- proxy = urllib2.ProxyHandler()
- opener = urllib2.build_opener(proxy)
- urllib2.install_opener(opener)
-
- projects = options.projects
-
- if (options.range is True):
- versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
- else:
- versions = [ Version(v) for v in options.versions ]
- versions.sort();
-
- if (options.title is None):
- title=projects[0]
- else:
- title=options.title
-
- haderrors=False
-
- for v in versions:
- vstr=str(v)
- jlist = JiraIter(vstr,projects)
-
- if vstr in releaseVersion:
- reldate=releaseVersion[vstr]
- elif options.usetoday:
- reldate=strftime("%Y-%m-%d", gmtime())
- else:
- reldate="Unreleased"
-
- if not os.path.exists(vstr):
- os.mkdir(vstr)
-
- reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
- "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
- choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
- "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
-
- if (options.license is True):
- reloutputs.writeAll(asflicense)
- choutputs.writeAll(asflicense)
-
- relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
- 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
- chhead = '# %(title)s Changelog\n\n' \
- '## Release %(ver)s - %(date)s\n'\
- '\n'
-
- reloutputs.writeAll(relhead)
- choutputs.writeAll(chhead)
- errorCount=0
- warningCount=0
- lintMessage=""
- incompatlist=[]
- buglist=[]
- improvementlist=[]
- newfeaturelist=[]
- subtasklist=[]
- tasklist=[]
- testlist=[]
- otherlist=[]
-
- for jira in sorted(jlist):
- if jira.getIncompatibleChange():
- incompatlist.append(jira)
- elif jira.getType() == "Bug":
- buglist.append(jira)
- elif jira.getType() == "Improvement":
- improvementlist.append(jira)
- elif jira.getType() == "New Feature":
- newfeaturelist.append(jira)
- elif jira.getType() == "Sub-task":
- subtasklist.append(jira)
- elif jira.getType() == "Task":
- tasklist.append(jira)
- elif jira.getType() == "Test":
- testlist.append(jira)
- else:
- otherlist.append(jira)
-
- line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
- notableclean(jira.getSummary()))
-
- if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
- warningCount+=1
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
- lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if jira.checkVersionString():
- warningCount+=1
- lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
-
- if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
- errorCount+=1
- errorMessage=[]
- jira.checkMissingComponent() and errorMessage.append("component")
- jira.checkMissingAssignee() and errorMessage.append("assignee")
- lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
-
- if (len(jira.getReleaseNote())>0):
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if (options.lint is True):
- print lintMessage
- print "======================================="
- print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
- if (errorCount>0):
- haderrors=True
- cleanOutputDir(vstr)
- continue
-
- reloutputs.writeAll("\n\n")
- reloutputs.close()
-
- choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(incompatlist)
-
- choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(newfeaturelist)
-
- choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(improvementlist)
-
- choutputs.writeAll("\n\n### BUG FIXES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(buglist)
-
- choutputs.writeAll("\n\n### TESTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(testlist)
-
- choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(subtasklist)
-
- choutputs.writeAll("\n\n### OTHER:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(otherlist)
- choutputs.writeList(tasklist)
-
- choutputs.writeAll("\n\n")
- choutputs.close()
-
- if options.index:
- buildindex(title,options.license)
-
- if haderrors is True:
- sys.exit(1)
-
-if __name__ == "__main__":
- main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
deleted file mode 100755
index ebcb660..0000000
--- a/dev-support/smart-apply-patch.sh
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Determine if the git diff patch file has prefixes.
-# These files are generated via "git diff" *without* the --no-prefix option.
-#
-# We can apply these patches more easily because we know that the a/ and b/
-# prefixes in the "diff" lines stands for the project root directory.
-# So we don't have to hunt for the project root.
-# And of course, we know that the patch file was generated using git, so we
-# know git apply can handle it properly.
-#
-# Arguments: git diff file name.
-# Return: 0 if it is a git diff with prefix; 1 otherwise.
-#
-has_prefix() {
- awk '/^diff --git / { if ($3 !~ "^a/" || $4 !~ "^b/") { exit 1 } }
- /^\+{3}|-{3} / { if ($2 !~ "^[ab]/" && $2 !~ "^/dev/null") { exit 1 } }' "$1"
- return $?
-}
-
-PATCH_FILE=$1
-DRY_RUN=$2
-if [ -z "$PATCH_FILE" ]; then
- echo usage: $0 patch-file
- exit 1
-fi
-
-TMPDIR=${TMPDIR:-/tmp}
-PATCH=${PATCH:-patch} # allow overriding patch binary
-
-# Cleanup handler for temporary files
-TOCLEAN=""
-cleanup() {
- rm $TOCLEAN
- exit $1
-}
-trap "cleanup 1" HUP INT QUIT TERM
-
-# Allow passing "-" for stdin patches
-if [ "$PATCH_FILE" == "-" ]; then
- PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM"
- cat /dev/fd/0 > $PATCH_FILE
- TOCLEAN="$TOCLEAN $PATCH_FILE"
-fi
-
-ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then
- # Allow downloading of patches
- PFILE="$TMPDIR/smart-apply.in.$RANDOM"
- TOCLEAN="$TOCLEAN $PFILE"
- if [[ ${PATCH_FILE} =~ ^http ]]; then
- patchURL="${PATCH_FILE}"
- else # Get URL of patch from JIRA
- wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}"
- if [[ $? != 0 ]]; then
- echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2
- cleanup 1
- elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then
- echo "${PATCH_FILE} is not \"Patch Available\". Exiting." 1>&2
- cleanup 1
- fi
- relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
- patchURL="http://issues.apache.org${relativePatchURL}"
- fi
- if [[ -n $DRY_RUN ]]; then
- echo "Downloading ${patchURL}"
- fi
- wget -q -O "${PFILE}" "${patchURL}"
- if [[ $? != 0 ]]; then
- echo "${PATCH_FILE} could not be downloaded." 1>&2
- cleanup 1
- fi
- PATCH_FILE="${PFILE}"
-fi
-
-# Case for git-diff patches
-if grep -q "^diff --git" "${PATCH_FILE}"; then
- GIT_FLAGS="--binary -v"
- if has_prefix "$PATCH_FILE"; then
- GIT_FLAGS="$GIT_FLAGS -p1"
- else
- GIT_FLAGS="$GIT_FLAGS -p0"
- fi
- if [[ -z $DRY_RUN ]]; then
- GIT_FLAGS="$GIT_FLAGS --stat --apply"
- echo Going to apply git patch with: git apply "${GIT_FLAGS}"
- else
- GIT_FLAGS="$GIT_FLAGS --check"
- fi
- # shellcheck disable=SC2086
- git apply ${GIT_FLAGS} "${PATCH_FILE}"
- if [[ $? == 0 ]]; then
- cleanup 0
- fi
- echo "git apply failed. Going to apply the patch with: ${PATCH}"
-fi
-
-# Come up with a list of changed files into $TMP
-TMP="$TMPDIR/smart-apply.paths.$RANDOM"
-TOCLEAN="$TOCLEAN $TMP"
-
-if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
- PLEVEL=0
- #if the patch applied at P0 there is the possability that all we are doing
- # is adding new files and they would apply anywhere. So try to guess the
- # correct place to put those files.
-
- TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM"
- TOCLEAN="$TOCLEAN $TMP2"
-
- egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2
-
- if [ ! -s $TMP2 ]; then
- echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
- cleanup 1
- fi
-
- #first off check that all of the files do not exist
- FOUND_ANY=0
- for CHECK_FILE in $(cat $TMP2)
- do
- if [[ -f $CHECK_FILE ]]; then
- FOUND_ANY=1
- fi
- done
-
- if [[ "$FOUND_ANY" = "0" ]]; then
- #all of the files are new files so we have to guess where the correct place to put it is.
-
- # if all of the lines start with a/ or b/, then this is a git patch that
- # was generated without --no-prefix
- if ! grep -qv '^a/\|^b/' $TMP2 ; then
- echo Looks like this is a git patch. Stripping a/ and b/ prefixes
- echo and incrementing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
- sed -i -e 's,^[ab]/,,' $TMP2
- fi
-
- PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u)
-
- # if we are at the project root then nothing more to do
- if [[ -d hadoop-common-project ]]; then
- echo Looks like this is being run at project root
-
- # if all of the lines start with hadoop-common/, hadoop-hdfs/, hadoop-yarn/ or hadoop-mapreduce/, this is
- # relative to the hadoop root instead of the subproject root, so we need
- # to chop off another layer
- elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-yarn-project|hadoop-mapreduce-project)$ ]]; then
-
- echo Looks like this is relative to project root. Increasing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
-
- elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-yarn-project\|hadoop-mapreduce-project' ; then
- echo Looks like this is a cross-subproject patch. Try applying from the project root
- cleanup 1
- fi
- fi
-elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=1
-elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=2
-else
- echo "The patch does not appear to apply with p0 to p2";
- cleanup 1;
-fi
-
-# If this is a dry run then exit instead of applying the patch
-if [[ -n $DRY_RUN ]]; then
- cleanup 0;
-fi
-
-echo Going to apply patch with: $PATCH -p$PLEVEL
-$PATCH -p$PLEVEL -E < $PATCH_FILE
-
-cleanup $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
deleted file mode 100755
index 6311584..0000000
--- a/dev-support/test-patch.d/checkstyle.sh
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin checkstyle
-
-CHECKSTYLE_TIMER=0
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function checkstyle_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test checkstyle
- fi
-}
-
-function checkstyle_mvnrunner
-{
- local logfile=$1
- local output=$2
- local tmp=${PATCH_DIR}/$$.${RANDOM}
- local j
-
- "${MVN}" clean test checkstyle:checkstyle -DskipTests \
- -Dcheckstyle.consoleOutput=true \
- "-D${PROJECT_NAME}PatchProcess" 2>&1 \
- | tee "${logfile}" \
- | ${GREP} ^/ \
- | ${SED} -e "s,${BASEDIR},.,g" \
- > "${tmp}"
-
- # the checkstyle output files are massive, so
- # let's reduce the work by filtering out files
- # that weren't changed. Some modules are
- # MASSIVE and this can cut the output down to
- # by orders of magnitude!!
- for j in ${CHANGED_FILES}; do
- ${GREP} "${j}" "${tmp}" >> "${output}"
- done
-
- rm "${tmp}" 2>/dev/null
-}
-
-function checkstyle_preapply
-{
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: prepatch"
-
- start_clock
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
- add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
- fi
- popd >/dev/null
- done
-
- # keep track of how much as elapsed for us already
- CHECKSTYLE_TIMER=$(stop_clock)
- return 0
-}
-
-function checkstyle_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/cs.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function checkstyle_postapply
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local numprepatch=0
- local numpostpatch=0
- local diffpostpatch=0
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: postpatch"
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${CHECKSTYLE_TIMER}"
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch checkstyle compilation is broken."
- add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
- continue
- fi
-
- #shellcheck disable=SC2016
- diffpostpatch=$(checkstyle_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
- "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
-
- if [[ ${diffpostpatch} -gt 0 ]] ; then
- ((rc = rc + 1))
-
- # shellcheck disable=SC2016
- numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
-
- add_jira_table -1 checkstyle "The applied patch generated "\
- "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
- footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
- fi
-
- popd >/dev/null
- done
-
- if [[ ${rc} -gt 0 ]] ; then
- add_jira_footer checkstyle "${footer}"
- return 1
- fi
- add_jira_table +1 checkstyle "There were no new checkstyle issues."
- return 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
deleted file mode 100755
index 9277ea5..0000000
--- a/dev-support/test-patch.d/shellcheck.sh
+++ /dev/null
@@ -1,178 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin shellcheck
-
-SHELLCHECK_TIMER=0
-
-SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)}
-
-SHELLCHECK_SPECIFICFILES=""
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function shellcheck_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.sh$ ]]; then
- add_test shellcheck
- SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
- fi
-
- if [[ ! ${filename} =~ \. ]]; then
- add_test shellcheck
- fi
-}
-
-function shellcheck_private_findbash
-{
- local i
- local value
- local list
-
- while read line; do
- value=$(find "${line}" ! -name '*.cmd' -type f \
- | ${GREP} -E -v '(.orig$|.rej$)')
- list="${list} ${value}"
- done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
- # shellcheck disable=SC2086
- echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
-}
-
-function shellcheck_preapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: prepatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- return 0
- fi
-
- start_clock
-
- # shellcheck disable=SC2016
- SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
-
- echo "Running shellcheck against all identifiable shell scripts"
- pushd "${BASEDIR}" >/dev/null
- for i in $(shellcheck_private_findbash); do
- if [[ -f ${i} ]]; then
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
- fi
- done
- popd > /dev/null
- # keep track of how much as elapsed for us already
- SHELLCHECK_TIMER=$(stop_clock)
- return 0
-}
-
-function shellcheck_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/sc.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function shellcheck_postapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: postpatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- add_jira_table 0 shellcheck "Shellcheck was not available."
- return 0
- fi
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${SHELLCHECK_TIMER}"
-
- echo "Running shellcheck against all identifiable shell scripts"
- # we re-check this in case one has been added
- for i in $(shellcheck_private_findbash); do
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
- done
-
- # shellcheck disable=SC2016
- numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
-
- diffPostpatch=$(shellcheck_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
- "${PATCH_DIR}/patchshellcheck-result.txt" \
- "${PATCH_DIR}/diffpatchshellcheck.txt"
- )
-
- if [[ ${diffPostpatch} -gt 0 ]] ; then
- add_jira_table -1 shellcheck "The applied patch generated "\
- "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
- add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
- return 1
- fi
-
- add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
- return 0
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0d3b0c1/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
deleted file mode 100755
index 324481c..0000000
--- a/dev-support/test-patch.d/whitespace.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin whitespace
-
-function whitespace_postapply
-{
- local count
- local j
-
- big_console_header "Checking for whitespace at the end of lines"
- start_clock
-
- pushd "${BASEDIR}" >/dev/null
- for j in ${CHANGED_FILES}; do
- ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
- done
-
- # shellcheck disable=SC2016
- count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
-
- if [[ ${count} -gt 0 ]]; then
- add_jira_table -1 whitespace "The patch has ${count}"\
- " line(s) that end in whitespace. Use git apply --whitespace=fix."
- add_jira_footer whitespace "@@BASE@@/whitespace.txt"
- popd >/dev/null
- return 1
- fi
-
- popd >/dev/null
- add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
- return 0
-}