yetus git commit: YETUS-288. move precommit-admin from Hadoop

[aw...@apache.org]

Repository: yetus
Updated Branches:
  refs/heads/master ef2b90a80 -> 34b1ff77c


YETUS-288. move precommit-admin from Hadoop

This is a copy of the jenkinsPrecommitAdmin.py written by
Brock Noland <br...@apache.org> as series of commits in the
svn://svn.apache.org/repos/asf/hadoop/nightly source tree.

The commits correspond with HADOOP-9765, HADOOP-10199, and
HADOOP-10200.

Signed-off-by: Brock Nelson <br...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/34b1ff77
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/34b1ff77
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/34b1ff77

Branch: refs/heads/master
Commit: 34b1ff77cf13389ca21c41a2ecc9f335e6da7aea
Parents: ef2b90a
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Dec 8 11:13:24 2017 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Sat Dec 9 16:14:39 2017 -0800

----------------------------------------------------------------------
 precommit/jenkins/jenkins-admin.py | 143 ++++++++++++++++++++++++++++++++
 1 file changed, 143 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/34b1ff77/precommit/jenkins/jenkins-admin.py
----------------------------------------------------------------------
diff --git a/precommit/jenkins/jenkins-admin.py b/precommit/jenkins/jenkins-admin.py
new file mode 100755
index 0000000..0fe34b0
--- /dev/null
+++ b/precommit/jenkins/jenkins-admin.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Script uses a JIRA search filter to submit test Jenkins
+# jobs for patch avilable issues. For more information see:
+#  http://wiki.apache.org/general/PreCommitBuilds
+#  https://builds.apache.org/job/PreCommit-Admin/
+
+from optparse import OptionParser
+from tempfile import NamedTemporaryFile
+from xml.etree import ElementTree
+import os, re, sys, urllib2
+
+# max number of entries to keep in the patch_tested.txt file
+MAX_HISTORY = 5000
+
+def httpGet(resource, ignoreError=False):
+  if ignoreError:
+    try:
+      return urllib2.urlopen(resource).read()
+    except urllib2.URLError, e:
+      print "ERROR retrieving resource %s: %s" % (resource, e)
+      return ""
+  return urllib2.urlopen(resource).read()
+
+
+# returns a map of (project, issue) => attachment id
+def parseJiraData(fileName):
+  tree = ElementTree.parse(fileName)
+  root = tree.getroot()
+  jiraPattern = re.compile('([A-Z]+)\-([0-9]+)')
+  result = {}
+  for item in root.findall("./channel/item"):
+    key = item.find('key')
+    if key == None: continue
+    issue = key.text
+    matcher = jiraPattern.match(issue)
+    if not matcher: continue
+    issue = matcher.group(1), matcher.group(2)
+    attachmentIds = []
+    for attachment in item.findall('./attachments/attachment'):
+      attachmentId = attachment.get('id')
+      try:
+        attachmentIds.append(int(attachmentId))
+      except ValueError:
+        pass
+    if len(attachmentIds) > 0:
+      attachmentIds.sort()
+      result[issue] = attachmentIds[-1]
+  return result
+
+if __name__ == '__main__':
+  parser = OptionParser()
+  parser.add_option("--jenkins-url", dest="jenkinsUrl",
+                    help="Jenkins base URL", metavar="URL")
+  parser.add_option("--jenkins-token", dest="jenkinsToken",
+                    help="Jenkins Token", metavar="TOKEN")
+  parser.add_option("--jira-filter", dest="jiraFilter",
+                    help="JIRA filter URL", metavar="URL")
+  parser.add_option("--jenkins-url-override", dest="jenkinsUrlOverrides", action="append",
+                    help="Project specific Jenkins base URL", metavar="PROJECT=URL")
+  parser.add_option("--live", dest="live", action="store_true",
+                    help="Submit Job to jenkins")
+  (options, args) = parser.parse_args()
+  if not options.jiraFilter:
+    parser.error("JIRA Filter is a required argument")
+  if not options.jenkinsUrl:
+    parser.error("Jenkins URL is a required argument")
+  if options.live and not options.jenkinsToken:
+    parser.error("Jenkins Token is required when in live mode")
+  jenkinsUrlOverrides = {}
+  if options.jenkinsUrlOverrides:
+    for override in options.jenkinsUrlOverrides:
+      if "=" not in override:
+        parser.error("Invalid Jenkins Url Override: " + override)
+      (project, url) = override.split("=", 1)
+      jenkinsUrlOverrides[project.upper()] = url
+  tempFile = NamedTemporaryFile(delete=False)
+  try:
+    jobLogHistory = httpGet(options.jenkinsUrl + \
+      "/job/PreCommit-Admin/lastSuccessfulBuild/artifact/patch_tested.txt", True)
+    # if we don't have a successful build available try the last build
+    if not jobLogHistory:
+      jobLogHistory = httpGet(options.jenkinsUrl + \
+        "/job/PreCommit-Admin/lastCompletedBuild/artifact/patch_tested.txt")
+    jobLogHistory = jobLogHistory.strip().split("\n")
+    if "TESTED ISSUES" not in jobLogHistory[0]:
+      print "Downloaded patch_tested.txt control file may be corrupted. Failing."
+      sys.exit(1)
+    jobLog = open('patch_tested.txt', 'w+')
+    if len(jobLogHistory)  > MAX_HISTORY:
+      jobLogHistory = [ jobLogHistory[0] ] + jobLogHistory[len(jobLogHistory) - MAX_HISTORY:]
+    for jobHistoryRecord in jobLogHistory:
+      jobLog.write(jobHistoryRecord + "\n")
+    jobLog.flush()
+    rssData = httpGet(options.jiraFilter)
+    tempFile.write(rssData)
+    tempFile.flush()
+    for key, attachment in parseJiraData(tempFile.name).items():
+      (project, issue) = key
+      if jenkinsUrlOverrides.has_key(project):
+        url = jenkinsUrlOverrides[project]
+      else:
+        url = options.jenkinsUrl
+      jenkinsUrlTemplate = url + "/job/PreCommit-{project}-Build/buildWithParameters" + \
+        "?token={token}&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}"
+      urlArgs = {'token': options.jenkinsToken, 'project': project, 'issue': issue, 'attachment': attachment }
+      jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs)
+      # submit job
+      jobName = "%s-%s,%s" % (project, issue, attachment)
+      if jobName not in jobLogHistory:
+        print jobName + " has not been processed, submitting"
+        jobLog.write(jobName + "\n")
+        jobLog.flush()
+        if options.live:
+          httpGet(jenkinsUrl, True)
+        else:
+          print "GET " + jenkinsUrl
+      else:
+        print jobName + " has been processed, ignoring"
+    jobLog.close()
+  finally:
+    if options.live:
+      os.remove(tempFile.name)
+    else:
+      print "JIRA Data is located: " + tempFile.name