You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ch...@apache.org on 2018/07/10 08:45:28 UTC
[incubator-openwhisk] branch master updated: Adjust citool to work
with Travis build matrix. (#3848)
This is an automated email from the ASF dual-hosted git repository.
chetanm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 238ad96 Adjust citool to work with Travis build matrix. (#3848)
238ad96 is described below
commit 238ad965e1b4143837e64302d98febd55a96780e
Author: rodric rabbah <ro...@gmail.com>
AuthorDate: Tue Jul 10 04:45:23 2018 -0400
Adjust citool to work with Travis build matrix. (#3848)
Enables fetching of logs for various jobs in a matrix by adding sub job index as suffix
./citool monitor -p 402088870.1
---
tools/build/README.md | 1 +
tools/build/citool | 72 +++++++++++++++++++++++++++++++++++++--------------
2 files changed, 54 insertions(+), 19 deletions(-)
diff --git a/tools/build/README.md b/tools/build/README.md
index f6166a5..77d77dd 100644
--- a/tools/build/README.md
+++ b/tools/build/README.md
@@ -63,6 +63,7 @@ To change the Travis (or Jenkins) host URL, use `-u`.
- monitor a Travis CI build with job number `N`: `citool monitor N`
- monitor same job `N` until completion: `citool monitor -p N`
- save job output to a file: `citool -o monitor N`
+- for Travis CI matrix builds, use the matrix index after the job number as in `citool monitor N.i` where 1 <= i <= matrix buidls.
To monitor a Jenkins build `B` with job number `N` on host `https://jenkins.host:port`:
```
diff --git a/tools/build/citool b/tools/build/citool
index 371b646..2af9342 100755
--- a/tools/build/citool
+++ b/tools/build/citool
@@ -37,13 +37,17 @@ import sys
import time
import traceback
import argparse
-import httplib
+if sys.version_info.major >= 3:
+ from http.client import HTTPConnection, HTTPSConnection, IncompleteRead, TEMPORARY_REDIRECT, OK
+ from urllib.parse import urlparse
+else:
+ from httplib import HTTPConnection, HTTPSConnection, IncompleteRead, TEMPORARY_REDIRECT, OK
+ from urlparse import urlparse
import re
import threading
import json
from xml.dom import minidom
from subprocess import Popen, PIPE, STDOUT
-from urlparse import urlparse
def main():
exitCode = 0
@@ -64,7 +68,8 @@ def parseArgs():
parser = argparse.ArgumentParser(description='tool for analyzing logs from CI')
subparsers = parser.add_subparsers(title='available commands', dest='cmd')
- parser.add_argument('job', help='job number')
+ parser.add_argument('job', help='job number; for matrix jobs add the matrix index after a period (e.g., 401881768.2)')
+
parser.add_argument('-b', '--build', help='build name', default='travis')
parser.add_argument('-v', '--verbose', help='verbose output', action='store_true')
parser.add_argument('-i', '--input-file', help='read logs from file rather than CI', action='store_true', dest='ifile')
@@ -87,9 +92,9 @@ def parseArgs():
def request(method, urlString, body = "", headers = {}, auth = None, verbose = False):
url = urlparse(urlString)
if url.scheme == 'http':
- conn = httplib.HTTPConnection(url.netloc)
+ conn = HTTPConnection(url.netloc)
else:
- conn = httplib.HTTPSConnection(url.netloc)
+ conn = HTTPSConnection(url.netloc)
if verbose:
print("%s %s" % (method, urlString))
@@ -124,22 +129,48 @@ def shell(cmd, data = None, verbose = False):
def getTravisHeaders():
return {'User-Agent': 'wsk citool/0.0.1',
+ 'Travis-API-Version': 3,
'Accept': 'application/vnd.travis-ci.2+json'
}
+def getTravisMatrixId(parts, values):
+ N = len(values)
+ if len(parts) == 1:
+ return -1
+ else:
+ try:
+ matrix = int(parts[1]) -1
+ if matrix < 0:
+ print('Matrix id must be positive. Valid values are [1..%s].' % N)
+ exit(-1)
+ if matrix >= N:
+ print('Matrix id is out of bounds. Valid values are [1..%s].' % N)
+ exit(-1)
+ return matrix
+ except Exception:
+ print('Matrix id is not an integer as expected. Valid values are [1..%s].' % N)
+ exit(-1)
+
def getJobUrl(args):
if args.build.lower() == 'travis':
# Get build information
- buildUrl = '%s/builds/%s' % (args.url, args.job)
+ parts = args.job.split('.')
+ jobid = parts[0]
+ if len(parts) > 2:
+ print('Job is malformed')
+ exit(-1)
+
+ buildUrl = '%s/build/%s' % (args.url, jobid)
buildRes = request('get', buildUrl, headers = getTravisHeaders(), verbose = args.verbose)
body = validateResponse(buildRes)
try:
body = json.loads(body)
- job = body['build']['job_ids'][-1]
+ index = getTravisMatrixId(parts, body['jobs'])
+ job = body['jobs'][index]['id']
except Exception:
- print('expected response to contain build and job-ids properties in %s' % body)
+ print('Expected response to contain build and job-ids properties in %s' % body)
exit(-1)
- url = '%s/jobs/%s' % (args.url, job)
+ url = '%s/job/%s' % (args.url, job)
else: # assume jenkins
url = '%s/job/%s/%s' % (args.url, args.build, args.job)
return url
@@ -164,8 +195,8 @@ def monitorOnce(args):
else:
if args.build.lower() == 'travis':
url = '%s/log.txt' % getJobUrl(args)
- res = request('get', url, verbose = args.verbose)
- if res.status == httplib.TEMPORARY_REDIRECT:
+ res = request('get', url, headers = getTravisHeaders(), verbose = args.verbose)
+ if res.status == TEMPORARY_REDIRECT:
url = res.getheader('location')
res = request('get', url, headers = getTravisHeaders(), verbose = args.verbose)
else: # assume jenkins
@@ -178,7 +209,7 @@ def monitorOnce(args):
file = open('%s-console.log' % args.job, 'w')
file.write(body)
file.close()
- if args.ifile or res.status == httplib.OK:
+ if args.ifile or res.status == OK:
grepForFailingTests(args, body)
return reportBuildStatus(args, body)
elif args.ofile is False:
@@ -188,7 +219,7 @@ def monitorOnce(args):
def validateResponse(res):
body = res.read()
- if res.status != httplib.OK:
+ if res.status != OK:
if body.startswith('<'):
dom = minidom.parseString(body)
print(dom.toprettyxml()),
@@ -196,7 +227,7 @@ def validateResponse(res):
print(body)
exit(res.status)
elif not body:
- print('build log is empty')
+ print('Build log is empty.')
exit(-1)
else:
return body
@@ -206,21 +237,24 @@ def grepForFailingTests(args, body):
# check that tests ran
(time, output, error) = shell(cmd, body, args.verbose)
if output == '':
- print('no tests detected')
+ print('No tests detected.')
# no tests: either build failure or task not yet reached, skip further check
else:
cmd = 'grep -E "^\w+\.*.*[>|>] \w*.* FAILED%s"' % ("|PASSED" if args.all else "")
(time, output, error) = shell(cmd, body, args.verbose)
if output == '':
- print('all tests passing')
+ print('All tests passing.')
else:
print(output.replace('>', '>')),
def reportBuildStatus(args, body):
- lines = body.rstrip('\n').rsplit('\n', 1)
+ lines = body.decode('utf8').rstrip('\n').rsplit('\n', 1)
+
if len(lines) == 2:
output = lines[1]
output = re.sub('<[^<]+?>', '', output).strip()
+ else:
+ output = None
if output and ('Finished: ' in output or output.startswith('Done.') or ('exceeded' in output and 'terminated' in output)):
print(output)
@@ -246,7 +280,7 @@ def cat(args):
url = '%s/artifact/%s/%s/%s_logs.log' % (getJobUrl(args), args.artifactPath, component, component)
res = request('get', url, verbose = args.verbose)
body = res.read()
- if res.status == httplib.OK:
+ if res.status == OK:
return body
else:
return ''
@@ -267,7 +301,7 @@ def cat(args):
joined = file.read()
file.close()
elif args.build.lower == 'travis':
- print('feature not yet supported for Travis builds')
+ print('Feature not yet supported for Travis builds.')
return 2
else:
components = {