You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by wo...@apache.org on 2017/05/10 23:08:00 UTC
[couchdb] 01/01: Add auto log uploader for CI workflow
This is an automated email from the ASF dual-hosted git repository.
wohali pushed a commit to branch upload-failed-logfiles
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4cfdb51257941d139a0e4f9e25273208324e60fa
Author: Joan Touzet <jo...@atypical.net>
AuthorDate: Wed May 10 18:45:19 2017 -0400
Add auto log uploader for CI workflow
---
.gitignore | 12 +++++
.travis.yml | 21 ++++++--
build-aux/logfile-uploader.py | 111 ++++++++++++++++++++++++++++++++++++++++++
test/javascript/run | 6 ++-
4 files changed, 145 insertions(+), 5 deletions(-)
diff --git a/.gitignore b/.gitignore
index e3601f7..f6e0c37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,8 @@ log
apache-couchdb-*/
bin/
config.erl
+*.tar.gz
+*.tar.bz2
dev/boot_node.beam
dev/lib/
dev/logs/
@@ -106,3 +108,13 @@ src/mango/ebin/
src/mango/test/*.pyc
src/mango/venv/
src/mango/.eunit
+
+src/chttpd/.eunit/
+src/couch_event/.eunit/
+src/couch_mrview/.eunit/
+src/couch_plugins/.eunit/
+src/couch_stats/.eunit/
+src/ddoc_cache/.eunit/
+src/fabric/.eunit/
+src/mem3/.eunit/
+src/rexi/.eunit/
diff --git a/.travis.yml b/.travis.yml
index 580ac6c..156856c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,19 +17,29 @@ addons:
- libicu-dev
- libmozjs185-dev
- pkg-config
+ - python-requests
+ - python-sphinx
- help2man
- shunit2
+python:
+ - "2.7"
+
git:
depth: 10
+# logfile uploader uses requests
+cache:
+ - pip
+
+# logfile uploader credentials
+env:
+ global:
+ - secure: "UdA/gKIlyuXaW+hUgRx40t1TYjLCGxMqHvM5Uw7UbUH2dqEkgJiLfhZGchS1JVzl8M01VKZUUzS7v2nvRLiHZN1kvaw5kfq31VRoafUah8jfmvqNWZVdLovHl3aw5UX/HRt0RkbWbhdbdknTfh6+YinSZ+Nb54jCErMg9nabXtM="
+
# Enable this block if you want to build docs & fauxton too
-#cache:
-# - pip
#node_js:
# - 6
-#install:
-# - pip install sphinx
#before_script:
# - ./configure -c
@@ -40,6 +50,9 @@ before_script:
script:
- make check
+after_failure:
+ - build-aux/logfile-uploader.py
+
# Re-enable once test suite is reliable
#notifications:
# email: false
diff --git a/build-aux/logfile-uploader.py b/build-aux/logfile-uploader.py
new file mode 100755
index 0000000..b656670
--- /dev/null
+++ b/build-aux/logfile-uploader.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+from __future__ import print_function
+
+import datetime
+import glob
+import json
+import os
+import tarfile
+import time
+
+import requests
+
+COUCH_URL = "http://couchdb-vm.apache.org:5984/ci_errorlogs"
+TARFILE = "couchlog.tar.gz"
+
+def _tojson(req):
+ """Support requests v0.x as well as 1.x+"""
+ if requests.__version__[0] == '0':
+ return json.loads(req.content)
+ return req.json()
+
+def collect_logfiles():
+ """ Find and tarball all logfiles """
+ tb = tarfile.open(name=TARFILE, mode='w:gz')
+ # EUnit
+ for log in glob.glob('src/*/.eunit/couch.log'):
+ tb.add(log)
+ # JS harness
+ for log in glob.glob('dev/logs/node1.log'):
+ tb.add(log)
+ tb.close()
+
+def build_ci_doc():
+ """ Build a metadata document with relevant detail from CI env """
+ doc = {}
+ if 'TRAVIS' in os.environ:
+ doc['builder'] = 'travis'
+ doc['build_id'] = os.environ['TRAVIS_JOB_ID']
+ doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
+ doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
+ os.environ['TRAVIS_JOB_ID']
+ doc['branch'] = os.environ['TRAVIS_BRANCH']
+ doc['commit'] = os.environ['TRAVIS_COMMIT']
+ doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
+ elif 'JENKINS_URL' in os.environ:
+ doc['builder'] = 'jenkins'
+ doc['build_id'] = os.environ['BUILD_NUMBER']
+ doc['url'] = os.environ['BUILD_URL']
+ doc['branch'] = os.environ['GIT_BRANCH']
+ doc['commit'] = os.environ['GIT_COMMIT']
+ doc['repo'] = os.environ['GIT_URL']
+ else:
+ doc['builder'] = 'manual'
+ # TODO: shell out to get correct repo, commit, branch info?
+ doc['repo'] = 'https://github.com/apache/couchdb'
+ doc['build_id'] = str(time.time())
+
+ # shorten doc id
+ repo = doc['repo'].split('/')[-1]
+ repo = repo.replate('.git', '')
+
+ doc['_id'] = doc['builder'] + '-' + repo + '-' + \
+ doc['build_id'] + \
+ '-' + datetime.datetime.utcnow().isoformat()
+
+ return doc
+
+def upload_logs():
+ lp = os.environ['COUCHAUTH'].split(':')
+ creds = (lp[0], lp[1])
+ doc = build_ci_doc()
+ req = requests.post(COUCH_URL,
+ data=json.dumps(doc),
+ auth=creds,
+ headers={'Content-type': 'application/json'})
+ req.raise_for_status()
+ req = _tojson(req)
+ with open(TARFILE, 'rb') as f:
+ req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
+ headers={'Content-type': 'application/x-gtar'},
+ auth=creds,
+ params={'rev': req['rev']},
+ data=f)
+ req2.raise_for_status()
+ return req2
+
+
+def main():
+ """ Find latest logfile and upload to Couch logfile db. """
+ print ("Uploading logfiles...")
+ collect_logfiles()
+ req = upload_logs()
+ print (req.url)
+ print (req.text)
+ print ("Upload complete!")
+
+if __name__ == '__main__':
+ main()
diff --git a/test/javascript/run b/test/javascript/run
index 10d00d8..7f366eb 100755
--- a/test/javascript/run
+++ b/test/javascript/run
@@ -99,7 +99,9 @@ def options():
return [
op.make_option("-s", "--start", metavar="FILENAME", default=None,
help="Start from the given filename if multiple files "
- "are passed")
+ "are passed"),
+ op.make_option("-a", "--all", action="store_true", dest="all",
+ help="Run all tests, even if one or more fail")
]
@@ -141,6 +143,8 @@ def main():
passed += 1
else:
failed += 1
+ if not opts.all:
+ break
sys.stderr.write("======================================================="
+ os.linesep)
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.