You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by ba...@apache.org on 2016/02/27 17:49:41 UTC
couchdb-ci git commit: Node.js scripts to download all Jenkins logs
and analyze them.
Repository: couchdb-ci
Updated Branches:
refs/heads/master fec584319 -> 045028834
Node.js scripts to download all Jenkins logs and analyze them.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-ci/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-ci/commit/04502883
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-ci/tree/04502883
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-ci/diff/04502883
Branch: refs/heads/master
Commit: 045028834477dce9c4e26d6909896f0c64ef4d44
Parents: fec5843
Author: Bastian Krol <ba...@apache.org>
Authored: Thu Feb 18 14:24:48 2016 +0100
Committer: Bastian Krol <ba...@apache.org>
Committed: Sat Feb 27 17:48:28 2016 +0100
----------------------------------------------------------------------
scraping-jenkins/.gitignore | 3 -
scraping-jenkins/jobnames.txt | 13 --
scraping-jenkins/readme.markdown | 4 -
scraping-jenkins/rename.sh | 19 --
scraping-jenkins/scrape.sh | 36 ----
utils/analyze-jenkins-logs/.gitignore | 1 +
.../bin/analyze-jenkins-logs | 3 +
utils/analyze-jenkins-logs/bin/run.sh | 2 +
utils/analyze-jenkins-logs/ci-errors.markdown | 154 +++++++++++++++
utils/analyze-jenkins-logs/index.js | 197 +++++++++++++++++++
utils/analyze-jenkins-logs/package.json | 20 ++
utils/auto-download-jenkins-logs/.gitignore | 1 +
.../bin/auto-download-jenkins-logs | 3 +
utils/auto-download-jenkins-logs/index.js | 182 +++++++++++++++++
utils/auto-download-jenkins-logs/package.json | 24 +++
15 files changed, 587 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/.gitignore
----------------------------------------------------------------------
diff --git a/scraping-jenkins/.gitignore b/scraping-jenkins/.gitignore
deleted file mode 100644
index 4f7c67f..0000000
--- a/scraping-jenkins/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-jenkins-api-token
-root.xml
-jobs/
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/jobnames.txt
----------------------------------------------------------------------
diff --git a/scraping-jenkins/jobnames.txt b/scraping-jenkins/jobnames.txt
deleted file mode 100644
index e8a174d..0000000
--- a/scraping-jenkins/jobnames.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-2.0-developer-preview
-Apache-CouchDB-Matrix-Mac-OS-X-10-7-5-1.6.x
-Apache-CouchDB-Matrix-Mac-OS-X-10-7-5-master
-Apache-CouchDB-Matrix-Mac-OS-X-10-8-2-1.6.x
-Apache-CouchDB-Matrix-Mac-OS-X-10-8.2-master
-Apache-CouchDB-Matrix-Ubuntu-Server-64-1.6.x
-Apache-CouchDB-Windows
-Fauxton
-Rebuild%20Docs%201.3.x
-Rebuild%20Docs%201.4.x
-Rebuild%20Docs%201.5.x
-Rebuild%20Docs%201.6.x
-Rebuild%20Docs%20master
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/readme.markdown
----------------------------------------------------------------------
diff --git a/scraping-jenkins/readme.markdown b/scraping-jenkins/readme.markdown
deleted file mode 100644
index c44dc2e..0000000
--- a/scraping-jenkins/readme.markdown
+++ /dev/null
@@ -1,4 +0,0 @@
-Scraping the Old CouchDB CI Jobs
-================================
-
-Execute `./scrape.sh` to download all Jenkins jobs configurations from the old CI setup at ci.couchdb.org:8888.
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/rename.sh
----------------------------------------------------------------------
diff --git a/scraping-jenkins/rename.sh b/scraping-jenkins/rename.sh
deleted file mode 100755
index 32314ba..0000000
--- a/scraping-jenkins/rename.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-pushd `dirname $0`/jobs > /dev/null
-
-shopt -s nullglob # empty directory will return empty list
-
-# convert to all-lower-case
-for dir in ./*/;do
- mv "$dir" "${dir,,}" 2> /dev/null || true
-done
-
-# replace %20% by hyphen
-for dir in ./*/;do
- mv "$dir" "${dir//%20/-}" 2> /dev/null || true
-done
-
-popd > /dev/null
-
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/scraping-jenkins/scrape.sh
----------------------------------------------------------------------
diff --git a/scraping-jenkins/scrape.sh b/scraping-jenkins/scrape.sh
deleted file mode 100755
index a354b8f..0000000
--- a/scraping-jenkins/scrape.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-pushd `dirname $0` > /dev/null
-
-if [[ ! -f jenkins-api-token ]]; then
- echo Please create a file named jenkins-api-token and put your Jenkins API token in this file.
- exit 1
-fi
-
-JENKINS_API_TOKEN=$(<jenkins-api-token)
-echo "using Jenkins API token: $JENKINS_API_TOKEN"
-
-###############################################################################
-# root Jenkins config
-###############################################################################
-curl --show-error "http://basti1302:$JENKINS_API_TOKEN@ci.couchdb.org:8888/api/xml" > root.tmp.xml
-xmllint --format root.tmp.xml > root.xml || mv root.tmp.xml root.xml
-rm -f root.tmp.xml
-
-###############################################################################
-# Job configs
-###############################################################################
-mkdir -p jobs
-
-while IFS='' read -r jobname || [[ -n $jobname ]]; do
- echo "scraping job: $jobname"
- mkdir -p "jobs/$jobname"
- curl --show-error "http://basti1302:$JENKINS_API_TOKEN@ci.couchdb.org:8888/job/$jobname/config.xml" > "jobs/$jobname/config.xml"
-done < jobnames.txt
-
-# rename jobs to a sane naming pattern
-./rename.sh
-
-popd > /dev/null
-
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/.gitignore
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/.gitignore b/utils/analyze-jenkins-logs/.gitignore
new file mode 100644
index 0000000..c2658d7
--- /dev/null
+++ b/utils/analyze-jenkins-logs/.gitignore
@@ -0,0 +1 @@
+node_modules/
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs b/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs
new file mode 100755
index 0000000..229356d
--- /dev/null
+++ b/utils/analyze-jenkins-logs/bin/analyze-jenkins-logs
@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require('..');
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/bin/run.sh
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/bin/run.sh b/utils/analyze-jenkins-logs/bin/run.sh
new file mode 100755
index 0000000..239870b
--- /dev/null
+++ b/utils/analyze-jenkins-logs/bin/run.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+JENKINS_LOGS_DIR=/home/bastian/projekte/couchdb/ci-logs /home/bastian/.nvm/versions/node/v4.2.6/bin/node `dirname $0`/analyze-jenkins-logs
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/ci-errors.markdown
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/ci-errors.markdown b/utils/analyze-jenkins-logs/ci-errors.markdown
new file mode 100644
index 0000000..36ca487
--- /dev/null
+++ b/utils/analyze-jenkins-logs/ci-errors.markdown
@@ -0,0 +1,154 @@
+# Summary
+
+Successes: 38 Failures: 57
+
+
+# Uncategorized Failures
+
+* Number of failures: 6
+
+## Builds
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/18/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/22/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/23/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/26/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/18/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/28/consoleText>
+
+
+# Failures with reason "eunit_replicator"
+
+* Number of failures: 30
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/\\*\\*in function couch_replicator_filtered_tests:should_succeed/`
+* `/\\*\\*error:\{assertion_failed,\[\{module,couch_replicator_compact_tests\}/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/19/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/21/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/21/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/23/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/23/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/24/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/24/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/16/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/27/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/27/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/27/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/28/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/28/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/28/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/29/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/29/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/29/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/30/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/29/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/30/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/30/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/30/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/30/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/33/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/30/consoleText>
+
+
+# Failures with reason "eunit_compression"
+
+* Number of failures: 10
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/couchdb_file_compression_tests:110: should_compare_compression_methods.*\\*failed\\*/`
+* `/in call from couchdb_file_compression_tests:setup\/0 \(test\/couchdb_file_compression_tests.erl, line 38\)/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/12/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/12/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/17/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/15/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/20/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/19/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/24/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/26/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/29/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/29/consoleText>
+
+
+# Failures with reason "network"
+
+* Number of failures: 6
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/fatal: unable to access 'https:\/\/git-wip-us.apache.org/`
+* `/fatal: read error: Connection reset by peer/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=debian-8,label=docker/14/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=ubuntu-14.04,label=docker/20/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/24/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/24/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/28/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/28/consoleText>
+
+
+# Failures with reason "docker"
+
+* Number of failures: 3
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/Cannot connect to the Docker daemon. Is the docker daemon running on this host?/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/27/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=ubuntu-14.04,label=docker/27/consoleText>
+* <https://builds.apache.org/job/CouchDB/ERLANG=default,OS=centos-7,label=docker/27/consoleText>
+
+
+# Failures with reason "libdl"
+
+* Number of failures: 1
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/sed: error while loading shared libraries: libdl.so.2/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=debian-8,label=docker/14/consoleText>
+
+
+# Failures with reason "aborted"
+
+* Number of failures: 1
+## Regular Expressions
+When one of these regular expression has a match in the build log, I assume this build failure falls into this category.
+
+* `/Build was aborted/`
+
+## Builds
+
+Links to the build logs:
+
+* <https://builds.apache.org/job/CouchDB/ERLANG=18.2,OS=centos-7,label=docker/19/consoleText>
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/index.js
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/index.js b/utils/analyze-jenkins-logs/index.js
new file mode 100644
index 0000000..0656763
--- /dev/null
+++ b/utils/analyze-jenkins-logs/index.js
@@ -0,0 +1,197 @@
+'use strict';
+
+const _ = require('lodash');
+const fs = require('fs');
+const jsonfile = require('jsonfile');
+const glob = require('glob');
+const path = require('path');
+const process = require('process');
+
+
+let baseDir;
+let success = 0;
+let failure = 0;
+let unrecognized;
+
+// TODO Verify that the errors are indeed in the log file
+
+const reasons = {};
+
+const regexes = {
+ aborted: [ /Build was aborted/ ],
+ network: [
+ /fatal: unable to access 'https:\/\/git-wip-us.apache.org/,
+ /fatal: read error: Connection reset by peer/,
+ ],
+ docker: [
+ /Cannot connect to the Docker daemon. Is the docker daemon running on this host?/
+ ],
+ libdl: [ /sed: error while loading shared libraries: libdl.so.2/ ],
+ eunit_replicator: [
+ /\\*\\*in function couch_replicator_filtered_tests:should_succeed/,
+ /\\*\\*error:\{assertion_failed,\[\{module,couch_replicator_compact_tests\}/,
+ ],
+ eunit_compression: [
+ /couchdb_file_compression_tests:110: should_compare_compression_methods.*\\*failed\\*/,
+ /in call from couchdb_file_compression_tests:setup\/0 \(test\/couchdb_file_compression_tests.erl, line 38\)/
+ ],
+ eunit: [ /XRROR: One or more eunit tests failed./ ],
+};
+
+// from https://gist.github.com/colingourlay/82506396503c05e2bb94
+_.mixin({
+ 'sortKeysBy': function (obj, comparator) {
+ var keys = _.sortBy(_.keys(obj), function (key) {
+ return comparator ? comparator(obj[key], key) : key;
+ });
+
+ return _.zipObject(keys, _.map(keys, function (key) {
+ return obj[key];
+ }));
+ }
+});
+
+function init() {
+ if (!process.env.JENKINS_LOGS_DIR) {
+ console.log('WARNING: JENKINS_LOGS_DIR is not set.\n');
+ }
+ baseDir = process.env.JENKINS_LOGS_DIR || __dirname;
+ // console.log('Will read logs from', baseDir, '\n\n');
+
+ process.on('exit', function() {
+
+ console.log('# Summary');
+ console.log('\nSuccesses:', success, 'Failures:', failure);
+ if (reasons.unrecognized.counter > 0) {
+ console.log('\n\n# Uncategorized Failures');
+ console.log('\n* Number of failures: ' + reasons.unrecognized.counter);
+ console.log('\n## Builds');
+ reasons.unrecognized.urls.forEach( url => {
+ console.log('* <' + url + '>');
+ });
+ }
+
+ // print results, most frequent errors first
+ _(reasons)
+ .omit(['unrecognized'])
+ .sortKeysBy((value, key) => {
+ return -value.counter;
+ })
+ .forOwn((reasonObject, reasonKey) => {
+ console.log('\n\n# Failures with reason "' + reasonKey + '"');
+ console.log('\n* Number of failures: ' + reasonObject.counter);
+ console.log('## Regular Expressions');
+ console.log('When one of these regular expression has a match in the build log, I assume this build failure falls into this category.\n');
+ regexes[reasonKey].forEach( regex => {
+ console.log('* `' + regex + '`');
+ });
+ console.log('\n## Builds\n');
+ console.log('Links to the build logs:\n');
+ reasonObject.urls.forEach( url => {
+ console.log('* <' + url + '>');
+ });
+ });
+ });
+}
+
+
+function initReasonObject() {
+ return {
+ counter: 0,
+ urls: [],
+ directories: [],
+ };
+}
+
+
+function analyzeLogs() {
+ init();
+ glob('+([0123456789])/', { cwd: baseDir, }, function (err, buildDirectories) {
+ if (err) {
+ exitOnError(err);
+ }
+ if (buildDirectories.length === 0) {
+ exitOnError(new Error('No matching directories found.'));
+ }
+ buildDirectories.forEach(buildDir => {
+ readBuild(path.join(baseDir, buildDir));
+ });
+ });
+}
+
+
+function readBuild(buildDirectory) {
+ fs.readdir(buildDirectory, function(err, runDirectories) {
+ if (err) {
+ return printWarning(err);
+ }
+ runDirectories.forEach(runDirectory => {
+ readRun(path.join(buildDirectory, runDirectory));
+ });
+ });
+}
+
+
+function readRun(directory) {
+ const metaDataFile = path.join(directory, 'metadata.json');
+ const buildLogFile = path.join(directory, 'build.log');
+ jsonfile.readFile(metaDataFile, 'utf-8', (err, metaData) => {
+ if (err) {
+ return printWarning(err);
+ }
+ const result = metaData.result[0];
+ if (metaData.result[0] !== 'SUCCESS') {
+ // console.error('Failure:', directory, result);
+ failure++;
+ fs.readFile(buildLogFile, 'utf-8', (err, buildLog) => {
+ if (err) {
+ return printWarning(err);
+ }
+
+ for (var key in regexes) {
+ for (var i = 0; i < regexes[key].length; i++) {
+ if (contains(buildLog, regexes[key][i])) {
+ appendToReason(key, metaData, directory);
+ return;
+ }
+ }
+ }
+ // console.error('Uncategorized failure', buildLogFile);
+ appendToReason('unrecognized', metaData, directory);
+ });
+ } else {
+ // console.error('Success:', directory, result);
+ success++;
+ }
+ });
+}
+
+
+function appendToReason(key, metaData, directory) {
+ let reasonObject = reasons[key];
+ if (!reasonObject) {
+ reasonObject = reasons[key] = initReasonObject();
+ }
+ reasonObject.counter++;
+ reasonObject.urls.push(metaData.url + 'consoleText');
+ reasonObject.directories.push(directory);
+}
+
+
+function contains(buildLog, regex) {
+ return buildLog.search(regex) >= 0;
+}
+
+
+function exitOnError(error) {
+ console.error(error);
+ process.exit(1);
+}
+
+
+function printWarning(error) {
+ console.error(error);
+}
+
+
+analyzeLogs();
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/analyze-jenkins-logs/package.json
----------------------------------------------------------------------
diff --git a/utils/analyze-jenkins-logs/package.json b/utils/analyze-jenkins-logs/package.json
new file mode 100644
index 0000000..429ace7
--- /dev/null
+++ b/utils/analyze-jenkins-logs/package.json
@@ -0,0 +1,20 @@
+{
+ "name": "analyze-jenkins-logs",
+ "version": "1.0.0",
+ "description": "Analyzes Jenkins logs for failures",
+ "main": "index.js",
+ "bin": {
+ "analyze-jenkins-logs": "./bin/analyze-jenkins-logs"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://git-wip-us.apache.org/repos/asf/couchdb.git"
+ },
+ "author": "The Apache CouchDB contributors",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "glob": "^7.0.0",
+ "jsonfile": "^2.2.3",
+ "lodash": "^4.5.1"
+ }
+}
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/.gitignore
----------------------------------------------------------------------
diff --git a/utils/auto-download-jenkins-logs/.gitignore b/utils/auto-download-jenkins-logs/.gitignore
new file mode 100644
index 0000000..c2658d7
--- /dev/null
+++ b/utils/auto-download-jenkins-logs/.gitignore
@@ -0,0 +1 @@
+node_modules/
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs
----------------------------------------------------------------------
diff --git a/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs b/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs
new file mode 100755
index 0000000..229356d
--- /dev/null
+++ b/utils/auto-download-jenkins-logs/bin/auto-download-jenkins-logs
@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+require('..');
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/index.js
----------------------------------------------------------------------
diff --git a/utils/auto-download-jenkins-logs/index.js b/utils/auto-download-jenkins-logs/index.js
new file mode 100644
index 0000000..862172a
--- /dev/null
+++ b/utils/auto-download-jenkins-logs/index.js
@@ -0,0 +1,182 @@
+'use strict';
+
+const fs = require('fs');
+const isString = require('is-string');
+const jsonfile = require('jsonfile');
+const mkdirp = require('mkdirp');
+const parseXml = require('xml2js').parseString;
+const path = require('path');
+const process = require('process');
+const request = require('request');
+
+const apiXml = 'api/xml';
+let baseDir;
+
+function init() {
+ if (!process.env.JENKINS_LOGS_DIR) {
+ console.log('WARNING: JENKINS_LOGS_DIR is not set.');
+ }
+ baseDir = process.env.JENKINS_LOGS_DIR || __dirname;
+ console.log('Will write logs to', baseDir);
+}
+
+
+function crawlFromRootUrl(url) {
+ console.log(`
+================================================================================
+`);
+ console.log(new Date());
+ init();
+ request(url, function(err, message, body) {
+ if (err) {
+ return exitOnError(err);
+ }
+
+ parseXml(body, function (err, result) {
+ if (err) {
+ return exitOnError(err);
+ }
+
+ if (!result.matrixProject) {
+ return exitOnError(new Error('Not found: matrixProject'));
+ }
+ if (!result.matrixProject.build) {
+ return exitOnError(new Error('Not found: matrixProject.build'));
+ }
+ if (!Array.isArray(result.matrixProject.build)) {
+ return exitOnError(new Error('Not an array: matrixProject.build'));
+ }
+ result.matrixProject.build.forEach(build => {
+ fetchBuild(build);
+ });
+ });
+ });
+}
+
+
+function fetchBuild(build) {
+ if (build.number == null) {
+ return printWarning(new Error('Not found: build.number'));
+ }
+ if (build.url == null) {
+ return printWarning(new Error('Not found: build.url'));
+ }
+
+ const url = build.url + apiXml;
+ console.log('Retrieving meta data for build', build.number[0], 'from ', url);
+ request(url, function(err, message, body) {
+ if (err) {
+ return printWarning(err);
+ }
+ parseXml(body, function (err, result) {
+ if (err) {
+ return printWarning(err);
+ }
+
+ if (!result.matrixBuild) {
+ return printWarning(new Error('Not found: matrixBuild'));
+ }
+ if (!result.matrixBuild.run) {
+ return printWarning(new Error('Not found: matrixBuild.run'));
+ }
+ if (!Array.isArray(result.matrixBuild.run)) {
+ return printWarning(new Error('Not an array: matrixBuild.run'));
+ }
+ result.matrixBuild.run.forEach(run => {
+ fetchRun(run);
+ });
+ });
+ });
+}
+
+
+function fetchRun(run) {
+ if (run.number == null) {
+ return printWarning(new Error('Not found: run.number'));
+ }
+ if (run.url == null) {
+ return printWarning(new Error('Not found: run.url'));
+ }
+
+ const url = run.url + apiXml;
+ console.log('Retrieving meta data for run', run.number[0], 'from ', url);
+ request(url, function(err, message, body) {
+ if (err) {
+ return printWarning(err);
+ }
+ parseXml(body, function (err, result) {
+ if (err) {
+ return printWarning(err);
+ }
+
+ if (!result.matrixRun) {
+ return printWarning(new Error('Not found: matrixRun'));
+ }
+ if (!result.matrixRun.number) {
+ return printWarning(new Error('Not found: matrixRun.number'));
+ }
+ if (!result.matrixRun.fullDisplayName) {
+ return printWarning(new Error('Not found: matrixRun.fullDisplayName'));
+ }
+ if (!Array.isArray(result.matrixRun.fullDisplayName)) {
+ return printWarning(new Error('Not an array: matrixRun.fullDisplayName'));
+
+ }
+ if (result.matrixRun.fullDisplayName.length !== 1) {
+ return printWarning(new Error('Not an array of length 1: matrixRun.fullDisplayName'));
+ }
+ if (!isString(result.matrixRun.fullDisplayName[0])) {
+ return printWarning(new Error('Not a string: matrixRun.fullDisplayName[0]'));
+ }
+
+ fetchMatrixRunAndLog(result.matrixRun, run.url);
+ });
+ });
+}
+
+
+function fetchMatrixRunAndLog(metaData, runBaseUrl) {
+ const logUrl = runBaseUrl + 'consoleText';
+ request(logUrl, function(err, message, body) {
+ if (err) {
+ return printWarning(err);
+ }
+ saveToDisk(metaData, body);
+ });
+}
+
+
+function saveToDisk(metaData, buildLog) {
+ const dir = path.join(baseDir, String(metaData.number), metaData.fullDisplayName[0]);
+ mkdirp.sync(dir);
+ const metaDataFile = path.join(dir, 'metadata.json');
+ jsonfile.writeFile(metaDataFile, metaData, { spaces: 2 }, function (err) {
+ if (err) {
+ return printWarning(err);
+ }
+ console.log('Written meta data to ', metaDataFile);
+ });
+ const buildLogFile = path.join(dir, 'build.log');
+ fs.writeFile(buildLogFile, buildLog, function(err) {
+ if (err) {
+ return printWarning(err);
+ }
+ console.log('Written build log to ', buildLogFile);
+ });
+}
+
+
+function exitOnError(error) {
+ console.error(error);
+ process.exit(1);
+}
+
+
+function printWarning(error) {
+ console.error(error);
+}
+
+
+const rootUrl = 'https://builds.apache.org/job/CouchDB/api/xml';
+
+crawlFromRootUrl(rootUrl);
http://git-wip-us.apache.org/repos/asf/couchdb-ci/blob/04502883/utils/auto-download-jenkins-logs/package.json
----------------------------------------------------------------------
diff --git a/utils/auto-download-jenkins-logs/package.json b/utils/auto-download-jenkins-logs/package.json
new file mode 100644
index 0000000..5cbe0d3
--- /dev/null
+++ b/utils/auto-download-jenkins-logs/package.json
@@ -0,0 +1,24 @@
+{
+ "name": "auto-download-jenkins-logs",
+ "version": "1.0.0",
+ "description": "Automatically downloads Jenkins logs",
+ "main": "index.js",
+ "bin": {
+ "auto-download-jenkins-logs": "./bin/auto-download-jenkins-logs"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://git-wip-us.apache.org/repos/asf/couchdb.git"
+ },
+ "author": "The Apache CouchDB contributors",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "jsonfile": "^2.2.3",
+ "mkdirp": "^0.5.1",
+ "request": "^2.69.0",
+ "xml2js": "^0.4.16"
+ },
+ "devDependencies": {
+ "is-string": "^1.0.4"
+ }
+}