You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2015/09/18 03:01:18 UTC
[1/6] storm git commit: STORM-803: Better CI logs
Repository: storm
Updated Branches:
refs/heads/0.10.x-branch 6b18ee891 -> 150f7f6e3
STORM-803: Better CI logs
Conflicts:
dev-tools/travis/travis-script.sh
storm-core/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/098517b1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/098517b1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/098517b1
Branch: refs/heads/0.10.x-branch
Commit: 098517b1f7f4fcd93edb29268b7786eec461d73f
Parents: 6b18ee8
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Apr 27 19:28:15 2015 +0000
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 09:46:47 2015 +0900
----------------------------------------------------------------------
.travis.yml | 3 +-
dev-tools/test-ns.py | 21 +----
.../print-errors-from-clojure-test-reports.py | 58 -------------
.../travis/print-errors-from-test-reports.py | 66 ++++++++++++++
dev-tools/travis/save-logs.py | 54 ++++++++++++
dev-tools/travis/travis-build.sh | 50 -----------
dev-tools/travis/travis-install.sh | 37 ++++++++
dev-tools/travis/travis-script.sh | 43 +++++++++
pom.xml | 10 ++-
storm-core/pom.xml | 7 +-
storm-core/test/resources/log4j2-test.xml | 2 +-
storm-core/test/resources/test_runner.clj | 91 ++++++++++++++++++++
12 files changed, 309 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 27484eb..a7e2df4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,4 +17,5 @@ before_install:
- rvm use 2.1.5 --install
- nvm install 0.12.2
- nvm use 0.12.2
-script: /bin/bash ./dev-tools/travis/travis-build.sh `pwd`
+install: /bin/bash ./dev-tools/travis/travis-install.sh `pwd`
+script: /bin/bash ./dev-tools/travis/travis-script.sh `pwd`
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/test-ns.py
----------------------------------------------------------------------
diff --git a/dev-tools/test-ns.py b/dev-tools/test-ns.py
index 2fd1421..c0749e8 100755
--- a/dev-tools/test-ns.py
+++ b/dev-tools/test-ns.py
@@ -22,26 +22,7 @@ import os
os.chdir("storm-core")
ns = sys.argv[1]
-pipe = Popen(["mvn", "clojure:repl"], stdin=PIPE)
-
-
-pipe.stdin.write("""
-(do
- (use 'clojure.test)
- (require '%s :reload-all)
- (let [results (run-tests '%s)]
- (println results)
- (if (or
- (> (:fail results) 0)
- (> (:error results) 0))
- (System/exit 1)
- (System/exit 0))))
-""" % (ns, ns))
-
-
-
-pipe.stdin.write("\n")
-pipe.stdin.close()
+pipe = Popen(["mvn", "test", "-DfailIfNoTests=false", "-Dtest=%s"%ns])
pipe.wait()
os.chdir("..")
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/print-errors-from-clojure-test-reports.py
----------------------------------------------------------------------
diff --git a/dev-tools/travis/print-errors-from-clojure-test-reports.py b/dev-tools/travis/print-errors-from-clojure-test-reports.py
deleted file mode 100644
index 1cb88fc..0000000
--- a/dev-tools/travis/print-errors-from-clojure-test-reports.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-import glob
-from xml.etree.ElementTree import ElementTree
-
-
-def print_detail_information(testcase, fail_or_error):
- print "-" * 50
- print "classname: %s / testname: %s" % (testcase.get("classname"), testcase.get("name"))
- print fail_or_error.text
- print "-" * 50
-
-
-def print_error_reports_from_report_file(file_path):
- tree = ElementTree()
- tree.parse(file_path)
-
- testcases = tree.findall(".//testcase")
- for testcase in testcases:
- error = testcase.find("error")
- if error is not None:
- print_detail_information(testcase, error)
-
- fail = testcase.find("fail")
- if fail is not None:
- print_detail_information(testcase, fail)
-
-
-def main(report_dir_path):
- for test_report in glob.iglob(report_dir_path + '/*.xml'):
- file_path = os.path.abspath(test_report)
- try:
- print_error_reports_from_report_file(file_path)
- except Exception, e:
- print "Error while reading report file, %s" % file_path
- print "Exception: %s" % e
-
-
-if __name__ == "__main__":
- if sys.argv < 2:
- print "Usage: %s [report dir path]" % sys.argv[0]
- sys.exit(1)
-
- main(sys.argv[1])
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/print-errors-from-test-reports.py
----------------------------------------------------------------------
diff --git a/dev-tools/travis/print-errors-from-test-reports.py b/dev-tools/travis/print-errors-from-test-reports.py
new file mode 100644
index 0000000..95cc7b7
--- /dev/null
+++ b/dev-tools/travis/print-errors-from-test-reports.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import glob
+from xml.etree.ElementTree import ElementTree
+
+def print_detail_information(testcase, fail_or_error):
+ print "-" * 50
+ print "classname: %s / testname: %s" % (testcase.get("classname"), testcase.get("name"))
+ print fail_or_error.text
+ stdout = testcase.find("system-out")
+ if stdout != None:
+ print "-" * 20, "system-out", "-"*20
+ print stdout.text
+ stderr = testcase.find("system-err")
+ if stderr != None:
+ print "-" * 20, "system-err", "-"*20
+ print stderr.text
+ print "-" * 50
+
+
+def print_error_reports_from_report_file(file_path):
+ tree = ElementTree()
+ tree.parse(file_path)
+
+ testcases = tree.findall(".//testcase")
+ for testcase in testcases:
+ error = testcase.find("error")
+ if error is not None:
+ print_detail_information(testcase, error)
+
+ fail = testcase.find("fail")
+ if fail is not None:
+ print_detail_information(testcase, fail)
+
+
+def main(report_dir_path):
+ for test_report in glob.iglob(report_dir_path + '/*.xml'):
+ file_path = os.path.abspath(test_report)
+ try:
+ print "Checking %s" % test_report
+ print_error_reports_from_report_file(file_path)
+ except Exception, e:
+ print "Error while reading report file, %s" % file_path
+ print "Exception: %s" % e
+
+
+if __name__ == "__main__":
+ if sys.argv < 2:
+ print "Usage: %s [report dir path]" % sys.argv[0]
+ sys.exit(1)
+
+ main(sys.argv[1])
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/save-logs.py
----------------------------------------------------------------------
diff --git a/dev-tools/travis/save-logs.py b/dev-tools/travis/save-logs.py
new file mode 100755
index 0000000..5f4ad28
--- /dev/null
+++ b/dev-tools/travis/save-logs.py
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import subprocess
+from datetime import datetime, timedelta
+
+def main(file, cmd):
+ print cmd, "writing to", file
+ out = open(file, "w")
+ count = 0
+ process = subprocess.Popen(cmd,
+ stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE)
+
+ start = datetime.now()
+ nextPrint = datetime.now() + timedelta(seconds=1)
+ # wait for the process to terminate
+ pout = process.stdout
+ line = pout.readline()
+ while line:
+ count = count + 1
+ if datetime.now() > nextPrint:
+ diff = datetime.now() - start
+ sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
+ sys.stdout.flush()
+ nextPrint = datetime.now() + timedelta(seconds=10)
+ out.write(line)
+ line = pout.readline()
+ out.close()
+ errcode = process.wait()
+ diff = datetime.now() - start
+ sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
+ print
+ print cmd, "done", errcode
+ return errcode
+
+if __name__ == "__main__":
+ if sys.argv < 1:
+ print "Usage: %s [file info]" % sys.argv[0]
+ sys.exit(1)
+
+ sys.exit(main(sys.argv[1], sys.argv[2:]))
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/travis-build.sh
----------------------------------------------------------------------
diff --git a/dev-tools/travis/travis-build.sh b/dev-tools/travis/travis-build.sh
deleted file mode 100755
index a06151c..0000000
--- a/dev-tools/travis/travis-build.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PYTHON_VERSION_TO_FILE=`python -V > /tmp/python_version 2>&1`
-PYTHON_VERSION=`cat /tmp/python_version`
-RUBY_VERSION=`ruby -v`
-NODEJS_VERSION=`node -v`
-
-echo "Python version : $PYTHON_VERSION"
-echo "Ruby version : $RUBY_VERSION"
-echo "NodeJs version : $NODEJS_VERSION"
-
-
-STORM_SRC_ROOT_DIR=$1
-
-TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-
-cd ${STORM_SRC_ROOT_DIR}
-
-# Travis CI doesn't allow stdout bigger than 4M, so we have to reduce log while running tests
-export LOG_LEVEL=WARN
-# We should concern that Travis CI could be very slow cause it uses VM
-export STORM_TEST_TIMEOUT_MS=100000
-
-# We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script
-mvn test -fae
-
-BUILD_RET_VAL=$?
-
-if [ ${BUILD_RET_VAL} -ne 0 ]
-then
- echo "There may be clojure test errors from storm-core, printing error reports..."
- python ${TRAVIS_SCRIPT_DIR}/print-errors-from-clojure-test-reports.py ${STORM_SRC_ROOT_DIR}/storm-core/target/test-reports
-else
- echo "Double checking clojure test-report, Errors or Failures:"
- egrep -il '<fail|<error' */target/test-reports/*.xml | xargs -I '{}' bash -c "echo -n '{}':' '; egrep -ic '<fail|<error' '{}'"
- egrep -iq '<fail|<error' */target/test-reports/*.xml
-fi
-
-exit ${BUILD_RET_VAL}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/travis-install.sh
----------------------------------------------------------------------
diff --git a/dev-tools/travis/travis-install.sh b/dev-tools/travis/travis-install.sh
new file mode 100755
index 0000000..2812240
--- /dev/null
+++ b/dev-tools/travis/travis-install.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PYTHON_VERSION_TO_FILE=`python -V > /tmp/python_version 2>&1`
+PYTHON_VERSION=`cat /tmp/python_version`
+RUBY_VERSION=`ruby -v`
+NODEJS_VERSION=`node -v`
+
+echo "Python version : $PYTHON_VERSION"
+echo "Ruby version : $RUBY_VERSION"
+echo "NodeJs version : $NODEJS_VERSION"
+
+STORM_SRC_ROOT_DIR=$1
+
+TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+cd ${STORM_SRC_ROOT_DIR}
+
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install -DskipTests -Pnative
+BUILD_RET_VAL=$?
+
+if [[ "$BUILD_RET_VAL" != "0" ]];
+then
+ cat "install.txt"
+fi
+
+exit ${BUILD_RET_VAL}
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/dev-tools/travis/travis-script.sh
----------------------------------------------------------------------
diff --git a/dev-tools/travis/travis-script.sh b/dev-tools/travis/travis-script.sh
new file mode 100755
index 0000000..e791c9b
--- /dev/null
+++ b/dev-tools/travis/travis-script.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PYTHON_VERSION_TO_FILE=`python -V > /tmp/python_version 2>&1`
+PYTHON_VERSION=`cat /tmp/python_version`
+RUBY_VERSION=`ruby -v`
+NODEJS_VERSION=`node -v`
+
+echo "Python version : $PYTHON_VERSION"
+echo "Ruby version : $RUBY_VERSION"
+echo "NodeJs version : $NODEJS_VERSION"
+
+
+STORM_SRC_ROOT_DIR=$1
+
+TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+cd ${STORM_SRC_ROOT_DIR}
+
+# We should concern that Travis CI could be very slow cause it uses VM
+export STORM_TEST_TIMEOUT_MS=100000
+
+# We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "test.txt" mvn test -fae -Pnative
+BUILD_RET_VAL=$?
+
+for dir in `find . -type d -and -wholename \*/target/\*-reports`;
+do
+ echo "Looking for errors in ${dir}"
+ python ${TRAVIS_SCRIPT_DIR}/print-errors-from-test-reports.py "${dir}"
+done
+
+exit ${BUILD_RET_VAL}
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7dd2e29..8c5929f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -594,6 +594,14 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.18.1</version>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2.2</version>
</plugin>
@@ -630,7 +638,7 @@
<plugin>
<groupId>com.theoryinpractise</groupId>
<artifactId>clojure-maven-plugin</artifactId>
- <version>1.3.18</version>
+ <version>1.7.1</version>
<extensions>true</extensions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 69711ff..bd5ba2d 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -31,7 +31,8 @@
<properties>
<worker-launcher.conf.dir>/etc/storm</worker-launcher.conf.dir>
- <worker-launcher.additional_cflags />
+ <worker-launcher.additional_cflags></worker-launcher.additional_cflags>
+ <argLine></argLine>
</properties>
<dependencies>
@@ -368,9 +369,11 @@
<id>test-clojure</id>
<phase>test</phase>
<goals>
- <goal>test-with-junit</goal>
+ <goal>test</goal>
</goals>
<configuration>
+ <junitOutput>true</junitOutput>
+ <testScript>test/resources/test_runner.clj</testScript>
<!-- argLine is set by JaCoCo for code coverage -->
<vmargs>${argLine} ${test.extra.args}</vmargs>
</configuration>
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/storm-core/test/resources/log4j2-test.xml
----------------------------------------------------------------------
diff --git a/storm-core/test/resources/log4j2-test.xml b/storm-core/test/resources/log4j2-test.xml
index 4b12147..e8ae19e 100644
--- a/storm-core/test/resources/log4j2-test.xml
+++ b/storm-core/test/resources/log4j2-test.xml
@@ -18,7 +18,7 @@
<configuration monitorInterval="60">
<Appenders>
- <Console name="Console" target="SYSTEM_OUT">
+ <Console name="Console" target="SYSTEM_OUT" follow="true">
<PatternLayout pattern="%-4r [%t] %-5p %c{1.} - %msg%n"/>
</Console>
</Appenders>
http://git-wip-us.apache.org/repos/asf/storm/blob/098517b1/storm-core/test/resources/test_runner.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/resources/test_runner.clj b/storm-core/test/resources/test_runner.clj
new file mode 100644
index 0000000..ed13837
--- /dev/null
+++ b/storm-core/test/resources/test_runner.clj
@@ -0,0 +1,91 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+(ns org.apache.storm.testrunner)
+
+(import `java.util.Properties)
+(import `java.io.ByteArrayOutputStream)
+(import `java.io.FileInputStream)
+(import `java.io.FileOutputStream)
+(import `java.io.FileWriter)
+(import `java.io.File)
+(import `java.io.OutputStream)
+(import `java.io.PrintStream)
+(import `java.io.PrintWriter)
+(use 'clojure.test)
+(use 'clojure.test.junit)
+
+(def props (Properties.))
+(.load props (FileInputStream. (first *command-line-args*)))
+
+(def namespaces (into []
+ (for [[key val] props
+ :when (.startsWith key "ns.")]
+ (symbol val))))
+
+(def output-dir (.get props "outputDir"))
+
+(dorun (for [ns namespaces]
+ (require ns)))
+
+(.mkdirs (File. output-dir))
+
+(let [sys-out System/out
+ sys-err System/err
+ num-bad (atom 0)
+ original-junit-report junit-report
+ cached-out (atom nil)
+ test-error (atom 0)
+ orig-out *out*]
+ (dorun (for [ns namespaces]
+ (with-open [writer (FileWriter. (str output-dir "/" ns ".xml"))
+ out-stream (FileOutputStream. (str output-dir "/" ns "-output.txt"))
+ out-writer (PrintStream. out-stream true)]
+ (.println sys-out (str "Running " ns))
+ (try
+ (System/setOut out-writer)
+ (System/setErr out-writer)
+ (binding [*test-out* writer
+ *out* (PrintWriter. out-stream true)
+ junit-report (fn [data]
+ (let [type (data :type)]
+ (cond
+ (= type :begin-test-var) (do
+ (reset! cached-out (ByteArrayOutputStream.))
+ (let [writer (PrintStream. @cached-out true)]
+ (set! *out* (PrintWriter. @cached-out true))
+ (System/setOut writer)
+ (System/setErr writer))
+ (reset! test-error 0))
+ (= type :end-test-var) (do
+ (.write out-writer (.toByteArray @cached-out))
+ (when (> @test-error 0)
+ (with-test-out
+ (start-element 'system-out true)
+ (element-content (String. (.toByteArray @cached-out)))
+ (finish-element 'system-out true))))
+ (= type :fail) (swap! test-error inc)
+ (= type :error) (swap! test-error inc)))
+ (original-junit-report data))]
+ (with-junit-output
+ (let [result (run-tests ns)]
+ (.println sys-out (str "Tests run: " (result :test) ", Passed: " (result :pass) ", Failures: " (result :fail) ", Errors: " (result :error)))
+ (reset! num-bad (+ @num-bad (result :error) (result :fail))))))
+ (finally
+ (System/setOut sys-out)
+ (System/setErr sys-err))))))
+ (shutdown-agents)
+ (System/exit (if (> @num-bad 0) 1 0)))
[5/6] storm git commit: Switched to using CDATA for output and fixed
race conditions around XML output
Posted by ka...@apache.org.
Switched to using CDATA for output and fixed race conditions around XML output
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6825925b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6825925b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6825925b
Branch: refs/heads/0.10.x-branch
Commit: 6825925b8f50107929d65bbf51fce78c1d7a39ff
Parents: 524965a
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Jun 18 12:30:39 2015 -0500
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 09:47:13 2015 +0900
----------------------------------------------------------------------
.../storm/utils/XMLEscapeOutputStream.java | 74 --------------------
storm-core/test/resources/test_runner.clj | 35 ++++++---
2 files changed, 25 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6825925b/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java b/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
deleted file mode 100644
index ec6350a..0000000
--- a/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.utils;
-
-import java.io.FilterOutputStream;
-import java.io.OutputStream;
-import java.io.IOException;
-import java.io.ByteArrayOutputStream;
-
-import java.util.HashMap;
-
-/**
- * OutputStream that escapes XML characters. This is really slow and should only
- * be used for testing.
- */
-public class XMLEscapeOutputStream extends FilterOutputStream {
-
- private static final HashMap<Byte, byte[]> ESCAPE_MAP = new HashMap<Byte, byte[]>();
- private static void addMapping(String key, String val) {
- ESCAPE_MAP.put(key.getBytes()[0], val.getBytes());
- }
- static {
- addMapping("\"", """);
- addMapping("'", "'");
- addMapping("<", "<");
- addMapping(">", ">");
- addMapping("&", "&");
- }
-
- public XMLEscapeOutputStream(OutputStream other) {
- super(other);
- }
-
- public void write(byte[] b) throws IOException {
- this.write(b, 0, b.length);
- }
-
- public void write(byte[] b, int off, int len) throws IOException {
- ByteArrayOutputStream buff = new ByteArrayOutputStream();
- for (int i = 0; i < len; i++) {
- byte[] mapped = ESCAPE_MAP.get(b[i+off]);
- if (mapped == null) {
- buff.write(b[i+off]);
- } else {
- buff.write(mapped);
- }
- }
- out.write(buff.toByteArray());
- }
-
- public void write(int b) throws IOException {
- byte[] mapped = ESCAPE_MAP.get((byte)b);
- if (mapped == null) {
- out.write(b);
- } else {
- out.write(mapped);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/6825925b/storm-core/test/resources/test_runner.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/resources/test_runner.clj b/storm-core/test/resources/test_runner.clj
index 952a52c..c10fec3 100644
--- a/storm-core/test/resources/test_runner.clj
+++ b/storm-core/test/resources/test_runner.clj
@@ -26,7 +26,6 @@
(import `java.io.OutputStreamWriter)
(import `java.io.PrintStream)
(import `java.io.PrintWriter)
-(import `backtype.storm.utils.XMLEscapeOutputStream)
(use 'clojure.test)
(use 'clojure.test.junit)
@@ -52,40 +51,56 @@
orig-out *out*]
(dorun (for [ns namespaces]
(with-open [out-stream (FileOutputStream. (str output-dir "/" ns ".xml"))
- escape-out-stream (XMLEscapeOutputStream. out-stream)
- test-print-writer (PrintWriter. out-stream true)
- print-writer (PrintWriter. escape-out-stream true)
- print-stream (PrintStream. escape-out-stream true)]
- (System/setOut print-stream)
- (System/setErr print-stream)
+ print-writer (PrintWriter. out-stream true)
+ print-stream (PrintStream. out-stream true)]
(.println sys-out (str "Running " ns))
(try
(let [in-sys-out (atom false)]
- (binding [*test-out* test-print-writer
- *out* print-writer
+ (binding [*test-out* print-writer
+ *out* orig-out
junit-report (fn [data]
(let [type (data :type)]
(cond
(= type :begin-test-var) (do
(when @in-sys-out
(reset! in-sys-out false)
+ (System/setOut sys-out)
+ (System/setErr sys-err)
+ (set! *out* orig-out)
(with-test-out
+ (print "]]>")
(finish-element 'system-out true)))
(original-junit-report data)
(reset! in-sys-out true)
(with-test-out
- (start-element 'system-out true)))
+ (start-element 'system-out true)
+ (print "<![CDATA[") (flush))
+ (System/setOut print-stream)
+ (System/setErr print-stream)
+ (set! *out* print-writer))
(= type :end-test-var) (when @in-sys-out
(reset! in-sys-out false)
+ (System/setOut sys-out)
+ (System/setErr sys-err)
+ (set! *out* orig-out)
(with-test-out
+ (print "]]>")
(finish-element 'system-out true)))
(= type :fail) (when @in-sys-out
(reset! in-sys-out false)
+ (System/setOut sys-out)
+ (System/setErr sys-err)
+ (set! *out* orig-out)
(with-test-out
+ (print "]]>")
(finish-element 'system-out true)))
(= type :error) (when @in-sys-out
(reset! in-sys-out false)
+ (System/setOut sys-out)
+ (System/setErr sys-err)
+ (set! *out* orig-out)
(with-test-out
+ (print "]]>")
(finish-element 'system-out true))))
(if (not (= type :begin-test-var)) (original-junit-report data))))]
(with-junit-output
[4/6] storm git commit: Improved the collection of log messages,
so when Runtime.halt is called errors still show up
Posted by ka...@apache.org.
Improved the collection of log messages, so when Runtime.halt is called errors still show up
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/524965a7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/524965a7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/524965a7
Branch: refs/heads/0.10.x-branch
Commit: 524965a7342994a6f5e5ce915470c6e29ad9db12
Parents: 6c36860
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Jun 17 13:52:45 2015 -0500
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 09:47:06 2015 +0900
----------------------------------------------------------------------
.../storm/utils/XMLEscapeOutputStream.java | 74 ++++++++++++++++++++
storm-core/test/resources/test_runner.clj | 58 ++++++++-------
2 files changed, 107 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/524965a7/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java b/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
new file mode 100644
index 0000000..ec6350a
--- /dev/null
+++ b/storm-core/test/jvm/backtype/storm/utils/XMLEscapeOutputStream.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm.utils;
+
+import java.io.FilterOutputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+
+import java.util.HashMap;
+
+/**
+ * OutputStream that escapes XML characters. This is really slow and should only
+ * be used for testing.
+ */
+public class XMLEscapeOutputStream extends FilterOutputStream {
+
+ private static final HashMap<Byte, byte[]> ESCAPE_MAP = new HashMap<Byte, byte[]>();
+ private static void addMapping(String key, String val) {
+ ESCAPE_MAP.put(key.getBytes()[0], val.getBytes());
+ }
+ static {
+ addMapping("\"", """);
+ addMapping("'", "'");
+ addMapping("<", "<");
+ addMapping(">", ">");
+ addMapping("&", "&");
+ }
+
+ public XMLEscapeOutputStream(OutputStream other) {
+ super(other);
+ }
+
+ public void write(byte[] b) throws IOException {
+ this.write(b, 0, b.length);
+ }
+
+ public void write(byte[] b, int off, int len) throws IOException {
+ ByteArrayOutputStream buff = new ByteArrayOutputStream();
+ for (int i = 0; i < len; i++) {
+ byte[] mapped = ESCAPE_MAP.get(b[i+off]);
+ if (mapped == null) {
+ buff.write(b[i+off]);
+ } else {
+ buff.write(mapped);
+ }
+ }
+ out.write(buff.toByteArray());
+ }
+
+ public void write(int b) throws IOException {
+ byte[] mapped = ESCAPE_MAP.get((byte)b);
+ if (mapped == null) {
+ out.write(b);
+ } else {
+ out.write(mapped);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/524965a7/storm-core/test/resources/test_runner.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/resources/test_runner.clj b/storm-core/test/resources/test_runner.clj
index ed13837..952a52c 100644
--- a/storm-core/test/resources/test_runner.clj
+++ b/storm-core/test/resources/test_runner.clj
@@ -23,8 +23,10 @@
(import `java.io.FileWriter)
(import `java.io.File)
(import `java.io.OutputStream)
+(import `java.io.OutputStreamWriter)
(import `java.io.PrintStream)
(import `java.io.PrintWriter)
+(import `backtype.storm.utils.XMLEscapeOutputStream)
(use 'clojure.test)
(use 'clojure.test.junit)
@@ -47,43 +49,49 @@
sys-err System/err
num-bad (atom 0)
original-junit-report junit-report
- cached-out (atom nil)
- test-error (atom 0)
orig-out *out*]
(dorun (for [ns namespaces]
- (with-open [writer (FileWriter. (str output-dir "/" ns ".xml"))
- out-stream (FileOutputStream. (str output-dir "/" ns "-output.txt"))
- out-writer (PrintStream. out-stream true)]
+ (with-open [out-stream (FileOutputStream. (str output-dir "/" ns ".xml"))
+ escape-out-stream (XMLEscapeOutputStream. out-stream)
+ test-print-writer (PrintWriter. out-stream true)
+ print-writer (PrintWriter. escape-out-stream true)
+ print-stream (PrintStream. escape-out-stream true)]
+ (System/setOut print-stream)
+ (System/setErr print-stream)
(.println sys-out (str "Running " ns))
(try
- (System/setOut out-writer)
- (System/setErr out-writer)
- (binding [*test-out* writer
- *out* (PrintWriter. out-stream true)
+ (let [in-sys-out (atom false)]
+ (binding [*test-out* test-print-writer
+ *out* print-writer
junit-report (fn [data]
(let [type (data :type)]
(cond
(= type :begin-test-var) (do
- (reset! cached-out (ByteArrayOutputStream.))
- (let [writer (PrintStream. @cached-out true)]
- (set! *out* (PrintWriter. @cached-out true))
- (System/setOut writer)
- (System/setErr writer))
- (reset! test-error 0))
- (= type :end-test-var) (do
- (.write out-writer (.toByteArray @cached-out))
- (when (> @test-error 0)
+ (when @in-sys-out
+ (reset! in-sys-out false)
(with-test-out
- (start-element 'system-out true)
- (element-content (String. (.toByteArray @cached-out)))
- (finish-element 'system-out true))))
- (= type :fail) (swap! test-error inc)
- (= type :error) (swap! test-error inc)))
- (original-junit-report data))]
+ (finish-element 'system-out true)))
+ (original-junit-report data)
+ (reset! in-sys-out true)
+ (with-test-out
+ (start-element 'system-out true)))
+ (= type :end-test-var) (when @in-sys-out
+ (reset! in-sys-out false)
+ (with-test-out
+ (finish-element 'system-out true)))
+ (= type :fail) (when @in-sys-out
+ (reset! in-sys-out false)
+ (with-test-out
+ (finish-element 'system-out true)))
+ (= type :error) (when @in-sys-out
+ (reset! in-sys-out false)
+ (with-test-out
+ (finish-element 'system-out true))))
+ (if (not (= type :begin-test-var)) (original-junit-report data))))]
(with-junit-output
(let [result (run-tests ns)]
(.println sys-out (str "Tests run: " (result :test) ", Passed: " (result :pass) ", Failures: " (result :fail) ", Errors: " (result :error)))
- (reset! num-bad (+ @num-bad (result :error) (result :fail))))))
+ (reset! num-bad (+ @num-bad (result :error) (result :fail)))))))
(finally
(System/setOut sys-out)
(System/setErr sys-err))))))
[3/6] storm git commit: Updated error handling of print errors
Posted by ka...@apache.org.
Updated error handling of print errors
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6c36860a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6c36860a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6c36860a
Branch: refs/heads/0.10.x-branch
Commit: 6c36860a85478e2e237dcfccd96586a7fd34e3f9
Parents: 3207e94
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Jun 17 11:06:46 2015 -0500
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 09:47:03 2015 +0900
----------------------------------------------------------------------
dev-tools/travis/print-errors-from-test-reports.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6c36860a/dev-tools/travis/print-errors-from-test-reports.py
----------------------------------------------------------------------
diff --git a/dev-tools/travis/print-errors-from-test-reports.py b/dev-tools/travis/print-errors-from-test-reports.py
index 95cc7b7..a9fe9b8 100644
--- a/dev-tools/travis/print-errors-from-test-reports.py
+++ b/dev-tools/travis/print-errors-from-test-reports.py
@@ -34,7 +34,15 @@ def print_detail_information(testcase, fail_or_error):
def print_error_reports_from_report_file(file_path):
tree = ElementTree()
- tree.parse(file_path)
+ try:
+ tree.parse(file_path)
+ except:
+ print "-" * 50
+ print "Error parsing %s"%file_path
+ f = open(file_path, "r");
+ print f.read();
+ print "-" * 50
+ return
testcases = tree.findall(".//testcase")
for testcase in testcases:
[2/6] storm git commit: Logging is better now. Turned off logging to
file durring tests.
Posted by ka...@apache.org.
Logging is better now. Turned off logging to file durring tests.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3207e949
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3207e949
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3207e949
Branch: refs/heads/0.10.x-branch
Commit: 3207e949847ea655bf40b79faa01e739a38a0804
Parents: 098517b
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Jun 17 10:52:24 2015 -0500
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 09:46:59 2015 +0900
----------------------------------------------------------------------
dev-tools/travis/travis-script.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3207e949/dev-tools/travis/travis-script.sh
----------------------------------------------------------------------
diff --git a/dev-tools/travis/travis-script.sh b/dev-tools/travis/travis-script.sh
index e791c9b..77b9fb6 100755
--- a/dev-tools/travis/travis-script.sh
+++ b/dev-tools/travis/travis-script.sh
@@ -31,7 +31,7 @@ cd ${STORM_SRC_ROOT_DIR}
export STORM_TEST_TIMEOUT_MS=100000
# We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script
-python ${TRAVIS_SCRIPT_DIR}/save-logs.py "test.txt" mvn test -fae -Pnative
+mvn test -fae -Pnative
BUILD_RET_VAL=$?
for dir in `find . -type d -and -wholename \*/target/\*-reports`;
[6/6] storm git commit: add STORM-803 to CHANGELOG.md
Posted by ka...@apache.org.
add STORM-803 to CHANGELOG.md
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/150f7f6e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/150f7f6e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/150f7f6e
Branch: refs/heads/0.10.x-branch
Commit: 150f7f6e349ff58003a36186a8ee2264f70556d8
Parents: 6825925
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Sep 18 10:00:59 2015 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Sep 18 10:00:59 2015 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/150f7f6e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f14c37..c8c60bf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.10.0
+ * STORM-803: Cleanup travis-ci build and logs
* STORM-1027: Use overflow buffer for emitting metrics
* STORM-1024: log4j changes leaving ${sys:storm.log.dir} under STORM_HOME dir
* STORM-996: netty-unit-tests/test-batch demonstrates out-of-order delivery