You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by aw...@apache.org on 2017/11/15 21:36:21 UTC
cassandra-dtest git commit: Add plumbing to cassandra dtests to allow
collecting test names for multiple builds as well as merging test output
Repository: cassandra-dtest
Updated Branches:
refs/heads/master 8e65211b9 -> ac0ce6044
Add plumbing to cassandra dtests to allow collecting test names for multiple builds as well as merging test output
Patch by Ariel Weisberg; Reviewd by Michael Kjellman for CASSANDRA-14017
Project: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/commit/ac0ce604
Tree: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/tree/ac0ce604
Diff: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/diff/ac0ce604
Branch: refs/heads/master
Commit: ac0ce60443ee59a8ad333f5d5a08bc6efa574a53
Parents: 8e65211
Author: Ariel Weisberg <aw...@apple.com>
Authored: Wed Nov 15 16:17:39 2017 -0500
Committer: Ariel Weisberg <aw...@apple.com>
Committed: Wed Nov 15 16:32:47 2017 -0500
----------------------------------------------------------------------
plugins/dtestcollect.py | 93 ++++++++++++
plugins/dtesttag.py | 47 ++++++
plugins/dtestxunit.py | 346 +++++++++++++++++++++++++++++++++++++++++++
run_dtests.py | 20 ++-
4 files changed, 500 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestcollect.py
----------------------------------------------------------------------
diff --git a/plugins/dtestcollect.py b/plugins/dtestcollect.py
new file mode 100644
index 0000000..e8a3c8a
--- /dev/null
+++ b/plugins/dtestcollect.py
@@ -0,0 +1,93 @@
+from collections import namedtuple
+
+from pprint import pprint
+import os
+import inspect
+from nose.plugins.base import Plugin
+from nose.case import Test
+import logging
+import unittest
+
+log = logging.getLogger(__name__)
+
+class DTestCollect(Plugin):
+ """
+ Collect and output test names only, don't run any tests.
+ """
+ name = 'dtest_collect'
+ enableOpt = 'dtest_collect_only'
+
+ def options(self, parser, env):
+ """Register commandline options.
+ """
+ parser.add_option('--dtest-collect-only',
+ action='store_true',
+ dest=self.enableOpt,
+ default=env.get('DTEST_NOSE_COLLECT_ONLY'),
+ help="Enable collect-only: %s [COLLECT_ONLY]" %
+ (self.help()))
+
+ def prepareTestLoader(self, loader):
+ """Install collect-only suite class in TestLoader.
+ """
+ # Disable context awareness
+ log.debug("Preparing test loader")
+ loader.suiteClass = TestSuiteFactory(self.conf)
+
+ def prepareTestCase(self, test):
+ """Replace actual test with dummy that always passes.
+ """
+ # Return something that always passes
+ log.debug("Preparing test case %s", test)
+ if not isinstance(test, Test):
+ return
+ def run(result):
+ # We need to make these plugin calls because there won't be
+ # a result proxy, due to using a stripped-down test suite
+ self.conf.plugins.startTest(test)
+ result.startTest(test)
+ self.conf.plugins.addSuccess(test)
+ result.addSuccess(test)
+ self.conf.plugins.stopTest(test)
+ result.stopTest(test)
+ return run
+
+ def describeTest(self, test):
+ tag = os.getenv('TEST_TAG', '')
+ if tag == '':
+ tag = test.test._testMethodName
+ else:
+ tag = test.test._testMethodName + "-" + tag
+ retval = "%s:%s.%s" % (test.test.__module__, test.test.__class__.__name__, tag)
+ return retval;
+
+class TestSuiteFactory:
+ """
+ Factory for producing configured test suites.
+ """
+ def __init__(self, conf):
+ self.conf = conf
+
+ def __call__(self, tests=(), **kw):
+ return TestSuite(tests, conf=self.conf)
+
+
+class TestSuite(unittest.TestSuite):
+ """
+ Basic test suite that bypasses most proxy and plugin calls, but does
+ wrap tests in a nose.case.Test so prepareTestCase will be called.
+ """
+ def __init__(self, tests=(), conf=None):
+ self.conf = conf
+ # Exec lazy suites: makes discovery depth-first
+ if callable(tests):
+ tests = tests()
+ log.debug("TestSuite(%r)", tests)
+ unittest.TestSuite.__init__(self, tests)
+
+ def addTest(self, test):
+ log.debug("Add test %s", test)
+ if isinstance(test, unittest.TestSuite):
+ self._tests.append(test)
+ else:
+ self._tests.append(Test(test, config=self.conf))
http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtesttag.py
----------------------------------------------------------------------
diff --git a/plugins/dtesttag.py b/plugins/dtesttag.py
new file mode 100644
index 0000000..94effcb
--- /dev/null
+++ b/plugins/dtesttag.py
@@ -0,0 +1,47 @@
+from collections import namedtuple
+
+from nose import plugins
+from pprint import pprint
+import os
+import inspect
+
+class DTestTag(plugins.Plugin):
+ enabled = True # if this plugin is loaded at all, we're using it
+ name = 'dtest_tag'
+
+ def __init__(self):
+ pass
+
+ def configure(self, options, conf):
+ pass
+
+ def nice_classname(self, obj):
+ """Returns a nice name for class object or class instance.
+
+ >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+ '...Exception'
+ >>> nice_classname(Exception) # doctest: +ELLIPSIS
+ '...Exception'
+
+ """
+ if inspect.isclass(obj):
+ cls_name = obj.__name__
+ else:
+ cls_name = obj.__class__.__name__
+ mod = inspect.getmodule(obj)
+ if mod:
+ name = mod.__name__
+ # jython
+ if name.startswith('org.python.core.'):
+ name = name[len('org.python.core.'):]
+ return "%s.%s" % (name, cls_name)
+ else:
+ return cls_name
+
+ def describeTest(self, test):
+ tag = os.getenv('TEST_TAG', '')
+ if tag == '':
+ tag = test.test._testMethodName
+ else:
+ tag = test.test._testMethodName + "-" + tag
+ return "%s (%s)" % (tag, self.nice_classname(test.test))
http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestxunit.py
----------------------------------------------------------------------
diff --git a/plugins/dtestxunit.py b/plugins/dtestxunit.py
new file mode 100644
index 0000000..c4a041f
--- /dev/null
+++ b/plugins/dtestxunit.py
@@ -0,0 +1,346 @@
+"""This plugin provides test results in the standard XUnit XML format.
+
+It's designed for the `Jenkins`_ (previously Hudson) continuous build
+system, but will probably work for anything else that understands an
+XUnit-formatted XML representation of test results.
+
+Add this shell command to your builder ::
+
+ nosetests --with-dtestxunit
+
+And by default a file named nosetests.xml will be written to the
+working directory.
+
+In a Jenkins builder, tick the box named "Publish JUnit test result report"
+under the Post-build Actions and enter this value for Test report XMLs::
+
+ **/nosetests.xml
+
+If you need to change the name or location of the file, you can set the
+``--dtestxunit-file`` option.
+
+If you need to change the name of the test suite, you can set the
+``--dtestxunit-testsuite-name`` option.
+
+Here is an abbreviated version of what an XML test report might look like::
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuite name="nosetests" tests="1" errors="1" failures="0" skip="0">
+ <testcase classname="path_to_test_suite.TestSomething"
+ name="test_it" time="0">
+ <error type="exceptions.TypeError" message="oops, wrong type">
+ Traceback (most recent call last):
+ ...
+ TypeError: oops, wrong type
+ </error>
+ </testcase>
+ </testsuite>
+
+.. _Jenkins: http://jenkins-ci.org/
+
+"""
+import codecs
+import doctest
+import os
+import sys
+import traceback
+import re
+import inspect
+from StringIO import StringIO
+from time import time
+from xml.sax import saxutils
+
+from nose.plugins.base import Plugin
+from nose.exc import SkipTest
+from nose.pyversion import force_unicode, format_exception
+
+# Invalid XML characters, control characters 0-31 sans \t, \n and \r
+CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]")
+
+TEST_ID = re.compile(r'^(.*?)(\(.*\))$')
+
+def xml_safe(value):
+ """Replaces invalid XML characters with '?'."""
+ return CONTROL_CHARACTERS.sub('?', value)
+
+def escape_cdata(cdata):
+ """Escape a string for an XML CDATA section."""
+ return xml_safe(cdata).replace(']]>', ']]>]]><![CDATA[')
+
+def id_split(idval):
+ m = TEST_ID.match(idval)
+ retval = []
+ if m:
+ name, fargs = m.groups()
+ head, tail = name.rsplit(".", 1)
+ retval = [head, tail+fargs]
+ else:
+ retval = idval.rsplit(".", 1)
+ tag = os.getenv('TEST_TAG', '')
+ if tag != '':
+ retval[-1] = retval[-1] + "-" + tag
+ return retval
+
+def nice_classname(obj):
+ """Returns a nice name for class object or class instance.
+
+ >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+ '...Exception'
+ >>> nice_classname(Exception) # doctest: +ELLIPSIS
+ '...Exception'
+
+ """
+ if inspect.isclass(obj):
+ cls_name = obj.__name__
+ else:
+ cls_name = obj.__class__.__name__
+ mod = inspect.getmodule(obj)
+ if mod:
+ name = mod.__name__
+ # jython
+ if name.startswith('org.python.core.'):
+ name = name[len('org.python.core.'):]
+ return "%s.%s" % (name, cls_name)
+ else:
+ return cls_name
+
+def exc_message(exc_info):
+ """Return the exception's message."""
+ exc = exc_info[1]
+ if exc is None:
+ # str exception
+ result = exc_info[0]
+ else:
+ try:
+ result = str(exc)
+ except UnicodeEncodeError:
+ try:
+ result = unicode(exc)
+ except UnicodeError:
+ # Fallback to args as neither str nor
+ # unicode(Exception(u'\xe6')) work in Python < 2.6
+ result = exc.args[0]
+ result = force_unicode(result, 'UTF-8')
+ return xml_safe(result)
+
+class Tee(object):
+ def __init__(self, encoding, *args):
+ self._encoding = encoding
+ self._streams = args
+
+ def write(self, data):
+ data = force_unicode(data, self._encoding)
+ for s in self._streams:
+ s.write(data)
+
+ def writelines(self, lines):
+ for line in lines:
+ self.write(line)
+
+ def flush(self):
+ for s in self._streams:
+ s.flush()
+
+ def isatty(self):
+ return False
+
+
+class DTestXunit(Plugin):
+ """This plugin provides test results in the standard XUnit XML format."""
+ name = 'dtestxunit'
+ score = 1500
+ encoding = 'UTF-8'
+ error_report_file = None
+
+ def __init__(self):
+ super(DTestXunit, self).__init__()
+ self._capture_stack = []
+ self._currentStdout = None
+ self._currentStderr = None
+
+ def _timeTaken(self):
+ if hasattr(self, '_timer'):
+ taken = time() - self._timer
+ else:
+ # test died before it ran (probably error in setup())
+ # or success/failure added before test started probably
+ # due to custom TestResult munging
+ taken = 0.0
+ return taken
+
+ def _quoteattr(self, attr):
+ """Escape an XML attribute. Value can be unicode."""
+ attr = xml_safe(attr)
+ return saxutils.quoteattr(attr)
+
+ def options(self, parser, env):
+ """Sets additional command line options."""
+ Plugin.options(self, parser, env)
+ parser.add_option(
+ '--dtestxunit-file', action='store',
+ dest='dtestxunit_file', metavar="FILE",
+ default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'),
+ help=("Path to xml file to store the xunit report in. "
+ "Default is nosetests.xml in the working directory "
+ "[NOSE_XUNIT_FILE]"))
+
+ parser.add_option(
+ '--dtestxunit-testsuite-name', action='store',
+ dest='dtestxunit_testsuite_name', metavar="PACKAGE",
+ default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'),
+ help=("Name of the testsuite in the xunit xml, generated by plugin. "
+ "Default test suite name is nosetests."))
+
+ def configure(self, options, config):
+ """Configures the xunit plugin."""
+ Plugin.configure(self, options, config)
+ self.config = config
+ if self.enabled:
+ self.stats = {'errors': 0,
+ 'failures': 0,
+ 'passes': 0,
+ 'skipped': 0
+ }
+ self.errorlist = []
+ self.error_report_file_name = os.path.realpath(options.dtestxunit_file)
+ self.xunit_testsuite_name = options.dtestxunit_testsuite_name
+
+ def report(self, stream):
+ """Writes an Xunit-formatted XML file
+
+ The file includes a report of test errors and failures.
+
+ """
+ self.error_report_file = codecs.open(self.error_report_file_name, 'w',
+ self.encoding, 'replace')
+ self.stats['encoding'] = self.encoding
+ self.stats['testsuite_name'] = self.xunit_testsuite_name
+ self.stats['total'] = (self.stats['errors'] + self.stats['failures']
+ + self.stats['passes'] + self.stats['skipped'])
+ self.error_report_file.write(
+ u'<?xml version="1.0" encoding="%(encoding)s"?>'
+ u'<testsuite name="%(testsuite_name)s" tests="%(total)d" '
+ u'errors="%(errors)d" failures="%(failures)d" '
+ u'skip="%(skipped)d">' % self.stats)
+ self.error_report_file.write(u''.join([force_unicode(e, self.encoding)
+ for e in self.errorlist]))
+ self.error_report_file.write(u'</testsuite>')
+ self.error_report_file.close()
+ if self.config.verbosity > 1:
+ stream.writeln("-" * 70)
+ stream.writeln("XML: %s" % self.error_report_file.name)
+
+ def _startCapture(self):
+ self._capture_stack.append((sys.stdout, sys.stderr))
+ self._currentStdout = StringIO()
+ self._currentStderr = StringIO()
+ sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout)
+ sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr)
+
+ def startContext(self, context):
+ self._startCapture()
+
+ def stopContext(self, context):
+ self._endCapture()
+
+ def beforeTest(self, test):
+ """Initializes a timer before starting a test."""
+ self._timer = time()
+ self._startCapture()
+
+ def _endCapture(self):
+ if self._capture_stack:
+ sys.stdout, sys.stderr = self._capture_stack.pop()
+
+ def afterTest(self, test):
+ self._endCapture()
+ self._currentStdout = None
+ self._currentStderr = None
+
+ def finalize(self, test):
+ while self._capture_stack:
+ self._endCapture()
+
+ def _getCapturedStdout(self):
+ if self._currentStdout:
+ value = self._currentStdout.getvalue()
+ if value:
+ return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata(
+ value)
+ return ''
+
+ def _getCapturedStderr(self):
+ if self._currentStderr:
+ value = self._currentStderr.getvalue()
+ if value:
+ return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata(
+ value)
+ return ''
+
+ def addError(self, test, err, capt=None):
+ """Add error output to Xunit report.
+ """
+ taken = self._timeTaken()
+
+ if issubclass(err[0], SkipTest):
+ type = 'skipped'
+ self.stats['skipped'] += 1
+ else:
+ type = 'error'
+ self.stats['errors'] += 1
+
+ tb = format_exception(err, self.encoding)
+ id = test.id()
+
+ self.errorlist.append(
+ u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+ u'<%(type)s type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+ u'</%(type)s>%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'type': type,
+ 'errtype': self._quoteattr(nice_classname(err[0])),
+ 'message': self._quoteattr(exc_message(err)),
+ 'tb': escape_cdata(tb),
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
+
+ def addFailure(self, test, err, capt=None, tb_info=None):
+ """Add failure output to Xunit report.
+ """
+ taken = self._timeTaken()
+ tb = format_exception(err, self.encoding)
+ self.stats['failures'] += 1
+ id = test.id()
+
+ self.errorlist.append(
+ u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+ u'<failure type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+ u'</failure>%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'errtype': self._quoteattr(nice_classname(err[0])),
+ 'message': self._quoteattr(exc_message(err)),
+ 'tb': escape_cdata(tb),
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
+
+ def addSuccess(self, test, capt=None):
+ """Add success output to Xunit report.
+ """
+ taken = self._timeTaken()
+ self.stats['passes'] += 1
+ id = test.id()
+ self.errorlist.append(
+ '<testcase classname=%(cls)s name=%(name)s '
+ 'time="%(taken).3f">%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/run_dtests.py
----------------------------------------------------------------------
diff --git a/run_dtests.py b/run_dtests.py
index cb68827..4f2d187 100755
--- a/run_dtests.py
+++ b/run_dtests.py
@@ -23,9 +23,11 @@ example:
from __future__ import print_function
import subprocess
+import sys
+import os
from collections import namedtuple
from itertools import product
-from os import getcwd
+from os import getcwd, environ
from tempfile import NamedTemporaryFile
from docopt import docopt
@@ -212,9 +214,15 @@ if __name__ == '__main__':
# How do we execute code in a new interpreter each time? Generate the
# code as text, then shell out to a new interpreter.
to_execute = (
- 'import nose\n'
- 'from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n'
- 'nose.main(addplugins=[DtestConfigPlugin({config})])\n'
+ "import nose\n" +
+ "from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n" +
+ "from plugins.dtestxunit import DTestXunit\n" +
+ "from plugins.dtesttag import DTestTag\n" +
+ "from plugins.dtestcollect import DTestCollect\n" +
+ "import sys\n" +
+ "print sys.getrecursionlimit()\n" +
+ "print sys.setrecursionlimit(8000)\n" +
+ ("nose.main(addplugins=[DtestConfigPlugin({config}), DTestXunit(), DTestCollect(), DTestTag()])\n" if "TEST_TAG" in environ else "nose.main(addplugins=[DtestConfigPlugin({config}), DTestCollect(), DTestXunit()])\n")
).format(config=repr(config))
temp = NamedTemporaryFile(dir=getcwd())
debug('Writing the following to {}:'.format(temp.name))
@@ -228,7 +236,7 @@ if __name__ == '__main__':
# command line are treated one way, args passed in as
# nose.main(argv=...) are treated another. Compare with the options
# -xsv for an example.
- cmd_list = ['python', temp.name] + nose_argv
+ cmd_list = [sys.executable, temp.name] + nose_argv
debug('subprocess.call-ing {cmd_list}'.format(cmd_list=cmd_list))
if options['--dry-run']:
@@ -240,7 +248,7 @@ if __name__ == '__main__':
contents=contents
))
else:
- results.append(subprocess.call(cmd_list))
+ results.append(subprocess.call(cmd_list, env=os.environ.copy()))
# separate the end of the last subprocess.call output from the
# beginning of the next by printing a newline.
print()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org