You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by aw...@apache.org on 2017/11/15 21:47:18 UTC
[50/50] cassandra git commit: Add plumbing to cassandra dtests to
allow collecting test names for multiple builds as well as merging test
output
Add plumbing to cassandra dtests to allow collecting test names for multiple builds as well as merging test output
Patch by Ariel Weisberg; Reviewd by Michael Kjellman for CASSANDRA-14017
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ac0ce604
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ac0ce604
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ac0ce604
Branch: refs/heads/master
Commit: ac0ce60443ee59a8ad333f5d5a08bc6efa574a53
Parents: 8e65211
Author: Ariel Weisberg <aw...@apple.com>
Authored: Wed Nov 15 16:17:39 2017 -0500
Committer: Ariel Weisberg <aw...@apple.com>
Committed: Wed Nov 15 16:32:47 2017 -0500
----------------------------------------------------------------------
plugins/dtestcollect.py | 93 ++++++++++++
plugins/dtesttag.py | 47 ++++++
plugins/dtestxunit.py | 346 +++++++++++++++++++++++++++++++++++++++++++
run_dtests.py | 20 ++-
4 files changed, 500 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ac0ce604/plugins/dtestcollect.py
----------------------------------------------------------------------
diff --git a/plugins/dtestcollect.py b/plugins/dtestcollect.py
new file mode 100644
index 0000000..e8a3c8a
--- /dev/null
+++ b/plugins/dtestcollect.py
@@ -0,0 +1,93 @@
+from collections import namedtuple
+
+from pprint import pprint
+import os
+import inspect
+from nose.plugins.base import Plugin
+from nose.case import Test
+import logging
+import unittest
+
+log = logging.getLogger(__name__)
+
+class DTestCollect(Plugin):
+ """
+ Collect and output test names only, don't run any tests.
+ """
+ name = 'dtest_collect'
+ enableOpt = 'dtest_collect_only'
+
+ def options(self, parser, env):
+ """Register commandline options.
+ """
+ parser.add_option('--dtest-collect-only',
+ action='store_true',
+ dest=self.enableOpt,
+ default=env.get('DTEST_NOSE_COLLECT_ONLY'),
+ help="Enable collect-only: %s [COLLECT_ONLY]" %
+ (self.help()))
+
+ def prepareTestLoader(self, loader):
+ """Install collect-only suite class in TestLoader.
+ """
+ # Disable context awareness
+ log.debug("Preparing test loader")
+ loader.suiteClass = TestSuiteFactory(self.conf)
+
+ def prepareTestCase(self, test):
+ """Replace actual test with dummy that always passes.
+ """
+ # Return something that always passes
+ log.debug("Preparing test case %s", test)
+ if not isinstance(test, Test):
+ return
+ def run(result):
+ # We need to make these plugin calls because there won't be
+ # a result proxy, due to using a stripped-down test suite
+ self.conf.plugins.startTest(test)
+ result.startTest(test)
+ self.conf.plugins.addSuccess(test)
+ result.addSuccess(test)
+ self.conf.plugins.stopTest(test)
+ result.stopTest(test)
+ return run
+
+ def describeTest(self, test):
+ tag = os.getenv('TEST_TAG', '')
+ if tag == '':
+ tag = test.test._testMethodName
+ else:
+ tag = test.test._testMethodName + "-" + tag
+ retval = "%s:%s.%s" % (test.test.__module__, test.test.__class__.__name__, tag)
+ return retval;
+
+class TestSuiteFactory:
+ """
+ Factory for producing configured test suites.
+ """
+ def __init__(self, conf):
+ self.conf = conf
+
+ def __call__(self, tests=(), **kw):
+ return TestSuite(tests, conf=self.conf)
+
+
+class TestSuite(unittest.TestSuite):
+ """
+ Basic test suite that bypasses most proxy and plugin calls, but does
+ wrap tests in a nose.case.Test so prepareTestCase will be called.
+ """
+ def __init__(self, tests=(), conf=None):
+ self.conf = conf
+ # Exec lazy suites: makes discovery depth-first
+ if callable(tests):
+ tests = tests()
+ log.debug("TestSuite(%r)", tests)
+ unittest.TestSuite.__init__(self, tests)
+
+ def addTest(self, test):
+ log.debug("Add test %s", test)
+ if isinstance(test, unittest.TestSuite):
+ self._tests.append(test)
+ else:
+ self._tests.append(Test(test, config=self.conf))
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ac0ce604/plugins/dtesttag.py
----------------------------------------------------------------------
diff --git a/plugins/dtesttag.py b/plugins/dtesttag.py
new file mode 100644
index 0000000..94effcb
--- /dev/null
+++ b/plugins/dtesttag.py
@@ -0,0 +1,47 @@
+from collections import namedtuple
+
+from nose import plugins
+from pprint import pprint
+import os
+import inspect
+
+class DTestTag(plugins.Plugin):
+ enabled = True # if this plugin is loaded at all, we're using it
+ name = 'dtest_tag'
+
+ def __init__(self):
+ pass
+
+ def configure(self, options, conf):
+ pass
+
+ def nice_classname(self, obj):
+ """Returns a nice name for class object or class instance.
+
+ >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+ '...Exception'
+ >>> nice_classname(Exception) # doctest: +ELLIPSIS
+ '...Exception'
+
+ """
+ if inspect.isclass(obj):
+ cls_name = obj.__name__
+ else:
+ cls_name = obj.__class__.__name__
+ mod = inspect.getmodule(obj)
+ if mod:
+ name = mod.__name__
+ # jython
+ if name.startswith('org.python.core.'):
+ name = name[len('org.python.core.'):]
+ return "%s.%s" % (name, cls_name)
+ else:
+ return cls_name
+
+ def describeTest(self, test):
+ tag = os.getenv('TEST_TAG', '')
+ if tag == '':
+ tag = test.test._testMethodName
+ else:
+ tag = test.test._testMethodName + "-" + tag
+ return "%s (%s)" % (tag, self.nice_classname(test.test))
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ac0ce604/plugins/dtestxunit.py
----------------------------------------------------------------------
diff --git a/plugins/dtestxunit.py b/plugins/dtestxunit.py
new file mode 100644
index 0000000..c4a041f
--- /dev/null
+++ b/plugins/dtestxunit.py
@@ -0,0 +1,346 @@
+"""This plugin provides test results in the standard XUnit XML format.
+
+It's designed for the `Jenkins`_ (previously Hudson) continuous build
+system, but will probably work for anything else that understands an
+XUnit-formatted XML representation of test results.
+
+Add this shell command to your builder ::
+
+ nosetests --with-dtestxunit
+
+And by default a file named nosetests.xml will be written to the
+working directory.
+
+In a Jenkins builder, tick the box named "Publish JUnit test result report"
+under the Post-build Actions and enter this value for Test report XMLs::
+
+ **/nosetests.xml
+
+If you need to change the name or location of the file, you can set the
+``--dtestxunit-file`` option.
+
+If you need to change the name of the test suite, you can set the
+``--dtestxunit-testsuite-name`` option.
+
+Here is an abbreviated version of what an XML test report might look like::
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <testsuite name="nosetests" tests="1" errors="1" failures="0" skip="0">
+ <testcase classname="path_to_test_suite.TestSomething"
+ name="test_it" time="0">
+ <error type="exceptions.TypeError" message="oops, wrong type">
+ Traceback (most recent call last):
+ ...
+ TypeError: oops, wrong type
+ </error>
+ </testcase>
+ </testsuite>
+
+.. _Jenkins: http://jenkins-ci.org/
+
+"""
+import codecs
+import doctest
+import os
+import sys
+import traceback
+import re
+import inspect
+from StringIO import StringIO
+from time import time
+from xml.sax import saxutils
+
+from nose.plugins.base import Plugin
+from nose.exc import SkipTest
+from nose.pyversion import force_unicode, format_exception
+
+# Invalid XML characters, control characters 0-31 sans \t, \n and \r
+CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]")
+
+TEST_ID = re.compile(r'^(.*?)(\(.*\))$')
+
+def xml_safe(value):
+ """Replaces invalid XML characters with '?'."""
+ return CONTROL_CHARACTERS.sub('?', value)
+
+def escape_cdata(cdata):
+ """Escape a string for an XML CDATA section."""
+ return xml_safe(cdata).replace(']]>', ']]>]]><![CDATA[')
+
+def id_split(idval):
+ m = TEST_ID.match(idval)
+ retval = []
+ if m:
+ name, fargs = m.groups()
+ head, tail = name.rsplit(".", 1)
+ retval = [head, tail+fargs]
+ else:
+ retval = idval.rsplit(".", 1)
+ tag = os.getenv('TEST_TAG', '')
+ if tag != '':
+ retval[-1] = retval[-1] + "-" + tag
+ return retval
+
+def nice_classname(obj):
+ """Returns a nice name for class object or class instance.
+
+ >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+ '...Exception'
+ >>> nice_classname(Exception) # doctest: +ELLIPSIS
+ '...Exception'
+
+ """
+ if inspect.isclass(obj):
+ cls_name = obj.__name__
+ else:
+ cls_name = obj.__class__.__name__
+ mod = inspect.getmodule(obj)
+ if mod:
+ name = mod.__name__
+ # jython
+ if name.startswith('org.python.core.'):
+ name = name[len('org.python.core.'):]
+ return "%s.%s" % (name, cls_name)
+ else:
+ return cls_name
+
+def exc_message(exc_info):
+ """Return the exception's message."""
+ exc = exc_info[1]
+ if exc is None:
+ # str exception
+ result = exc_info[0]
+ else:
+ try:
+ result = str(exc)
+ except UnicodeEncodeError:
+ try:
+ result = unicode(exc)
+ except UnicodeError:
+ # Fallback to args as neither str nor
+ # unicode(Exception(u'\xe6')) work in Python < 2.6
+ result = exc.args[0]
+ result = force_unicode(result, 'UTF-8')
+ return xml_safe(result)
+
+class Tee(object):
+ def __init__(self, encoding, *args):
+ self._encoding = encoding
+ self._streams = args
+
+ def write(self, data):
+ data = force_unicode(data, self._encoding)
+ for s in self._streams:
+ s.write(data)
+
+ def writelines(self, lines):
+ for line in lines:
+ self.write(line)
+
+ def flush(self):
+ for s in self._streams:
+ s.flush()
+
+ def isatty(self):
+ return False
+
+
+class DTestXunit(Plugin):
+ """This plugin provides test results in the standard XUnit XML format."""
+ name = 'dtestxunit'
+ score = 1500
+ encoding = 'UTF-8'
+ error_report_file = None
+
+ def __init__(self):
+ super(DTestXunit, self).__init__()
+ self._capture_stack = []
+ self._currentStdout = None
+ self._currentStderr = None
+
+ def _timeTaken(self):
+ if hasattr(self, '_timer'):
+ taken = time() - self._timer
+ else:
+ # test died before it ran (probably error in setup())
+ # or success/failure added before test started probably
+ # due to custom TestResult munging
+ taken = 0.0
+ return taken
+
+ def _quoteattr(self, attr):
+ """Escape an XML attribute. Value can be unicode."""
+ attr = xml_safe(attr)
+ return saxutils.quoteattr(attr)
+
+ def options(self, parser, env):
+ """Sets additional command line options."""
+ Plugin.options(self, parser, env)
+ parser.add_option(
+ '--dtestxunit-file', action='store',
+ dest='dtestxunit_file', metavar="FILE",
+ default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'),
+ help=("Path to xml file to store the xunit report in. "
+ "Default is nosetests.xml in the working directory "
+ "[NOSE_XUNIT_FILE]"))
+
+ parser.add_option(
+ '--dtestxunit-testsuite-name', action='store',
+ dest='dtestxunit_testsuite_name', metavar="PACKAGE",
+ default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'),
+ help=("Name of the testsuite in the xunit xml, generated by plugin. "
+ "Default test suite name is nosetests."))
+
+ def configure(self, options, config):
+ """Configures the xunit plugin."""
+ Plugin.configure(self, options, config)
+ self.config = config
+ if self.enabled:
+ self.stats = {'errors': 0,
+ 'failures': 0,
+ 'passes': 0,
+ 'skipped': 0
+ }
+ self.errorlist = []
+ self.error_report_file_name = os.path.realpath(options.dtestxunit_file)
+ self.xunit_testsuite_name = options.dtestxunit_testsuite_name
+
+ def report(self, stream):
+ """Writes an Xunit-formatted XML file
+
+ The file includes a report of test errors and failures.
+
+ """
+ self.error_report_file = codecs.open(self.error_report_file_name, 'w',
+ self.encoding, 'replace')
+ self.stats['encoding'] = self.encoding
+ self.stats['testsuite_name'] = self.xunit_testsuite_name
+ self.stats['total'] = (self.stats['errors'] + self.stats['failures']
+ + self.stats['passes'] + self.stats['skipped'])
+ self.error_report_file.write(
+ u'<?xml version="1.0" encoding="%(encoding)s"?>'
+ u'<testsuite name="%(testsuite_name)s" tests="%(total)d" '
+ u'errors="%(errors)d" failures="%(failures)d" '
+ u'skip="%(skipped)d">' % self.stats)
+ self.error_report_file.write(u''.join([force_unicode(e, self.encoding)
+ for e in self.errorlist]))
+ self.error_report_file.write(u'</testsuite>')
+ self.error_report_file.close()
+ if self.config.verbosity > 1:
+ stream.writeln("-" * 70)
+ stream.writeln("XML: %s" % self.error_report_file.name)
+
+ def _startCapture(self):
+ self._capture_stack.append((sys.stdout, sys.stderr))
+ self._currentStdout = StringIO()
+ self._currentStderr = StringIO()
+ sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout)
+ sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr)
+
+ def startContext(self, context):
+ self._startCapture()
+
+ def stopContext(self, context):
+ self._endCapture()
+
+ def beforeTest(self, test):
+ """Initializes a timer before starting a test."""
+ self._timer = time()
+ self._startCapture()
+
+ def _endCapture(self):
+ if self._capture_stack:
+ sys.stdout, sys.stderr = self._capture_stack.pop()
+
+ def afterTest(self, test):
+ self._endCapture()
+ self._currentStdout = None
+ self._currentStderr = None
+
+ def finalize(self, test):
+ while self._capture_stack:
+ self._endCapture()
+
+ def _getCapturedStdout(self):
+ if self._currentStdout:
+ value = self._currentStdout.getvalue()
+ if value:
+ return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata(
+ value)
+ return ''
+
+ def _getCapturedStderr(self):
+ if self._currentStderr:
+ value = self._currentStderr.getvalue()
+ if value:
+ return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata(
+ value)
+ return ''
+
+ def addError(self, test, err, capt=None):
+ """Add error output to Xunit report.
+ """
+ taken = self._timeTaken()
+
+ if issubclass(err[0], SkipTest):
+ type = 'skipped'
+ self.stats['skipped'] += 1
+ else:
+ type = 'error'
+ self.stats['errors'] += 1
+
+ tb = format_exception(err, self.encoding)
+ id = test.id()
+
+ self.errorlist.append(
+ u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+ u'<%(type)s type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+ u'</%(type)s>%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'type': type,
+ 'errtype': self._quoteattr(nice_classname(err[0])),
+ 'message': self._quoteattr(exc_message(err)),
+ 'tb': escape_cdata(tb),
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
+
+ def addFailure(self, test, err, capt=None, tb_info=None):
+ """Add failure output to Xunit report.
+ """
+ taken = self._timeTaken()
+ tb = format_exception(err, self.encoding)
+ self.stats['failures'] += 1
+ id = test.id()
+
+ self.errorlist.append(
+ u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+ u'<failure type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+ u'</failure>%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'errtype': self._quoteattr(nice_classname(err[0])),
+ 'message': self._quoteattr(exc_message(err)),
+ 'tb': escape_cdata(tb),
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
+
+ def addSuccess(self, test, capt=None):
+ """Add success output to Xunit report.
+ """
+ taken = self._timeTaken()
+ self.stats['passes'] += 1
+ id = test.id()
+ self.errorlist.append(
+ '<testcase classname=%(cls)s name=%(name)s '
+ 'time="%(taken).3f">%(systemout)s%(systemerr)s</testcase>' %
+ {'cls': self._quoteattr(id_split(id)[0]),
+ 'name': self._quoteattr(id_split(id)[-1]),
+ 'taken': taken,
+ 'systemout': self._getCapturedStdout(),
+ 'systemerr': self._getCapturedStderr(),
+ })
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ac0ce604/run_dtests.py
----------------------------------------------------------------------
diff --git a/run_dtests.py b/run_dtests.py
index cb68827..4f2d187 100755
--- a/run_dtests.py
+++ b/run_dtests.py
@@ -23,9 +23,11 @@ example:
from __future__ import print_function
import subprocess
+import sys
+import os
from collections import namedtuple
from itertools import product
-from os import getcwd
+from os import getcwd, environ
from tempfile import NamedTemporaryFile
from docopt import docopt
@@ -212,9 +214,15 @@ if __name__ == '__main__':
# How do we execute code in a new interpreter each time? Generate the
# code as text, then shell out to a new interpreter.
to_execute = (
- 'import nose\n'
- 'from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n'
- 'nose.main(addplugins=[DtestConfigPlugin({config})])\n'
+ "import nose\n" +
+ "from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n" +
+ "from plugins.dtestxunit import DTestXunit\n" +
+ "from plugins.dtesttag import DTestTag\n" +
+ "from plugins.dtestcollect import DTestCollect\n" +
+ "import sys\n" +
+ "print sys.getrecursionlimit()\n" +
+ "print sys.setrecursionlimit(8000)\n" +
+ ("nose.main(addplugins=[DtestConfigPlugin({config}), DTestXunit(), DTestCollect(), DTestTag()])\n" if "TEST_TAG" in environ else "nose.main(addplugins=[DtestConfigPlugin({config}), DTestCollect(), DTestXunit()])\n")
).format(config=repr(config))
temp = NamedTemporaryFile(dir=getcwd())
debug('Writing the following to {}:'.format(temp.name))
@@ -228,7 +236,7 @@ if __name__ == '__main__':
# command line are treated one way, args passed in as
# nose.main(argv=...) are treated another. Compare with the options
# -xsv for an example.
- cmd_list = ['python', temp.name] + nose_argv
+ cmd_list = [sys.executable, temp.name] + nose_argv
debug('subprocess.call-ing {cmd_list}'.format(cmd_list=cmd_list))
if options['--dry-run']:
@@ -240,7 +248,7 @@ if __name__ == '__main__':
contents=contents
))
else:
- results.append(subprocess.call(cmd_list))
+ results.append(subprocess.call(cmd_list, env=os.environ.copy()))
# separate the end of the last subprocess.call output from the
# beginning of the next by printing a newline.
print()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org