You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ac...@apache.org on 2017/04/25 22:12:51 UTC
[2/5] qpid-dispatch git commit: DISPATCH-739: Define test runner with
`cmake -DTEST_RUNNER`
DISPATCH-739: Define test runner with `cmake -DTEST_RUNNER`
Allow test runners other than valgrind (e.g. the amazing "rr" recording debugger)
Also makes it easier to edit valgrind options in CMakeCache.txt
Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/9c7b5e93
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/9c7b5e93
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/9c7b5e93
Branch: refs/heads/master
Commit: 9c7b5e936c6b302ba5a1c8f97d2500ea6f57e6ea
Parents: 92020aa
Author: Alan Conway <ac...@redhat.com>
Authored: Mon Apr 24 16:01:32 2017 -0400
Committer: Alan Conway <ac...@redhat.com>
Committed: Tue Apr 25 17:52:31 2017 -0400
----------------------------------------------------------------------
CMakeLists.txt | 6 +++
README | 23 ++++-------
run.py.in | 57 ++++++---------------------
tests/CMakeLists.txt | 18 ++++-----
tests/system_test.py | 86 +++++++++++++++++------------------------
tests/system_tests_http.py | 2 -
6 files changed, 68 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18b3be2..46f651e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -114,6 +114,12 @@ mark_as_advanced(VALGRIND_EXECUTABLE)
find_package_handle_standard_args(VALGRIND DEFAULT_MSG VALGRIND_EXECUTABLE)
option(USE_VALGRIND "Use valgrind when running tests" OFF)
+if (USE_VALGRIND)
+ set(VG_RUNNER "${VALGRIND_EXECUTABLE} --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --error-exitcode=42 --suppressions=${CMAKE_SOURCE_DIR}/tests/valgrind.supp --quiet")
+endif()
+
+set(TEST_RUNNER "${VG_RUNNER}" CACHE STRING "Program to run test executables")
+
##
## Include directories used by all sub-directories.
##
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/README
----------------------------------------------------------------------
diff --git a/README b/README
index 409fc4f..641ee64 100644
--- a/README
+++ b/README
@@ -65,19 +65,10 @@ This does the following:
- Run system tests on the installation in 'install'.
-Valgrind support
-================
-
-If valgrind is installed it will be used by default when running the tests.
-Set the cmake option 'USE_VALGRIND' to 'ON' or 'OFF' to enable/disable valgrind.
-You can also set the environment variable 'USE_VALGRIND' to 'ON or 'OFF'.
-If set the environment variable takes precendence over the cmake option.
-
-By default valgrind uses the memcheck tool to find memory-related
-issues. The tool can be overrided using the 'VALGRIND_TOOL'
-environment variable. The location of the valgrind suppression file
-can be overrided by setting the 'VALGRIND_SUPPRESSIONS' environment
-variable to an alternate suppression file. Additional valgrind
-options can be passed by setting the 'VALGRIND_OPTS' to a space
-separated string containing the additional options.
-E.g. VALGRIND_OPTS="--xml=yes --max-stackframe=10000000"
+Using Valgrind
+==============
+
+If valgrind is installed and cmake option 'USE_VALGRIND' is 'ON' the tests will
+be run with valgrind's memcheck debugger. You can set other types of test runner
+or modify the valgrind flags by setting the TEST_RUNNER cmake variable.
+
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/run.py.in
----------------------------------------------------------------------
diff --git a/run.py.in b/run.py.in
index cfafde8..292e57c 100755
--- a/run.py.in
+++ b/run.py.in
@@ -31,16 +31,8 @@ Usage:
run.py <program> [<arg>....] # Run a program, can be an interactive shell. Use PATH.
run.py -m <python-module> [<arg>....] # Run a python module. Use PYTHONPATH.
run.py -s <python-script>.py [<arg>....] # Run a python script.
-run.py --vg <program> [<arg>....] # Run a program with valgrind if enabled. Use PATH.
+run.py -x <program> [<arg>....] # Run an executable with the test runner
run.py --sh # Print a shell script to set the run.py environment.
-
-Valgrind can be enabled or disabled by the cmake 'USE_VALGRIND' option and the
-environment variable 'USE_VALGRIND' (set to 'ON' or 'OFF'). The environment
-variable takes precendence if set. By default valgrind runs the 'memcheck'
-tool. The valgrind tool and supression file can be overridden by setting the
-'VALGRIND_TOOL' and 'VALGRIND_SUPPRESSIONS' environment variables,
-respectively. Additional options can be provided via the 'VALGRIND_OPTS'
-environment variable.
"""
import os, sys, runpy
@@ -63,6 +55,9 @@ def getpath(env):
return path.split(os.pathsep)
return []
+# Test runner program and args (e.g. valgrind)
+test_runner = "${TEST_RUNNER}"
+
env_vars = {
'PYTHONPATH': os.pathsep.join(sys.path),
'PATH': os.pathsep.join(dedup(["${CMAKE_BINARY_DIR}",
@@ -77,21 +72,11 @@ env_vars = {
'SOURCE_DIR': "${CMAKE_SOURCE_DIR}",
'BUILD_DIR': "${CMAKE_BINARY_DIR}",
'QPID_DISPATCH_HOME': "${CMAKE_SOURCE_DIR}",
- 'QPID_DISPATCH_LIB': "${CMAKE_BINARY_DIR}/src/${QPID_DISPATCH_LIB}"
+ 'QPID_DISPATCH_LIB': "${CMAKE_BINARY_DIR}/src/${QPID_DISPATCH_LIB}",
+ 'QPID_DISPATCH_TEST_RUNNER': test_runner
}
os.environ.update(env_vars)
-# Valgrind setup
-valgrind_exe = "${VALGRIND_EXECUTABLE}"
-
-def use_valgrind():
- """True if we should use valgrind"""
- if not os.path.exists(valgrind_exe): return False
- def on(str):
- return str.lower() in ['on', 'yes', '1']
- env = os.environ.get('USE_VALGRIND')
- if env: return on(env)
- return on("${USE_VALGRIND}")
def find_exe(program):
"""Find an executable in the system PATH"""
@@ -112,26 +97,6 @@ def is_binary_exe(program):
p = Popen(['file', '-bi', program], stdout=PIPE, stderr=PIPE)
return p.communicate()[0].startswith('application/x-executable')
-VALGRIND_ERROR = 42 # magic number indicating valgrind found errors
-def with_valgrind(args, outfile=None):
- if use_valgrind() and is_binary_exe(find_exe(args[0])):
- opts = ['--leak-check=full',
- # Python generates a ton of "possibly lost" and "still in use"
- # false alarms, restrict to "definite" leaks.
- # Ideally we should have more specific python exclusions.
- '--show-leak-kinds=definite',
- '--errors-for-leak-kinds=definite',
- '--error-exitcode=%d' % VALGRIND_ERROR,
- '--quiet']
- opts.append('--tool=%s' % os.environ.get('VALGRIND_TOOL', 'memcheck'))
- supp = os.environ.get('VALGRIND_SUPPRESSIONS',
- '${CMAKE_SOURCE_DIR}/tests/valgrind.supp')
- opts.append('--suppressions=%s' % supp)
- if outfile: opts.append('--log-file=%s' % outfile)
- opts.extend(os.environ.get('VALGRIND_OPTS', "").split())
- return ([valgrind_exe]+opts+args, VALGRIND_ERROR)
- return (args, 0)
-
def run_path(file_path, run_name=None):
"""Wrapper for run path that falls back to exec python for python < 2.7"""
if hasattr(runpy, 'run_path'):
@@ -153,17 +118,17 @@ if __name__ == "__main__":
error_prefix = "Run python script '%s':"%(sys.argv[0])
run_path(sys.argv[0], run_name="__main__")
elif sys.argv[1] == '--sh':
- for name, value in env_vars.iteritems(): print "%s=%s"%(name, value)
+ for name, value in env_vars.iteritems(): print '%s="%s"'%(name, value)
print "export %s"%' '.join(env_vars.keys())
- elif sys.argv[1] == '--vg':
- args, ignore = with_valgrind(sys.argv[2:])
- error_prefix = "Run executable '%s' with valgrind: "%(args[0])
+ elif sys.argv[1] == '-x':
+ args = test_runner.split() + sys.argv[2:]
+ error_prefix = "Running '%s': "%(args)
os.execvp(args[0], args)
elif sys.argv[1].startswith('-'):
print usage
else:
args = sys.argv[1:]
- error_prefix = "Run executable '%s': "%(args[0])
+ error_prefix = "Running '%s': "%(args)
os.execvp(args[0], args)
except Exception, e:
print "%s%s: %s"%(error_prefix, type(e).__name__, e)
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d961e3a..314ad50 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -53,15 +53,15 @@ target_link_libraries(unit_tests_size qpid-dispatch)
set(TEST_WRAP ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/run.py)
-add_test(unit_tests_size_10000 ${TEST_WRAP} --vg unit_tests_size 10000)
-add_test(unit_tests_size_512 ${TEST_WRAP} --vg unit_tests_size 512)
-add_test(unit_tests_size_10 ${TEST_WRAP} --vg unit_tests_size 10)
-add_test(unit_tests_size_7 ${TEST_WRAP} --vg unit_tests_size 7)
-add_test(unit_tests_size_5 ${TEST_WRAP} --vg unit_tests_size 5)
-add_test(unit_tests_size_3 ${TEST_WRAP} --vg unit_tests_size 3)
-add_test(unit_tests_size_2 ${TEST_WRAP} --vg unit_tests_size 2)
-add_test(unit_tests_size_1 ${TEST_WRAP} --vg unit_tests_size 1)
-add_test(unit_tests ${TEST_WRAP} --vg unit_tests ${CMAKE_CURRENT_SOURCE_DIR}/threads4.conf)
+add_test(unit_tests_size_10000 ${TEST_WRAP} -x unit_tests_size 10000)
+add_test(unit_tests_size_512 ${TEST_WRAP} -x unit_tests_size 512)
+add_test(unit_tests_size_10 ${TEST_WRAP} -x unit_tests_size 10)
+add_test(unit_tests_size_7 ${TEST_WRAP} -x unit_tests_size 7)
+add_test(unit_tests_size_5 ${TEST_WRAP} -x unit_tests_size 5)
+add_test(unit_tests_size_3 ${TEST_WRAP} -x unit_tests_size 3)
+add_test(unit_tests_size_2 ${TEST_WRAP} -x unit_tests_size 2)
+add_test(unit_tests_size_1 ${TEST_WRAP} -x unit_tests_size 1)
+add_test(unit_tests ${TEST_WRAP} -x unit_tests ${CMAKE_CURRENT_SOURCE_DIR}/threads4.conf)
# Unit test python modules
add_test(router_engine_test ${TEST_WRAP} -m unittest -v router_engine_test)
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/tests/system_test.py
----------------------------------------------------------------------
diff --git a/tests/system_test.py b/tests/system_test.py
index ad55812..3257432 100755
--- a/tests/system_test.py
+++ b/tests/system_test.py
@@ -34,13 +34,6 @@ import proton
from proton import Message
from qpid_dispatch.management.client import Node
-try:
- # NOTE: the tests can be run outside a build to test an installed dispatch.
- # In this case we won't have access to the run.py module so no valgrind.
- from run import with_valgrind
-except ImportError:
- def with_valgrind(args, outfile): return (args, 0)
-
# Optional modules
MISSING_MODULES = []
@@ -189,13 +182,13 @@ def message(**properties):
class Process(subprocess.Popen):
"""
Popen that can be torn down at the end of a TestCase and stores its output.
- Uses valgrind if enabled.
+ Use $TEST_RUNNER as a prefix to the executable if it is defined in the environment.
"""
# Expected states of a Process at teardown
- RUNNING = 1 # Still running
- EXIT_OK = 2 # Exit status 0
- EXIT_FAIL = 3 # Exit status not 0
+ RUNNING = -1 # Still running
+ EXIT_OK = 0 # Exit status 0
+ EXIT_FAIL = 1 # Exit status 1
unique_id = 0
@classmethod
@@ -209,7 +202,8 @@ class Process(subprocess.Popen):
@param expect: Raise error if process staus not as expected at end of test:
L{RUNNING} - expect still running.
L{EXIT_OK} - expect proces to have terminated with 0 exit status.
- L{EXIT_ERROR} - expect proces to have terminated with non-0 exit status.
+ L{EXIT_FAIL} - expect proces to have terminated with exit status 1.
+ integer - expected return code
@keyword stdout: Defaults to the file name+".out"
@keyword stderr: Defaults to be the same as stdout
"""
@@ -217,56 +211,45 @@ class Process(subprocess.Popen):
self.args, self.expect = args, expect
self.outdir = os.getcwd()
self.outfile = os.path.abspath(self.unique(self.name))
- self.out = open(self.outfile + '.out', 'w')
- with open(self.outfile + '.cmd', 'w') as f: f.write("%s\n" % ' '.join(args))
self.torndown = False
- kwargs.setdefault('stdout', self.out)
- kwargs.setdefault('stderr', subprocess.STDOUT)
- args, self.valgrind_error = with_valgrind(args, self.outfile + '.vg')
- try:
- super(Process, self).__init__(args, **kwargs)
- except Exception, e:
- raise Exception("subprocess.Popen(%s, %s) failed: %s: %s" %
- (args, kwargs, type(e).__name__, e))
+ args = os.environ.get('QPID_DISPATCH_TEST_RUNNER', '').split() + args
+ with open(self.outfile + '.out', 'w') as out:
+ kwargs.setdefault('stdout', out)
+ kwargs.setdefault('stderr', subprocess.STDOUT)
+ try:
+ super(Process, self).__init__(args, **kwargs)
+ with open(self.outfile + '.cmd', 'w') as f:
+ f.write("%s\npid=%s\n" % (' '.join(args), self.pid))
+ except Exception, e:
+ raise Exception("subprocess.Popen(%s, %s) failed: %s: %s" %
+ (args, kwargs, type(e).__name__, e))
def assert_running(self):
"""Assert that the proces is still running"""
- assert self.poll() is None, "%s exited" % self.name
+ assert self.poll() is None, "%s: exited" % ' '.join(self.args)
def teardown(self):
"""Check process status and stop the process if necessary"""
if self.torndown:
return
self.torndown = True
+
+ def error(msg):
+ with open(self.outfile + '.out') as f:
+ raise RuntimeError("Process %s error: %s\n%s\n%s\n>>>>\n%s<<<<" % (
+ self.pid, msg, ' '.join(self.args),
+ self.outfile + '.cmd', f.read()));
+
status = self.poll()
- if status is None: # still running
+ if status is None: # Still running
self.terminate()
- rc = self.wait()
- if rc is None:
- self.kill()
- if self.valgrind_error and rc == self.valgrind_error:
- # Report that valgrind found errors
- status = rc;
- self.out.close()
- self.check_exit(status)
-
- def check_exit(self, status):
- """Check process exit status"""
- def check(condition, expect):
- """assert condition with a suitable message for status"""
- if status is None:
- actual = "still running"
- else:
- actual = "exit %s"%status
- assert condition, "Expected %s but %s: %s"%(expect, actual, self.name)
- assert not self.valgrind_error or status != self.valgrind_error, \
- "Valgrind errors (in %s)\n\n%s\n" % (self.outfile+".vg", open(self.outfile+".vg").read())
- if self.expect == Process.RUNNING:
- check(status is None, "still running")
- elif self.expect == Process.EXIT_OK:
- check(status == 0, "exit 0")
- elif self.expect == Process.EXIT_FAIL:
- check(status != 0, "exit non-0")
+ if self.expect != None and self.expect != Process.RUNNING:
+ error("still running")
+ self.expect = 0 # Expect clean exit after terminate
+ status = self.wait()
+ if self.expect != None and self.expect != status:
+ error("exit code %s, expected %s" % (status, self.expect))
+
class Config(object):
"""Base class for configuration objects that provide a convenient
@@ -556,7 +539,8 @@ class Tester(object):
break
except Exception, e:
errors.append(e)
- assert not errors, "Errors during teardown: \n%s" % "\n----".join([str(e) for e in errors])
+ if errors:
+ raise RuntimeError("Errors during teardown: \n\n%s" % "\n\n".join([str(e) for e in errors]))
def cleanup(self, x):
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9c7b5e93/tests/system_tests_http.py
----------------------------------------------------------------------
diff --git a/tests/system_tests_http.py b/tests/system_tests_http.py
index 2dfc93d..5627277 100644
--- a/tests/system_tests_http.py
+++ b/tests/system_tests_http.py
@@ -83,8 +83,6 @@ class RouterTestHttp(TestCase):
self.assertRaises(urllib2.URLError, urllib2.urlopen, "https://localhost:%d/nosuch" % r.ports[0])
def test_https_get(self):
- if run.use_valgrind(): self.skipTest("too slow for valgrind")
-
def listener(**kwargs):
args = dict(kwargs)
args.update({'port': self.get_port(), 'httpRoot': os.path.dirname(__file__)})
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org