You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2014/01/09 21:36:51 UTC
[04/18] git commit: [#6994] Run individual tests in parallel
[#6994] Run individual tests in parallel
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d4c06e8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d4c06e8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d4c06e8c
Branch: refs/heads/tv/6905
Commit: d4c06e8c5e4e205bd29907bd126b147e5ad6ddf5
Parents: 8b15d49
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Fri Dec 20 04:41:32 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Tue Jan 7 17:59:45 2014 +0000
----------------------------------------------------------------------
Allura/allura/tests/__init__.py | 4 ++++
Allura/allura/tests/test_utils.py | 1 +
requirements-common.txt | 2 +-
run_tests | 38 +++++++++++++++++++++++++++++-----
4 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/__init__.py b/Allura/allura/tests/__init__.py
index 3fb7e93..c715a48 100644
--- a/Allura/allura/tests/__init__.py
+++ b/Allura/allura/tests/__init__.py
@@ -21,6 +21,10 @@
import alluratest.controller
+# HACK: prevents test suite from crashing when running under the nose
+# MultiProcessing plugin
+import socket
+socket.setdefaulttimeout(None)
class TestController(alluratest.controller.TestController):
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/test_utils.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 494e893..b8b5de7 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -94,6 +94,7 @@ class TestAntispam(unittest.TestCase):
def setUp(self):
setup_unit_test()
+ pylons.request._push_object(Request.blank('/'))
pylons.request.remote_addr = '127.0.0.1'
self.a = utils.AntiSpam()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/requirements-common.txt
----------------------------------------------------------------------
diff --git a/requirements-common.txt b/requirements-common.txt
index 7f48285..5ffd19f 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -74,7 +74,7 @@ smmap==0.8.1
datadiff==1.1.5
ipython==0.11
mock==1.0.1
-nose==1.1.2
+nose==1.3.0
pyflakes==0.5.0
WebTest==1.4.0
clonedigger==1.1.0
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index a85aefe..38e63e4 100755
--- a/run_tests
+++ b/run_tests
@@ -20,12 +20,26 @@
import argparse
from copy import copy
from glob import glob
+import multiprocessing
from multiprocessing.pool import ThreadPool
import subprocess
import sys
import threading
import textwrap
+CPUS = multiprocessing.cpu_count()
+CONCURRENT_SUITES = (CPUS // 4) or CPUS
+CONCURRENT_TESTS = (CPUS // CONCURRENT_SUITES) or 1
+PROC_TIMEOUT = 120
+
+ALT_PKG_PATHS = {
+ 'Allura': 'allura/tests/',
+ }
+
+NOT_MULTIPROC_SAFE = [
+ 'ForgeGit',
+ 'ForgeSVN',
+ ]
def run_one(cmd, **popen_kwargs):
print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
@@ -76,7 +90,7 @@ def run_many(cmds, processes=None):
def get_packages():
- packages = [p.split('/')[0] for p in glob("*/setup.py")]
+ packages = sorted([p.split('/')[0] for p in glob("*/setup.py")])
# make it first, to catch syntax errors
packages.remove('AlluraTest')
@@ -95,16 +109,26 @@ def check_packages(packages):
def run_tests_in_parallel(options, nosetests_args):
+ def get_pkg_path(pkg):
+ return ALT_PKG_PATHS.get(pkg, '')
+ def get_multiproc_args(pkg):
+ return ('--processes={procs_per_suite} --process-timeout={proc_timeout}'.format(
+ procs_per_suite=options.concurrent_tests,
+ proc_timeout=PROC_TIMEOUT)
+ if pkg not in NOT_MULTIPROC_SAFE else '')
+
cmds = []
for package in check_packages(options.packages):
cover_package = package.lower()
our_nosetests_args = copy(nosetests_args)
our_nosetests_args.append('--cover-package={}'.format(cover_package))
- cmd = "nosetests {nosetests_args}".format(
+ cmd = "nosetests {pkg_path} {nosetests_args} {multiproc_args}".format(
+ pkg_path=get_pkg_path(package),
nosetests_args=' '.join(our_nosetests_args),
+ multiproc_args=get_multiproc_args(package),
)
cmds.append((cmd, dict(cwd=package)))
- return run_many(cmds, processes=options.num_processes)
+ return run_many(cmds, processes=options.concurrent_suites)
def parse_args():
@@ -113,8 +137,12 @@ def parse_args():
All additional arguments are passed along to nosetests
(e.g. -v --with-coverage)
Note: --cover-package will be set automatically to the appropriate value'''))
- parser.add_argument('-n', help='Number of processes to use at once. Default: # CPUs',
- dest='num_processes', type=int, default=None)
+ parser.add_argument('-n', help='Number of test suites to run concurrently in separate '
+ 'processes. Default: # CPUs / 4',
+ dest='concurrent_suites', type=int, default=CONCURRENT_SUITES)
+ parser.add_argument('-m', help='Number of tests to run concurrently in separate '
+ 'processes, per suite. Default: # CPUs / # concurrent suites',
+ dest='concurrent_tests', type=int, default=CONCURRENT_TESTS)
parser.add_argument('-p', help='List of packages to run tests on. Default: all',
dest='packages', choices=get_packages(), default=get_packages(),
nargs='+')