You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/07/11 23:59:08 UTC
[48/50] [abbrv] git commit: [#4122] change ./run_tests to python,
run suites in parallel
[#4122] change ./run_tests to python, run suites in parallel
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/b57ec8d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/b57ec8d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/b57ec8d6
Branch: refs/heads/db/6277
Commit: b57ec8d61360ace076c3ff0bcbe9381d50541aa0
Parents: ff72589
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Mon May 6 18:30:30 2013 -0400
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Jul 11 21:30:46 2013 +0000
----------------------------------------------------------------------
run_tests | 138 ++++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 102 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b57ec8d6/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index 59e1825..5a958d9 100755
--- a/run_tests
+++ b/run_tests
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,38 +17,104 @@
# specific language governing permissions and limitations
# under the License.
-if [ -n "$SF_SYSTEM_FUNC" ]; then
- if [ -z "$VIRTUAL_ENV" ]; then
- source /var/local/env-allura/bin/activate
- fi
-fi
-
-# main
-
-if [ "$TEST_MODULES" == "" ]; then
- TEST_MODULES="\
- AlluraTest \
- Allura \
- ForgeBlog \
- ForgeLink \
- ForgeChat \
- ForgeDiscussion \
- ForgeGit \
- ForgeSVN \
- ForgeTracker \
- ForgeWiki \
- ForgeActivity \
- ForgeShortUrl \
- ForgeUserStats \
- "
-fi
-
-# Run with --with-coverage for coverage (add --cover-html for html report)
-for module in $TEST_MODULES; do
- (
- echo "Running tests in module $module"
- cd $module
- cover_package=$(echo $module | tr "[:upper:]" "[:lower:]")
- nosetests --cover-package=$cover_package --cover-html-dir=report.coverage --cover-erase $* || exit
- ) || exit
-done
+import argparse
+from glob import glob
+from multiprocessing.pool import ThreadPool
+import subprocess
+import sys
+import threading
+import textwrap
+
+
+def run_one(cmd, **popen_kwargs):
+ print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
+ all_popen_kwargs = dict(shell=True, stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE,
+ bufsize=1, # 1 == line-buffered
+ close_fds='posix' in sys.builtin_module_names)
+ all_popen_kwargs.update(popen_kwargs)
+ proc = subprocess.Popen(cmd, **all_popen_kwargs)
+ while proc.poll() is None:
+ line = proc.stdout.readline()
+ sys.stdout.write(line)
+ sys.stdout.flush()
+ # wait for completion and get remainder of output
+ out_remainder, _ = proc.communicate()
+ sys.stdout.write(out_remainder)
+ sys.stdout.flush()
+ return proc
+
+
+def run_many(cmds, processes=None):
+ """
+ cmds: list of shell commands, or list of (shell cmds, popen_kwargs)
+ processes: number of processes, or None for # of CPU cores
+ """
+ thread_pool = ThreadPool(processes=processes)
+
+ async_results = []
+ for cmd_kwds in cmds:
+ if type(cmd_kwds) == ():
+ cmd = cmd_kwds
+ kwds = {}
+ else:
+ cmd = cmd_kwds[0]
+ kwds = cmd_kwds[1]
+ result = thread_pool.apply_async(run_one, args=(cmd,), kwds=kwds)
+ async_results.append(result)
+
+ thread_pool.close()
+ thread_pool.join()
+
+ procs = [async_result.get() for async_result in async_results]
+ return [p.returncode for p in procs]
+
+
+def get_packages():
+ packages = [p.split('/')[0] for p in glob("*/setup.py")]
+
+ # make it first, to catch syntax errors
+ packages.remove('AlluraTest')
+ packages.insert(0, 'AlluraTest')
+ return packages
+
+
+def check_packages(packages):
+ for pkg in packages:
+ try:
+ __import__(pkg.lower())
+ except ImportError:
+ print "Not running tests for {}, since it isn't set up".format(pkg)
+ else:
+ yield pkg
+
+
+def run_tests_in_parallel(options, nosetests_args):
+ cmds = []
+ for package in check_packages(options.packages):
+ cover_package = package.lower()
+ cmd = "nosetests {nosetests_args} --cover-package={cover_package}".format(
+ cover_package=cover_package,
+ nosetests_args=' '.join(nosetests_args),
+ )
+ cmds.append((cmd, dict(cwd=package)))
+ return run_many(cmds, processes=options.num_processes)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=textwrap.dedent('''
+ All additional arguments are passed along to nosetests
+ (e.g. -v --with-coverage)
+ Note: --cover-package will be set automatically to the appropriate value'''))
+ parser.add_argument('-n', help='Number of processes to use at once. Default: # CPUs',
+ dest='num_processes', type=int, default=None)
+ parser.add_argument('-p', help='List of packages to run tests on. Default: all',
+ dest='packages', choices=get_packages(), default=get_packages(),
+ nargs='+')
+ return parser.parse_known_args()
+
+
+if __name__ == "__main__":
+ ret_codes = run_tests_in_parallel(*parse_args())
+ sys.exit(any(ret_codes))