You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/07/12 00:10:17 UTC
[1/2] git commit: [#6277] make forgeuserstats tmp repos not conflict
with forgegit
Updated Branches:
refs/heads/db/6277 7bdac52fe -> d2a60bbbf (forced update)
[#6277] make forgeuserstats tmp repos not conflict with forgegit
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d2a60bbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d2a60bbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d2a60bbb
Branch: refs/heads/db/6277
Commit: d2a60bbbf547798b74a42ee87d68b6a26480a373
Parents: 3ec3fed
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Tue May 28 10:32:26 2013 -0400
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Jul 11 22:10:00 2013 +0000
----------------------------------------------------------------------
ForgeUserStats/forgeuserstats/tests/test_model.py | 5 +++--
ForgeUserStats/forgeuserstats/tests/test_stats.py | 8 ++++++--
run_tests | 4 ++++
3 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d2a60bbb/ForgeUserStats/forgeuserstats/tests/test_model.py
----------------------------------------------------------------------
diff --git a/ForgeUserStats/forgeuserstats/tests/test_model.py b/ForgeUserStats/forgeuserstats/tests/test_model.py
index 2203469..6940385 100644
--- a/ForgeUserStats/forgeuserstats/tests/test_model.py
+++ b/ForgeUserStats/forgeuserstats/tests/test_model.py
@@ -29,10 +29,11 @@ from allura.model import User, Project, TroveCategory
from allura.lib import helpers as h
from allura import model as M
-from forgegit.tests import with_git
-
from forgeuserstats.model import stats as USM
+test_project_with_repo = 'test2' # important to be distinct from 'test' which ForgeGit uses, so that the tests can run in parallel and not clobber each other
+with_git = td.with_tool(test_project_with_repo, 'Git', 'src-git', 'Git', type='git')
+
class TestUserStats(unittest.TestCase):
def setUp(self):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d2a60bbb/ForgeUserStats/forgeuserstats/tests/test_stats.py
----------------------------------------------------------------------
diff --git a/ForgeUserStats/forgeuserstats/tests/test_stats.py b/ForgeUserStats/forgeuserstats/tests/test_stats.py
index b2b20ce..bbb43f5 100644
--- a/ForgeUserStats/forgeuserstats/tests/test_stats.py
+++ b/ForgeUserStats/forgeuserstats/tests/test_stats.py
@@ -27,10 +27,14 @@ from allura.lib import helpers as h
from allura.model import User
from allura import model as M
-from forgegit.tests import with_git
from forgewiki import model as WM
from forgetracker import model as TM
+
+test_project_with_repo = 'test2' # important to be distinct from 'test' which ForgeGit uses, so that the tests can run in parallel and not clobber each other
+with_git = td.with_tool(test_project_with_repo, 'Git', 'src-git', 'Git', type='git')
+
+
class TestStats(TestController):
def setUp(self):
@@ -185,7 +189,7 @@ class TestGitCommit(TestController, unittest.TestCase):
@td.with_wiki
def setup_with_tools(self):
setup_global_objects()
- h.set_context('test', 'src-git', neighborhood='Projects')
+ h.set_context(test_project_with_repo, 'src-git', neighborhood='Projects')
repo_dir = pkg_resources.resource_filename(
'forgeuserstats', 'tests/data')
c.app.repo.fs_path = repo_dir
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d2a60bbb/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index 7d033b1..a85aefe 100755
--- a/run_tests
+++ b/run_tests
@@ -29,6 +29,8 @@ import textwrap
def run_one(cmd, **popen_kwargs):
print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
+ sys.stdout.flush()
+
all_popen_kwargs = dict(shell=True, stderr=subprocess.STDOUT,
stdout=subprocess.PIPE,
bufsize=1, # 1 == line-buffered
@@ -43,6 +45,8 @@ def run_one(cmd, **popen_kwargs):
out_remainder, _ = proc.communicate()
sys.stdout.write(out_remainder)
sys.stdout.flush()
+ print 'finished {} {}'.format(cmd, popen_kwargs)
+ sys.stdout.flush()
return proc
[2/2] git commit: [#4122] change ./run_tests to python,
run suites in parallel
Posted by br...@apache.org.
[#4122] change ./run_tests to python, run suites in parallel
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3ec3fedb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3ec3fedb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3ec3fedb
Branch: refs/heads/db/6277
Commit: 3ec3fedb0daad79408694f90792272549f73e5e1
Parents: ff72589
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Mon May 6 18:30:30 2013 -0400
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Jul 11 22:10:00 2013 +0000
----------------------------------------------------------------------
run_tests | 140 ++++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 104 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3ec3fedb/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index 59e1825..7d033b1 100755
--- a/run_tests
+++ b/run_tests
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,38 +17,106 @@
# specific language governing permissions and limitations
# under the License.
-if [ -n "$SF_SYSTEM_FUNC" ]; then
- if [ -z "$VIRTUAL_ENV" ]; then
- source /var/local/env-allura/bin/activate
- fi
-fi
-
-# main
-
-if [ "$TEST_MODULES" == "" ]; then
- TEST_MODULES="\
- AlluraTest \
- Allura \
- ForgeBlog \
- ForgeLink \
- ForgeChat \
- ForgeDiscussion \
- ForgeGit \
- ForgeSVN \
- ForgeTracker \
- ForgeWiki \
- ForgeActivity \
- ForgeShortUrl \
- ForgeUserStats \
- "
-fi
-
-# Run with --with-coverage for coverage (add --cover-html for html report)
-for module in $TEST_MODULES; do
- (
- echo "Running tests in module $module"
- cd $module
- cover_package=$(echo $module | tr "[:upper:]" "[:lower:]")
- nosetests --cover-package=$cover_package --cover-html-dir=report.coverage --cover-erase $* || exit
- ) || exit
-done
+import argparse
+from copy import copy
+from glob import glob
+from multiprocessing.pool import ThreadPool
+import subprocess
+import sys
+import threading
+import textwrap
+
+
+def run_one(cmd, **popen_kwargs):
+ print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
+ all_popen_kwargs = dict(shell=True, stderr=subprocess.STDOUT,
+ stdout=subprocess.PIPE,
+ bufsize=1, # 1 == line-buffered
+ close_fds='posix' in sys.builtin_module_names)
+ all_popen_kwargs.update(popen_kwargs)
+ proc = subprocess.Popen(cmd, **all_popen_kwargs)
+ while proc.poll() is None:
+ line = proc.stdout.readline()
+ sys.stdout.write(line)
+ sys.stdout.flush()
+ # wait for completion and get remainder of output
+ out_remainder, _ = proc.communicate()
+ sys.stdout.write(out_remainder)
+ sys.stdout.flush()
+ return proc
+
+
+def run_many(cmds, processes=None):
+ """
+ cmds: list of shell commands, or list of (shell cmds, popen_kwargs)
+ processes: number of processes, or None for # of CPU cores
+ """
+ thread_pool = ThreadPool(processes=processes)
+
+ async_results = []
+ for cmd_kwds in cmds:
+ if type(cmd_kwds) == ():
+ cmd = cmd_kwds
+ kwds = {}
+ else:
+ cmd = cmd_kwds[0]
+ kwds = cmd_kwds[1]
+ result = thread_pool.apply_async(run_one, args=(cmd,), kwds=kwds)
+ async_results.append(result)
+
+ thread_pool.close()
+ thread_pool.join()
+
+ procs = [async_result.get() for async_result in async_results]
+ return [p.returncode for p in procs]
+
+
+def get_packages():
+ packages = [p.split('/')[0] for p in glob("*/setup.py")]
+
+ # make it first, to catch syntax errors
+ packages.remove('AlluraTest')
+ packages.insert(0, 'AlluraTest')
+ return packages
+
+
+def check_packages(packages):
+ for pkg in packages:
+ try:
+ __import__(pkg.lower())
+ except ImportError:
+ print "Not running tests for {}, since it isn't set up".format(pkg)
+ else:
+ yield pkg
+
+
+def run_tests_in_parallel(options, nosetests_args):
+ cmds = []
+ for package in check_packages(options.packages):
+ cover_package = package.lower()
+ our_nosetests_args = copy(nosetests_args)
+ our_nosetests_args.append('--cover-package={}'.format(cover_package))
+ cmd = "nosetests {nosetests_args}".format(
+ nosetests_args=' '.join(our_nosetests_args),
+ )
+ cmds.append((cmd, dict(cwd=package)))
+ return run_many(cmds, processes=options.num_processes)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=textwrap.dedent('''
+ All additional arguments are passed along to nosetests
+ (e.g. -v --with-coverage)
+ Note: --cover-package will be set automatically to the appropriate value'''))
+ parser.add_argument('-n', help='Number of processes to use at once. Default: # CPUs',
+ dest='num_processes', type=int, default=None)
+ parser.add_argument('-p', help='List of packages to run tests on. Default: all',
+ dest='packages', choices=get_packages(), default=get_packages(),
+ nargs='+')
+ return parser.parse_known_args()
+
+
+if __name__ == "__main__":
+ ret_codes = run_tests_in_parallel(*parse_args())
+ sys.exit(any(ret_codes))