You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2014/01/07 19:02:50 UTC
[1/3] git commit: [#4671] remove old-style LastCommitDoc model and
logic
Updated Branches:
refs/heads/master 8c9de147d -> dd3b5a824
[#4671] remove old-style LastCommitDoc model and logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/8b15d499
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/8b15d499
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/8b15d499
Branch: refs/heads/master
Commit: 8b15d4991d306c2ccee6cba625406a69ecc6dbf5
Parents: 8c9de14
Author: Dave Brondsema <da...@brondsema.net>
Authored: Thu Jan 2 17:51:11 2014 -0500
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Tue Jan 7 17:01:40 2014 +0000
----------------------------------------------------------------------
Allura/allura/model/repo.py | 93 ++--------------------------------------
1 file changed, 4 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8b15d499/Allura/allura/model/repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repo.py b/Allura/allura/model/repo.py
index 9977a0f..9228076 100644
--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -83,20 +83,6 @@ TreeDoc = collection(
Field('blob_ids', [dict(name=str, id=str)]),
Field('other_ids', [dict(name=str, id=str, type=SObjType)]))
-LastCommitDoc_old = collection(
- 'repo_last_commit', project_doc_session,
- Field('_id', str),
- Field('object_id', str, index=True),
- Field('name', str),
- Field('commit_info', dict(
- id=str,
- date=datetime,
- author=str,
- author_email=str,
- author_url=str,
- shortlink=str,
- summary=str)))
-
# Information about the last commit to touch a tree
LastCommitDoc = collection(
'repo_last_commit', main_doc_session,
@@ -555,22 +541,9 @@ class Tree(RepoObject):
def ls(self):
'''
List the entries in this tree, with historical commit info for
- each node. Eventually, ls_old can be removed and this can be
- replaced with the following:
-
- return self._lcd_map(LastCommit.get(self))
+ each node.
'''
- # look for existing new format first
- last_commit = LastCommit.get(self, create=False)
- if last_commit:
- return self._lcd_map(last_commit)
- # otherwise, try old format
- old_style_results = self.ls_old()
- if old_style_results:
- log.info('Using old-style results from ls_old()')
- return old_style_results
- # finally, use the new implentation that auto-vivifies
- last_commit = LastCommit.get(self, create=True)
+ last_commit = LastCommit.get(self)
# ensure that the LCD is saved, even if
# there is an error later in the request
if last_commit:
@@ -615,64 +588,6 @@ class Tree(RepoObject):
))
return results
- def ls_old(self):
- # Load last commit info
- id_re = re.compile("^{0}:{1}:".format(
- self.repo._id,
- re.escape(h.really_unicode(self.path()).encode('utf-8'))))
- lc_index = dict(
- (lc.name, lc.commit_info)
- for lc in LastCommitDoc_old.m.find(dict(_id=id_re)))
-
- # FIXME: Temporarily fall back to old, semi-broken lookup behavior until refresh is done
- oids = [ x.id for x in chain(self.tree_ids, self.blob_ids, self.other_ids) ]
- id_re = re.compile("^{0}:".format(self.repo._id))
- lc_index.update(dict(
- (lc.object_id, lc.commit_info)
- for lc in LastCommitDoc_old.m.find(dict(_id=id_re, object_id={'$in': oids}))))
- # /FIXME
-
- if not lc_index:
- # allow fallback to new method instead
- # of showing a bunch of Nones
- return []
-
- results = []
- def _get_last_commit(name, oid):
- lc = lc_index.get(name, lc_index.get(oid, None))
- if lc is None:
- lc = dict(
- author=None,
- author_email=None,
- author_url=None,
- date=None,
- id=None,
- href=None,
- shortlink=None,
- summary=None)
- if 'href' not in lc:
- lc['href'] = self.repo.url_for_commit(lc['id'])
- return lc
- for x in sorted(self.tree_ids, key=lambda x:x.name):
- results.append(dict(
- kind='DIR',
- name=x.name,
- href=x.name + '/',
- last_commit=_get_last_commit(x.name, x.id)))
- for x in sorted(self.blob_ids, key=lambda x:x.name):
- results.append(dict(
- kind='FILE',
- name=x.name,
- href=x.name,
- last_commit=_get_last_commit(x.name, x.id)))
- for x in sorted(self.other_ids, key=lambda x:x.name):
- results.append(dict(
- kind=x.type,
- name=x.name,
- href=None,
- last_commit=_get_last_commit(x.name, x.id)))
- return results
-
def path(self):
if self.parent:
assert self.parent is not self
@@ -840,13 +755,13 @@ class LastCommit(RepoObject):
return None
@classmethod
- def get(cls, tree, create=True):
+ def get(cls, tree):
'''Find or build the LastCommitDoc for the given tree.'''
cache = getattr(c, 'model_cache', '') or ModelCache()
path = tree.path().strip('/')
last_commit_id = cls._last_commit_id(tree.commit, path)
lcd = cache.get(cls, {'path': path, 'commit_id': last_commit_id})
- if lcd is None and create:
+ if lcd is None:
commit = cache.get(Commit, {'_id': last_commit_id})
commit.set_context(tree.repo)
lcd = cls._build(commit.get_path(path))
[2/3] git commit: [#6994] Run individual tests in parallel
Posted by jo...@apache.org.
[#6994] Run individual tests in parallel
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d4c06e8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d4c06e8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d4c06e8c
Branch: refs/heads/master
Commit: d4c06e8c5e4e205bd29907bd126b147e5ad6ddf5
Parents: 8b15d49
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Fri Dec 20 04:41:32 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Tue Jan 7 17:59:45 2014 +0000
----------------------------------------------------------------------
Allura/allura/tests/__init__.py | 4 ++++
Allura/allura/tests/test_utils.py | 1 +
requirements-common.txt | 2 +-
run_tests | 38 +++++++++++++++++++++++++++++-----
4 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/__init__.py b/Allura/allura/tests/__init__.py
index 3fb7e93..c715a48 100644
--- a/Allura/allura/tests/__init__.py
+++ b/Allura/allura/tests/__init__.py
@@ -21,6 +21,10 @@
import alluratest.controller
+# HACK: prevents test suite from crashing when running under the nose
+# MultiProcessing plugin
+import socket
+socket.setdefaulttimeout(None)
class TestController(alluratest.controller.TestController):
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/Allura/allura/tests/test_utils.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_utils.py b/Allura/allura/tests/test_utils.py
index 494e893..b8b5de7 100644
--- a/Allura/allura/tests/test_utils.py
+++ b/Allura/allura/tests/test_utils.py
@@ -94,6 +94,7 @@ class TestAntispam(unittest.TestCase):
def setUp(self):
setup_unit_test()
+ pylons.request._push_object(Request.blank('/'))
pylons.request.remote_addr = '127.0.0.1'
self.a = utils.AntiSpam()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/requirements-common.txt
----------------------------------------------------------------------
diff --git a/requirements-common.txt b/requirements-common.txt
index 7f48285..5ffd19f 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -74,7 +74,7 @@ smmap==0.8.1
datadiff==1.1.5
ipython==0.11
mock==1.0.1
-nose==1.1.2
+nose==1.3.0
pyflakes==0.5.0
WebTest==1.4.0
clonedigger==1.1.0
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d4c06e8c/run_tests
----------------------------------------------------------------------
diff --git a/run_tests b/run_tests
index a85aefe..38e63e4 100755
--- a/run_tests
+++ b/run_tests
@@ -20,12 +20,26 @@
import argparse
from copy import copy
from glob import glob
+import multiprocessing
from multiprocessing.pool import ThreadPool
import subprocess
import sys
import threading
import textwrap
+CPUS = multiprocessing.cpu_count()
+CONCURRENT_SUITES = (CPUS // 4) or CPUS
+CONCURRENT_TESTS = (CPUS // CONCURRENT_SUITES) or 1
+PROC_TIMEOUT = 120
+
+ALT_PKG_PATHS = {
+ 'Allura': 'allura/tests/',
+ }
+
+NOT_MULTIPROC_SAFE = [
+ 'ForgeGit',
+ 'ForgeSVN',
+ ]
def run_one(cmd, **popen_kwargs):
print '{} running {} {}'.format(threading.current_thread(), cmd, popen_kwargs)
@@ -76,7 +90,7 @@ def run_many(cmds, processes=None):
def get_packages():
- packages = [p.split('/')[0] for p in glob("*/setup.py")]
+ packages = sorted([p.split('/')[0] for p in glob("*/setup.py")])
# make it first, to catch syntax errors
packages.remove('AlluraTest')
@@ -95,16 +109,26 @@ def check_packages(packages):
def run_tests_in_parallel(options, nosetests_args):
+ def get_pkg_path(pkg):
+ return ALT_PKG_PATHS.get(pkg, '')
+ def get_multiproc_args(pkg):
+ return ('--processes={procs_per_suite} --process-timeout={proc_timeout}'.format(
+ procs_per_suite=options.concurrent_tests,
+ proc_timeout=PROC_TIMEOUT)
+ if pkg not in NOT_MULTIPROC_SAFE else '')
+
cmds = []
for package in check_packages(options.packages):
cover_package = package.lower()
our_nosetests_args = copy(nosetests_args)
our_nosetests_args.append('--cover-package={}'.format(cover_package))
- cmd = "nosetests {nosetests_args}".format(
+ cmd = "nosetests {pkg_path} {nosetests_args} {multiproc_args}".format(
+ pkg_path=get_pkg_path(package),
nosetests_args=' '.join(our_nosetests_args),
+ multiproc_args=get_multiproc_args(package),
)
cmds.append((cmd, dict(cwd=package)))
- return run_many(cmds, processes=options.num_processes)
+ return run_many(cmds, processes=options.concurrent_suites)
def parse_args():
@@ -113,8 +137,12 @@ def parse_args():
All additional arguments are passed along to nosetests
(e.g. -v --with-coverage)
Note: --cover-package will be set automatically to the appropriate value'''))
- parser.add_argument('-n', help='Number of processes to use at once. Default: # CPUs',
- dest='num_processes', type=int, default=None)
+ parser.add_argument('-n', help='Number of test suites to run concurrently in separate '
+ 'processes. Default: # CPUs / 4',
+ dest='concurrent_suites', type=int, default=CONCURRENT_SUITES)
+ parser.add_argument('-m', help='Number of tests to run concurrently in separate '
+ 'processes, per suite. Default: # CPUs / # concurrent suites',
+ dest='concurrent_tests', type=int, default=CONCURRENT_TESTS)
parser.add_argument('-p', help='List of packages to run tests on. Default: all',
dest='packages', choices=get_packages(), default=get_packages(),
nargs='+')
[3/3] git commit: Fixed intermittently failing test
Posted by jo...@apache.org.
Fixed intermittently failing test
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/dd3b5a82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/dd3b5a82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/dd3b5a82
Branch: refs/heads/master
Commit: dd3b5a824dee17784929b2526b0b2122df22ba25
Parents: d4c06e8
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Tue Jan 7 18:02:26 2014 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Tue Jan 7 18:02:26 2014 +0000
----------------------------------------------------------------------
Allura/allura/model/repository.py | 2 +-
Allura/allura/tests/model/test_repo.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dd3b5a82/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 7d0fae0..06ade3f 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -237,7 +237,7 @@ class RepositoryImplementation(object):
try:
commit_id = commit._id
while paths and commit_id:
- if time() - start_time > timeout:
+ if time() - start_time >= timeout:
log.error('last_commit_ids timeout for %s on %s', commit._id, ', '.join(paths))
break
commit_id, changes = self._get_last_commit(commit._id, paths)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dd3b5a82/Allura/allura/tests/model/test_repo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_repo.py b/Allura/allura/tests/model/test_repo.py
index edc76b2..a2a66ff 100644
--- a/Allura/allura/tests/model/test_repo.py
+++ b/Allura/allura/tests/model/test_repo.py
@@ -327,7 +327,7 @@ class TestLastCommit(unittest.TestCase):
commit1 = self._add_commit('Commit 1', ['file1'])
commit2 = self._add_commit('Commit 2', ['file1', 'dir1/file1'], ['dir1/file1'], [commit1])
commit3 = self._add_commit('Commit 3', ['file1', 'dir1/file1', 'file2'], ['file2'], [commit2])
- with h.push_config(config, lcd_timeout=0):
+ with h.push_config(config, lcd_timeout=-1000):
lcd = M.repo.LastCommit.get(commit3.tree)
self.assertEqual(self.repo._commits[lcd.commit_id].message, commit3.message)
self.assertEqual(lcd.commit_id, commit3._id)