You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2016/09/22 14:16:17 UTC
[1/3] allura git commit: Make these test classes runnable on their
own (they happen to pass if other stuff runs first)
Repository: allura
Updated Branches:
refs/heads/db/parallel_test_improvements [created] 8a70d6e45
Make these test classes runnable on their own (they happen to pass if other stuff runs first)
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/1969390a
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/1969390a
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/1969390a
Branch: refs/heads/db/parallel_test_improvements
Commit: 1969390a5a4d2d2428b6e50244d3409bf6d1278c
Parents: e305225
Author: Dave Brondsema <da...@brondsema.net>
Authored: Wed Sep 21 14:28:36 2016 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Wed Sep 21 14:28:36 2016 -0400
----------------------------------------------------------------------
.../forgeimporters/tests/forge/test_tracker.py | 13 ++++++++++---
.../forgeimporters/tests/github/test_tracker.py | 10 ++++++++++
2 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/1969390a/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/forge/test_tracker.py b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
index e985763..9be7743 100644
--- a/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
@@ -17,8 +17,6 @@
from datetime import datetime
from unittest import TestCase
-from cgi import FieldStorage
-from cStringIO import StringIO
import mock
from ming.odm import ThreadLocalORMSession
@@ -29,11 +27,20 @@ from allura.tests.decorators import with_tracker
from allura import model as M
from forgeimporters.forge import tracker
-from forgetracker import model as TM
class TestTrackerImporter(TestCase):
+ def setUp(self):
+ super(TestTrackerImporter, self).setUp()
+ # every single test method here creates an importer and ToolImporterMeta uses 'g'
+ self.patcher_g = mock.patch('forgeimporters.base.g', mock.MagicMock())
+ self.patcher_g.start()
+
+ def tearDown(self):
+ super(TestTrackerImporter, self).tearDown()
+ self.patcher_g.stop()
+
@mock.patch.object(tracker, 'File')
@mock.patch.object(tracker.h, 'make_app_admin_only')
@mock.patch.object(tracker, 'g')
http://git-wip-us.apache.org/repos/asf/allura/blob/1969390a/ForgeImporters/forgeimporters/tests/github/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_tracker.py b/ForgeImporters/forgeimporters/tests/github/test_tracker.py
index e02ac5a..fd7df5c 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_tracker.py
@@ -26,6 +26,16 @@ from forgeimporters.github.utils import GitHubMarkdownConverter
class TestTrackerImporter(TestCase):
+ def setUp(self):
+ super(TestTrackerImporter, self).setUp()
+ # every single test method here creates an importer and ToolImporterMeta uses 'g'
+ self.patcher_g = mock.patch('forgeimporters.base.g', mock.MagicMock())
+ self.patcher_g.start()
+
+ def tearDown(self):
+ super(TestTrackerImporter, self).tearDown()
+ self.patcher_g.stop()
+
@mock.patch.object(tracker, 'g')
@mock.patch.object(tracker, 'c')
@mock.patch.object(tracker, 'ThreadLocalORMSession')
[3/3] allura git commit: Make this test run successfully on its own
Posted by br...@apache.org.
Make this test run successfully on its own
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/8a70d6e4
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/8a70d6e4
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/8a70d6e4
Branch: refs/heads/db/parallel_test_improvements
Commit: 8a70d6e45b33d356666a95a01fe3c623a2d94557
Parents: 46dc9f9
Author: Dave Brondsema <da...@brondsema.net>
Authored: Thu Sep 22 10:16:04 2016 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Thu Sep 22 10:16:04 2016 -0400
----------------------------------------------------------------------
ForgeImporters/forgeimporters/tests/github/test_tasks.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/8a70d6e4/ForgeImporters/forgeimporters/tests/github/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_tasks.py b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
index cf58644..b048407 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_tasks.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
@@ -23,6 +23,7 @@ from ...github import tasks
@mock.patch.object(tasks, 'GitHubProjectExtractor')
@mock.patch.object(tasks, 'ThreadLocalORMSession')
@mock.patch.object(tasks, 'c')
+@mock.patch.object(tasks, 'g', mock.MagicMock())
def test_import_project_info(c, session, ghpe):
c.project = mock.Mock(name='project')
c.user = mock.Mock(name='user')
[2/3] allura git commit: Split up pylint test into chunks that can be
run with nose multiprocess; move pyflakes chunks into parallelized pattern
Posted by br...@apache.org.
Split up pylint test into chunks that can be run with nose multiprocess; move pyflakes chunks into parallelized pattern
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/46dc9f9b
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/46dc9f9b
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/46dc9f9b
Branch: refs/heads/db/parallel_test_improvements
Commit: 46dc9f9bc0a026e0b555add0dbd5e90cea4a7462
Parents: 1969390
Author: Dave Brondsema <da...@brondsema.net>
Authored: Wed Sep 21 14:32:35 2016 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Wed Sep 21 14:32:35 2016 -0400
----------------------------------------------------------------------
AlluraTest/alluratest/test_syntax.py | 89 +++++++++++++++++--------------
1 file changed, 50 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/46dc9f9b/AlluraTest/alluratest/test_syntax.py
----------------------------------------------------------------------
diff --git a/AlluraTest/alluratest/test_syntax.py b/AlluraTest/alluratest/test_syntax.py
index d29b00b..1db63c9 100644
--- a/AlluraTest/alluratest/test_syntax.py
+++ b/AlluraTest/alluratest/test_syntax.py
@@ -18,6 +18,7 @@
import os.path
from subprocess import Popen, PIPE
import sys
+from itertools import izip_longest
toplevel_dir = os.path.abspath(os.path.dirname(__file__) + "/../..")
@@ -30,52 +31,17 @@ def run(cmd):
sys.stderr.write(stderr)
return proc.returncode
-find_py = "find Allura Forge* -name '*.py'"
-# a recepe from itertools doc
-from itertools import izip_longest
+find_py = "find Allura Forge* -name '*.py'"
+# a recipe from itertools doc
def grouper(n, iterable, fillvalue=None):
"grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"
args = [iter(iterable)] * n
return izip_longest(fillvalue=fillvalue, *args)
-def test_pyflakes():
- # skip some that aren't critical errors
- skips = [
- 'imported but unused',
- 'redefinition of unused',
- 'assigned to but never used',
- '__version__',
- ]
- proc = Popen(find_py, shell=True, cwd=toplevel_dir,
- stdout=PIPE, stderr=PIPE)
- (find_stdout, stderr) = proc.communicate()
- sys.stderr.write(stderr)
- assert proc.returncode == 0, proc.returncode
-
- # run pyflakes in batches, so it doesn't take tons of memory
- error = False
- all_files = [f for f in find_stdout.split('\n')
- if '/migrations/' not in f and f.strip()]
- for files in grouper(20, all_files, fillvalue=''):
- cmd = "pyflakes " + \
- ' '.join(files) + " | grep -v '" + \
- "' | grep -v '".join(skips) + "'"
- # print 'Command was: %s' % cmd
- retval = run(cmd)
- if retval != 1:
- print
- # print 'Command was: %s' % cmd
- print 'Returned %s' % retval
- error = True
-
- if error:
- raise Exception('pyflakes failure, see stdout')
-
-
def test_no_local_tz_functions():
if run(find_py + " | xargs grep '\.now(' ") not in [1, 123]:
raise Exception("These should use .utcnow()")
@@ -104,6 +70,51 @@ def test_no_tabs():
if run(find_py + " | xargs grep ' ' ") not in [1, 123]:
raise Exception('These should not use tab chars')
-def test_linters():
- if run(find_py + ' | xargs pylint -E --disable=all --enable=exposed-api-needs-kwargs --load-plugins alluratest.pylint_checkers') != 0:
+
+def run_linter(files):
+ if run('pylint -E --disable=all --enable=exposed-api-needs-kwargs --load-plugins alluratest.pylint_checkers {}'.format(' '.join(files))) != 0:
raise Exception('Custom Allura pylint errors found.')
+
+
+def run_pyflakes(files):
+ # skip some that aren't critical errors
+ skips = [
+ 'imported but unused',
+ 'redefinition of unused',
+ 'assigned to but never used',
+ '__version__',
+ ]
+ files = [f for f in files if '/migrations/' not in f]
+ cmd = "pyflakes " + ' '.join(files) + " | grep -v '" + "' | grep -v '".join(skips) + "'"
+ if run(cmd) != 1:
+ # print 'Command was: %s' % cmd
+ raise Exception('pyflakes failure, see stdout')
+
+
+class TestLinters(object):
+ # this will get populated dynamically with test methods, see below
+ pass
+
+
+# Dynamically generate many test methods, to run pylint & pyflakes commands in separate batches
+# Can't use http://nose.readthedocs.io/en/latest/writing_tests.html#test-generators because nose doesn't run
+# those in parallel
+def create_many_lint_methods():
+ proc = Popen(find_py, shell=True, cwd=toplevel_dir, stdout=PIPE, stderr=PIPE)
+ (find_stdout, stderr) = proc.communicate()
+ sys.stderr.write(stderr)
+ assert proc.returncode == 0, proc.returncode
+ py_files = find_stdout.split('\n')
+
+ for i, files in enumerate(grouper(40, py_files)):
+ files = filter(None, files)
+
+ lint_test_method = lambda self, these_files=files: run_linter(these_files)
+ lint_test_method.__name__ = 'test_pylint_{}'.format(i)
+ setattr(TestLinters, 'test_pylint_{}'.format(i), lint_test_method)
+
+ pyflake_test_method = lambda self, these_files=files: run_pyflakes(these_files)
+ pyflake_test_method.__name__ = 'test_pyflakes_{}'.format(i)
+ setattr(TestLinters, 'test_pyflakes_{}'.format(i), pyflake_test_method)
+
+create_many_lint_methods()