You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/09/13 05:05:20 UTC
[1/2] git commit: [#6540] bump GoogleCodeWikiImporter and
TracWikiImporter versions for their rate-limiting additions
Updated Branches:
refs/heads/master cd1be45da -> b6776afe3
[#6540] bump GoogleCodeWikiImporter and TracWikiImporter versions for their rate-limiting additions
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/b6776afe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/b6776afe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/b6776afe
Branch: refs/heads/master
Commit: b6776afe3b6cdba86a7a75648044279b40476fdc
Parents: 3c0a7e0
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Thu Sep 12 19:56:33 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Sep 12 19:56:41 2013 +0000
----------------------------------------------------------------------
requirements-sf.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b6776afe/requirements-sf.txt
----------------------------------------------------------------------
diff --git a/requirements-sf.txt b/requirements-sf.txt
index ecad4a2..071e528 100644
--- a/requirements-sf.txt
+++ b/requirements-sf.txt
@@ -6,7 +6,7 @@ kombu==1.0.4
coverage==3.5a1-20110413
ForgeHg==0.1.16
ForgePastebin==0.2.7
-GoogleCodeWikiImporter==0.3.3
+GoogleCodeWikiImporter==0.4.3
mechanize==0.2.4
mercurial==1.4.3
MySQL-python==1.2.3c1
@@ -20,7 +20,7 @@ wsgipreload==1.2
pyzmq==2.1.7
html2text==3.200.3dev-20121112
PyMollom==0.1
-TracWikiImporter==0.2.2
+TracWikiImporter==0.3.2
# use version built from https://github.com/johnsca/GitPython/commits/tv/6000
# for unmerged fixes for [#5411], [#6000], and [#6078]
[2/2] git commit: [#6540] Added rate limiting on tool imports
Posted by br...@apache.org.
[#6540] Added rate limiting on tool imports
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3c0a7e0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3c0a7e0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3c0a7e0f
Branch: refs/heads/master
Commit: 3c0a7e0f5fd179096d48239e3db8c014960a2181
Parents: cd1be45
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Sep 9 22:35:20 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Sep 12 19:56:41 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 44 +++++++++++++++++++-
ForgeImporters/forgeimporters/github/code.py | 19 +++++----
.../forgeimporters/github/tests/test_code.py | 20 +++++++++
ForgeImporters/forgeimporters/google/code.py | 17 ++++----
.../forgeimporters/google/tests/test_code.py | 18 ++++++++
ForgeImporters/forgeimporters/google/tracker.py | 19 +++++----
.../forgeimporters/tests/google/test_tracker.py | 20 ++++++++-
.../forgeimporters/trac/tests/test_tickets.py | 18 ++++++++
ForgeImporters/forgeimporters/trac/tickets.py | 19 +++++----
9 files changed, 160 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index ffb1650..d7479b3 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -68,14 +68,16 @@ class ToolImportForm(schema.Schema):
class ImportErrorHandler(object):
- def __init__(self, importer, project_name):
+ def __init__(self, importer, project_name, project):
self.importer = importer
self.project_name = project_name
+ self.project = project
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
+ self.importer.clear_pending(self.project)
if exc_type:
g.post_event('import_tool_task_failed',
error=str(exc_val),
@@ -89,7 +91,7 @@ class ImportErrorHandler(object):
@task(notifications_disabled=True)
def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
importer = ToolImporter.by_name(importer_name)
- with ImportErrorHandler(importer, project_name):
+ with ImportErrorHandler(importer, project_name, c.project):
importer.import_tool(c.project, c.user, project_name=project_name,
mount_point=mount_point, mount_label=mount_label, **kw)
@@ -326,6 +328,44 @@ class ToolImporter(object):
importers[ep.name] = importer()
return importers
+ @property
+ def classname(self):
+ return self.__class__.__name__
+
+ def enforce_limit(self, project):
+ """
+ Enforce rate limiting of tool imports on a given project.
+
+ Returns False if limit is met / exceeded. Otherwise, increments the
+ count of pending / in-progress imports and returns True.
+ """
+ limit = config.get('tool_import.rate_limit', 1)
+ pending_key = 'tool_data.%s.pending' % self.classname
+ modified_project = M.Project.query.find_and_modify(
+ query={
+ '_id': project._id,
+ '$or': [
+ {pending_key: None},
+ {pending_key: {'$lt': limit}},
+ ],
+ },
+ update={'$inc': {pending_key: 1}},
+ new=True,
+ )
+ return modified_project is not None
+
+ def clear_pending(self, project):
+ """
+ Decrement the pending counter for this importer on the given project,
+ to indicate that an import is complete.
+ """
+ pending_key = 'tool_data.%s.pending' % self.classname
+ M.Project.query.find_and_modify(
+ query={'_id': project._id},
+ update={'$inc': {pending_key: -1}},
+ new=True,
+ )
+
def import_tool(self, project, user, project_name=None,
mount_point=None, mount_label=None, **kw):
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/github/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/code.py b/ForgeImporters/forgeimporters/github/code.py
index 92e5626..aa5fe7d 100644
--- a/ForgeImporters/forgeimporters/github/code.py
+++ b/ForgeImporters/forgeimporters/github/code.py
@@ -45,7 +45,7 @@ from forgeimporters.github import GitHubProjectExtractor
@task(notifications_disabled=True)
def import_tool(**kw):
importer = GitHubRepoImporter()
- with ImportErrorHandler(importer, kw.get('project_name')):
+ with ImportErrorHandler(importer, kw.get('project_name'), c.project):
importer.import_tool(c.project, c.user, **kw)
@@ -73,13 +73,16 @@ class GitHubRepoImportController(BaseController):
@require_post()
@validate(GitHubRepoImportForm(ForgeGitApp), error_handler=index)
def create(self, gh_project_name, gh_user_name, mount_point, mount_label, **kw):
- import_tool.post(
- project_name=gh_project_name,
- user_name=gh_user_name,
- mount_point=mount_point,
- mount_label=mount_label)
- flash('Repo import has begun. Your new repo will be available '
- 'when the import is complete.')
+ if GitHubRepoImporter().enforce_limit(c.project):
+ import_tool.post(
+ project_name=gh_project_name,
+ user_name=gh_user_name,
+ mount_point=mount_point,
+ mount_label=mount_label)
+ flash('Repo import has begun. Your new repo will be available '
+ 'when the import is complete.')
+ else:
+ flash('There are too many imports pending at this time. Please wait and try again.', 'error')
redirect(c.project.url() + 'admin/')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/github/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_code.py b/ForgeImporters/forgeimporters/github/tests/test_code.py
index 09afa87..729d8c5 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_code.py
@@ -17,9 +17,11 @@
from unittest import TestCase
from mock import Mock, patch
+from ming.odm import ThreadLocalORMSession
from allura.tests import TestController
from allura.tests.decorators import with_tool
+from allura import model as M
from forgeimporters.github.code import GitHubRepoImporter
@@ -76,3 +78,21 @@ class TestGitHubImportController(TestController, TestCase):
self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name'])
self.assertEqual(u'spooky', import_tool.post.call_args[1]['user_name'])
+
+ @with_git
+ @patch('forgeimporters.github.code.import_tool')
+ def test_create_limit(self, import_tool):
+ project = M.Project.query.get(shortname=test_project_with_repo)
+ project.set_tool_data('GitHubRepoImporter', pending=1)
+ ThreadLocalORMSession.flush_all()
+ params = dict(
+ gh_user_name='spooky',
+ gh_project_name='poop',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo),
+ params,
+ status=302).follow()
+ self.assertIn('Please wait and try again', r)
+ self.assertEqual(import_tool.post.call_count, 0)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/google/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py
index 134ecb6..f8e42b3 100644
--- a/ForgeImporters/forgeimporters/google/code.py
+++ b/ForgeImporters/forgeimporters/google/code.py
@@ -88,7 +88,7 @@ def get_repo_class(type_):
@task(notifications_disabled=True)
def import_tool(**kw):
importer = GoogleRepoImporter()
- with ImportErrorHandler(importer, kw.get('project_name')):
+ with ImportErrorHandler(importer, kw.get('project_name'), c.project):
importer.import_tool(c.project, c.user, **kw)
@@ -140,12 +140,15 @@ class GoogleRepoImportController(BaseController):
@require_post()
@validate(GoogleRepoImportForm(), error_handler=index)
def create(self, gc_project_name, mount_point, mount_label, **kw):
- import_tool.post(
- project_name=gc_project_name,
- mount_point=mount_point,
- mount_label=mount_label)
- flash('Repo import has begun. Your new repo will be available '
- 'when the import is complete.')
+ if GoogleRepoImporter().enforce_limit(c.project):
+ import_tool.post(
+ project_name=gc_project_name,
+ mount_point=mount_point,
+ mount_label=mount_label)
+ flash('Repo import has begun. Your new repo will be available '
+ 'when the import is complete.')
+ else:
+ flash('There are too many imports pending at this time. Please wait and try again.', 'error')
redirect(c.project.url() + 'admin/')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/google/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tests/test_code.py b/ForgeImporters/forgeimporters/google/tests/test_code.py
index 806a004..96dd687 100644
--- a/ForgeImporters/forgeimporters/google/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/google/tests/test_code.py
@@ -17,9 +17,11 @@
from unittest import TestCase
from mock import Mock, patch
+from ming.odm import ThreadLocalORMSession
from allura.tests import TestController
from allura.tests.decorators import with_tool
+from allura import model as M
# important to be distinct from 'test' which ForgeSVN uses, so that the tests can run in parallel and not clobber each other
@@ -113,3 +115,19 @@ class TestGoogleRepoImportController(TestController, TestCase):
self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point'])
self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name'])
+
+ @with_svn
+ @patch('forgeimporters.google.code.import_tool')
+ def test_create_limit(self, import_tool):
+ project = M.Project.query.get(shortname=test_project_with_repo)
+ project.set_tool_data('GoogleRepoImporter', pending=1)
+ ThreadLocalORMSession.flush_all()
+ params = dict(gc_project_name='poop',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/{}/admin/src/_importer/create'.format(test_project_with_repo),
+ params,
+ status=302).follow()
+ self.assertIn('Please wait and try again', r)
+ self.assertEqual(import_tool.post.call_count, 0)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/google/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tracker.py b/ForgeImporters/forgeimporters/google/tracker.py
index 9c259dd..d52945e 100644
--- a/ForgeImporters/forgeimporters/google/tracker.py
+++ b/ForgeImporters/forgeimporters/google/tracker.py
@@ -54,7 +54,7 @@ from forgeimporters.base import (
@task(notifications_disabled=True)
def import_tool(**kw):
importer = GoogleCodeTrackerImporter()
- with ImportErrorHandler(importer, kw.get('project_name')):
+ with ImportErrorHandler(importer, kw.get('project_name'), c.project):
importer.import_tool(c.project, c.user, **kw)
@@ -81,13 +81,16 @@ class GoogleCodeTrackerImportController(BaseController):
@require_post()
@validate(GoogleCodeTrackerImportForm(ForgeTrackerApp), error_handler=index)
def create(self, gc_project_name, mount_point, mount_label, **kw):
- import_tool.post(
- project_name=gc_project_name,
- mount_point=mount_point,
- mount_label=mount_label,
- )
- flash('Ticket import has begun. Your new tracker will be available '
- 'when the import is complete.')
+ if GoogleCodeTrackerImporter().enforce_limit(c.project):
+ import_tool.post(
+ project_name=gc_project_name,
+ mount_point=mount_point,
+ mount_label=mount_label,
+ )
+ flash('Ticket import has begun. Your new tracker will be available '
+ 'when the import is complete.')
+ else:
+ flash('There are too many imports pending at this time. Please wait and try again.', 'error')
redirect(c.project.url() + 'admin/')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/tests/google/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
index d795bab..98216f9 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
@@ -17,13 +17,16 @@
from datetime import datetime
from unittest import TestCase
+
import mock
from mock import patch
+from ming.odm import ThreadLocalORMSession
from allura.tests import TestController
from allura.tests.decorators import with_tracker
-from ...google import tracker
+from allura import model as M
+from forgeimporters.google import tracker
class TestTrackerImporter(TestCase):
@@ -299,3 +302,18 @@ class TestGoogleCodeTrackerImportController(TestController, TestCase):
self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point'])
self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
self.assertEqual(u'test', import_tool.post.call_args[1]['project_name'])
+
+ @with_tracker
+ @patch('forgeimporters.google.tracker.import_tool')
+ def test_create_limit(self, import_tool):
+ project = M.Project.query.get(shortname='test')
+ project.set_tool_data('GoogleCodeTrackerImporter', pending=1)
+ ThreadLocalORMSession.flush_all()
+ params = dict(gc_project_name='test',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ status=302).follow()
+ self.assertIn('Please wait and try again', r)
+ self.assertEqual(import_tool.post.call_count, 0)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 92e7854..b3d2fcb 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -18,12 +18,14 @@
import json
from unittest import TestCase
from mock import Mock, patch
+from ming.orm import ThreadLocalORMSession
from tg import config
from allura.tests import TestController
from allura.tests.decorators import with_tracker
+from allura import model as M
from forgeimporters.trac.tickets import (
TracTicketImporter,
TracTicketImportController,
@@ -132,3 +134,19 @@ class TestTracTicketImportController(TestController, TestCase):
self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
self.assertEqual('{"orig_user": "new_user"}', import_tool.post.call_args[1]['user_map'])
self.assertEqual(u'http://example.com/trac/url', import_tool.post.call_args[1]['trac_url'])
+
+ @with_tracker
+ @patch('forgeimporters.trac.tickets.import_tool')
+ def test_create_limit(self, import_tool):
+ project = M.Project.query.get(shortname='test')
+ project.set_tool_data('TracTicketImporter', pending=1)
+ ThreadLocalORMSession.flush_all()
+ params = dict(trac_url='http://example.com/trac/url',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ upload_files=[('user_map', 'myfile', '{"orig_user": "new_user"}')],
+ status=302).follow()
+ self.assertIn('Please wait and try again', r)
+ self.assertEqual(import_tool.post.call_count, 0)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c0a7e0f/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 9e4f493..cafd403 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -61,7 +61,7 @@ from forgetracker.scripts.import_tracker import import_tracker
@task(notifications_disabled=True)
def import_tool(**kw):
importer = TracTicketImporter()
- with ImportErrorHandler(importer, kw.get('trac_url')):
+ with ImportErrorHandler(importer, kw.get('trac_url'), c.project):
importer.import_tool(c.project, c.user, **kw)
@@ -89,13 +89,16 @@ class TracTicketImportController(BaseController):
@require_post()
@validate(TracTicketImportForm(ForgeTrackerApp), error_handler=index)
def create(self, trac_url, mount_point, mount_label, user_map=None, **kw):
- import_tool.post(
- mount_point=mount_point,
- mount_label=mount_label,
- trac_url=trac_url,
- user_map=user_map)
- flash('Ticket import has begun. Your new tracker will be available '
- 'when the import is complete.')
+ if TracTicketImporter().enforce_limit(c.project):
+ import_tool.post(
+ mount_point=mount_point,
+ mount_label=mount_label,
+ trac_url=trac_url,
+ user_map=user_map)
+ flash('Ticket import has begun. Your new tracker will be available '
+ 'when the import is complete.')
+ else:
+ flash('There are too many imports pending at this time. Please wait and try again.', 'error')
redirect(c.project.url() + 'admin/')