You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/05/16 20:53:16 UTC
[1/3] git commit: [#6192] ticket:335 Add --skip-solr-delete option to
ReindexCommand
Updated Branches:
refs/heads/master 69f660068 -> fbf88bb0c
[#6192] ticket:335 Add --skip-solr-delete option to ReindexCommand
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/94ee729f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/94ee729f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/94ee729f
Branch: refs/heads/master
Commit: 94ee729f4ebc26ab8079a76626d98922f84169c6
Parents: 69f6600
Author: Igor Bondarenko <je...@gmail.com>
Authored: Tue May 7 07:10:46 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu May 16 18:44:22 2013 +0000
----------------------------------------------------------------------
Allura/allura/command/show_models.py | 4 +++-
Allura/allura/tests/test_commands.py | 12 ++++++++++++
2 files changed, 15 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/94ee729f/Allura/allura/command/show_models.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index 890805d..ef8767f 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -57,6 +57,8 @@ class ReindexCommand(base.Command):
parser.add_option('--solr', action='store_true', dest='solr',
help='Solr needs artifact references to already exist.')
+ parser.add_option('--skip-solr-delete', action='store_true', dest='skip_solr_delete',
+ help='Skip clearing solr index.')
parser.add_option('--refs', action='store_true', dest='refs',
help='Update artifact references and shortlinks')
parser.add_option('--tasks', action='store_true', dest='tasks',
@@ -86,7 +88,7 @@ class ReindexCommand(base.Command):
c.project = p
base.log.info('Reindex project %s', p.shortname)
# Clear index for this project
- if self.options.solr:
+ if self.options.solr and not self.options.skip_solr_delete:
g.solr.delete(q='project_id_s:%s' % p._id)
if self.options.refs:
M.ArtifactReference.query.remove({'artifact_reference.project_id':p._id})
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/94ee729f/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 06524c3..8a1463f 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -338,3 +338,15 @@ class TestBackgroundCommand(object):
command.__name__ = 'ReindexCommand'
base.run_command(self.cmd, 'dev.ini -p "project 3"')
command(command.__name__).run.assert_called_with(['dev.ini', '-p', 'project 3'])
+
+
+class TestReindexCommand(object):
+
+ @patch('allura.command.show_models.g')
+ def test_skip_solr_delete(self, g):
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.run([test_config, '-p', 'test', '--solr'])
+ assert g.solr.delete.called, 'solr.delete() must be called'
+ g.solr.delete.reset_mock()
+ cmd.run([test_config, '-p', 'test', '--solr', '--skip-solr-delete'])
+ assert not g.solr.delete.called, 'solr.delete() must not be called'
[3/3] git commit: [#6192] ticket:336 Split add_artifacts tasks into
smaller sizes
Posted by br...@apache.org.
[#6192] ticket:336 Split add_artifacts tasks into smaller sizes
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/fbf88bb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/fbf88bb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/fbf88bb0
Branch: refs/heads/master
Commit: fbf88bb0c4be366b2a25191a32999ca15f389d7f
Parents: 74dfc19
Author: Igor Bondarenko <je...@gmail.com>
Authored: Wed May 8 09:58:57 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu May 16 18:44:24 2013 +0000
----------------------------------------------------------------------
Allura/allura/command/show_models.py | 39 +++++++++++++++++++----
Allura/allura/tests/test_commands.py | 47 +++++++++++++++++++++++++++++
2 files changed, 79 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/fbf88bb0/Allura/allura/command/show_models.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index 17d4b7f..c4e49d1 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -20,12 +20,12 @@ from collections import defaultdict
from itertools import groupby
from pylons import tmpl_context as c, app_globals as g
-from pymongo.errors import DuplicateKeyError
+from pymongo.errors import DuplicateKeyError, InvalidDocument
from ming.orm import mapper, session, Mapper
from ming.orm.declarative import MappedClass
-import allura.tasks.index_tasks
+from allura.tasks.index_tasks import add_artifacts
from allura.lib.exceptions import CompoundError
from allura.lib import utils
from . import base
@@ -119,12 +119,12 @@ class ReindexCommand(base.Command):
M.main_orm_session.flush()
M.artifact_orm_session.clear()
try:
- add_artifacts = allura.tasks.index_tasks.add_artifacts
if self.options.tasks:
- add_artifacts = add_artifacts.post
- add_artifacts(ref_ids,
- update_solr=self.options.solr,
- update_refs=self.options.refs)
+ self._chunked_add_artifacts(ref_ids)
+ else:
+ add_artifacts(ref_ids,
+ update_solr=self.options.solr,
+ update_refs=self.options.refs)
except CompoundError, err:
base.log.exception('Error indexing artifacts:\n%r', err)
base.log.error('%s', err.format_error())
@@ -132,6 +132,31 @@ class ReindexCommand(base.Command):
M.main_orm_session.clear()
base.log.info('Reindex %s', 'queued' if self.options.tasks else 'done')
+ def _chunked_add_artifacts(self, ref_ids):
+ # ref_ids contains solr index ids which can easily be over
+ # 100 bytes. Here we allow for 160 bytes avg, plus
+ # room for other document overhead.
+ for chunk in utils.chunked_list(ref_ids, 100 * 1000):
+ self._post_add_artifacts(chunk)
+
+ def _post_add_artifacts(self, chunk):
+ """
+ Post task, recursively splitting and re-posting if the resulting
+ mongo document is too large.
+ """
+ try:
+ add_artifacts.post(chunk,
+ update_solr=self.options.solr,
+ update_refs=self.options.refs)
+ except InvalidDocument as e:
+ # there are many types of InvalidDocument, only recurse if its expected to help
+ if str(e).startswith('BSON document too large'):
+ self._post_add_artifacts(chunk[:len(chunk) // 2])
+ self._post_add_artifacts(chunk[len(chunk) // 2:])
+ else:
+ raise
+
+
class EnsureIndexCommand(base.Command):
min_args=1
max_args=1
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/fbf88bb0/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index bfaf350..5f4963c 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -19,6 +19,7 @@ from nose.tools import assert_raises, assert_in
from datadiff.tools import assert_equal
from ming.orm import ThreadLocalORMSession
from mock import Mock, call, patch
+import pymongo
from alluratest.controller import setup_basic_test, setup_global_objects
from allura.command import base, script, set_neighborhood_features, \
@@ -26,6 +27,7 @@ from allura.command import base, script, set_neighborhood_features, \
from allura import model as M
from forgeblog import model as BM
from allura.lib.exceptions import InvalidNBFeatureValueError
+from allura.tests import decorators as td
test_config = 'test.ini#main'
@@ -357,3 +359,48 @@ class TestReindexCommand(object):
cmd.run([test_config, '--project-regex', '^test'])
utils.chunked_find.assert_called_once_with(
M.Project, {'shortname': {'$regex': '^test'}})
+
+ @patch('allura.command.show_models.add_artifacts')
+ def test_chunked_add_artifacts(self, add_artifacts):
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.options = Mock()
+ ref_ids = list(range(100 * 1000 * 2 + 20))
+ cmd._chunked_add_artifacts(ref_ids)
+ assert_equal(len(add_artifacts.post.call_args_list), 3)
+ assert_equal(len(add_artifacts.post.call_args_list[0][0][0]), 100 * 1000)
+ assert_equal(len(add_artifacts.post.call_args_list[1][0][0]), 100 * 1000)
+ assert_equal(len(add_artifacts.post.call_args_list[2][0][0]), 20)
+
+ @patch('allura.command.show_models.add_artifacts')
+ def test_post_add_artifacts_too_large(self, add_artifacts):
+ def on_post(chunk, **kw):
+ if len(chunk) > 1:
+ raise pymongo.errors.InvalidDocument(
+ "BSON document too large (16906035 bytes) - the connected server supports BSON document sizes up to 16777216 bytes.")
+ add_artifacts.post.side_effect = on_post
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.options = Mock()
+ cmd._post_add_artifacts(range(5))
+ kw = {'update_solr': cmd.options.solr, 'update_refs': cmd.options.refs}
+ expected = [
+ call([0, 1, 2, 3, 4], **kw),
+ call([0, 1], **kw),
+ call([0], **kw),
+ call([1], **kw),
+ call([2, 3, 4], **kw),
+ call([2], **kw),
+ call([3, 4], **kw),
+ call([3], **kw),
+ call([4], **kw)
+ ]
+ assert_equal(expected, add_artifacts.post.call_args_list)
+
+ @patch('allura.command.show_models.add_artifacts')
+ def test_post_add_artifacts_other_error(self, add_artifacts):
+ def on_post(chunk, **kw):
+ raise pymongo.errors.InvalidDocument("Cannot encode object...")
+ add_artifacts.post.side_effect = on_post
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.options = Mock()
+ with td.raises(pymongo.errors.InvalidDocument):
+ cmd._post_add_artifacts(range(5))
[2/3] git commit: [#6192] ticket:335 Add --project-regex option to
ReindexCommand
Posted by br...@apache.org.
[#6192] ticket:335 Add --project-regex option to ReindexCommand
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/74dfc196
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/74dfc196
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/74dfc196
Branch: refs/heads/master
Commit: 74dfc19607d969046edd0247cc26350b15e345f9
Parents: 94ee729
Author: Igor Bondarenko <je...@gmail.com>
Authored: Tue May 7 07:32:34 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu May 16 18:44:24 2013 +0000
----------------------------------------------------------------------
Allura/allura/command/show_models.py | 5 +++++
Allura/allura/tests/test_commands.py | 7 +++++++
2 files changed, 12 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/74dfc196/Allura/allura/command/show_models.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index ef8767f..17d4b7f 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -52,6 +52,9 @@ class ReindexCommand(base.Command):
parser = base.Command.standard_parser(verbose=True)
parser.add_option('-p', '--project', dest='project', default=None,
help='project to reindex')
+ parser.add_option('--project-regex', dest='project_regex', default='',
+ help='Restrict reindex to projects for which the shortname matches '
+ 'the provided regex.')
parser.add_option('-n', '--neighborhood', dest='neighborhood', default=None,
help='neighborhood to reindex (e.g. p)')
@@ -72,6 +75,8 @@ class ReindexCommand(base.Command):
graph = build_model_inheritance_graph()
if self.options.project:
q_project = dict(shortname=self.options.project)
+ elif self.options.project_regex:
+ q_project = dict(shortname={'$regex': self.options.project_regex})
elif self.options.neighborhood:
neighborhood_id = M.Neighborhood.query.get(
url_prefix='/%s/' % self.options.neighborhood)._id
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/74dfc196/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index 8a1463f..bfaf350 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -350,3 +350,10 @@ class TestReindexCommand(object):
g.solr.delete.reset_mock()
cmd.run([test_config, '-p', 'test', '--solr', '--skip-solr-delete'])
assert not g.solr.delete.called, 'solr.delete() must not be called'
+
+ @patch('allura.command.show_models.utils')
+ def test_project_regex(self, utils):
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.run([test_config, '--project-regex', '^test'])
+ utils.chunked_find.assert_called_once_with(
+ M.Project, {'shortname': {'$regex': '^test'}})