You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2013/06/08 01:42:19 UTC
[24/27] git commit: [#6276] add --solr-hosts option to reindex cmd
[#6276] add --solr-hosts option to reindex cmd
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/afe5b380
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/afe5b380
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/afe5b380
Branch: refs/heads/db/6276
Commit: afe5b3802dd76ada987cc2b731b92dc3feeb84b1
Parents: 577b65f
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Tue Jun 4 16:44:17 2013 +0000
Committer: Tim Van Steenburgh <tv...@gmail.com>
Committed: Fri Jun 7 21:19:38 2013 +0000
----------------------------------------------------------------------
Allura/allura/command/show_models.py | 17 +++++++++++++----
Allura/allura/lib/app_globals.py | 14 +++++---------
Allura/allura/lib/solr.py | 19 +++++++++++++++++--
Allura/allura/tasks/index_tasks.py | 16 +++++++++++++---
Allura/allura/tests/test_commands.py | 18 ++++++++++++++++++
5 files changed, 66 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/afe5b380/Allura/allura/command/show_models.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index c4e49d1..68a55ac 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -68,6 +68,8 @@ class ReindexCommand(base.Command):
help='Run each individual index operation as a background task. '
'Note: this is often better, since tasks have "request" objects '
'which are needed for some markdown macros to run properly')
+ parser.add_option('--solr-hosts', dest='solr_hosts',
+ help='Override the solr host(s) to post to. Comma-separated list of solr server URLs')
def command(self):
from allura import model as M
@@ -88,6 +90,11 @@ class ReindexCommand(base.Command):
if not self.options.solr and not self.options.refs:
self.options.solr = self.options.refs = True
+ if self.options.solr_hosts:
+ self.add_artifact_kwargs = {'solr_hosts': self.options.solr_hosts.split(',')}
+ else:
+ self.add_artifact_kwargs = {}
+
for projects in utils.chunked_find(M.Project, q_project):
for p in projects:
c.project = p
@@ -123,8 +130,9 @@ class ReindexCommand(base.Command):
self._chunked_add_artifacts(ref_ids)
else:
add_artifacts(ref_ids,
- update_solr=self.options.solr,
- update_refs=self.options.refs)
+ update_solr=self.options.solr,
+ update_refs=self.options.refs,
+ **self.add_artifact_kwargs)
except CompoundError, err:
base.log.exception('Error indexing artifacts:\n%r', err)
base.log.error('%s', err.format_error())
@@ -146,8 +154,9 @@ class ReindexCommand(base.Command):
"""
try:
add_artifacts.post(chunk,
- update_solr=self.options.solr,
- update_refs=self.options.refs)
+ update_solr=self.options.solr,
+ update_refs=self.options.refs,
+ **self.add_artifact_kwargs)
except InvalidDocument as e:
# there are many types of InvalidDocument, only recurse if its expected to help
if str(e).startswith('BSON document too large'):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/afe5b380/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 26e0854..bdcacf7 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -57,7 +57,7 @@ from allura.lib import helpers as h
from allura.lib.widgets import analytics
from allura.lib.security import Credentials
from allura.lib.async import Connection, MockAMQ
-from allura.lib.solr import Solr, MockSOLR
+from allura.lib.solr import MockSOLR, make_solr_from_config
from allura.lib.zarkov_helpers import ZarkovClient, zmq
log = logging.getLogger(__name__)
@@ -103,14 +103,10 @@ class Globals(object):
if asbool(config.get('solr.mock')):
self.solr = self.solr_short_timeout = MockSOLR()
elif self.solr_server:
- self.solr = Solr(self.solr_server, self.solr_query_server,
- commit=asbool(config.get('solr.commit', True)),
- commitWithin=config.get('solr.commitWithin'),
- timeout=int(config.get('solr.long_timeout', 60)))
- self.solr_short_timeout = Solr(self.solr_server, self.solr_query_server,
- commit=asbool(config.get('solr.commit', True)),
- commitWithin=config.get('solr.commitWithin'),
- timeout=int(config.get('solr.short_timeout', 10)))
+ self.solr = make_solr_from_config(self.solr_server, self.solr_query_server)
+ self.solr_short_timeout = make_solr_from_config(
+ self.solr_server, self.solr_query_server,
+ timeout=int(config.get('solr.short_timeout', 10)))
else: # pragma no cover
self.solr = None
self.solr_short_timeout = None
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/afe5b380/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index e1e9567..4fa0c77 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -16,8 +16,24 @@
# under the License.
import shlex
+
+from tg import config
+from paste.deploy.converters import asbool
import pysolr
-from pysolr import SolrError
+
+
+def make_solr_from_config(push_servers, query_server=None, **kwargs):
+ """
+ Make a :class:`Solr <Solr>` instance from config defaults. Use
+ `**kwargs` to override any value
+ """
+ solr_kwargs = dict(
+ commit=asbool(config.get('solr.commit', True)),
+ commitWithin=config.get('solr.commitWithin'),
+ timeout=int(config.get('solr.long_timeout', 60)),
+ )
+ solr_kwargs.update(kwargs)
+ return Solr(push_servers, query_server, **solr_kwargs)
class Solr(object):
@@ -131,4 +147,3 @@ class MockSOLR(object):
elif kwargs.get('q', None):
for doc in self.search(kwargs['q']):
self.delete(id=doc['id'])
-
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/afe5b380/Allura/allura/tasks/index_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py
index ac5049c..27ada83 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -23,14 +23,24 @@ from pylons import app_globals as g
from allura.lib.decorators import task
from allura.lib.exceptions import CompoundError
+from allura.lib.solr import make_solr_from_config
log = logging.getLogger(__name__)
@task
-def add_artifacts(ref_ids, update_solr=True, update_refs=True):
- '''Add the referenced artifacts to SOLR and shortlinks'''
+def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
+ '''
+ Add the referenced artifacts to SOLR and shortlinks.
+
+ :param solr_hosts: a list of solr hosts to use instead of the defaults
+ :type solr_hosts: [str]
+ '''
from allura import model as M
from allura.lib.search import find_shortlinks, solarize
+ if solr_hosts:
+ solr = make_solr_from_config(solr_hosts)
+ else:
+ solr = g.solr
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
@@ -50,7 +60,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True):
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
- g.solr.add(solr_updates)
+ solr.add(solr_updates)
if len(exceptions) == 1:
raise exceptions[0][0], exceptions[0][1], exceptions[0][2]
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/afe5b380/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index fee6e21..1d72ff6 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -362,6 +362,21 @@ class TestReindexCommand(object):
cmd.run([test_config, '-p', 'test', '--solr', '--skip-solr-delete'])
assert not g.solr.delete.called, 'solr.delete() must not be called'
+ @patch('pysolr.Solr')
+ def test_solr_hosts_1(self, Solr):
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge'])
+ assert_equal(Solr.call_args[0][0], 'http://blah.com/solr/forge')
+
+ @patch('pysolr.Solr')
+ def test_solr_hosts_list(self, Solr):
+ cmd = show_models.ReindexCommand('reindex')
+ cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge,https://other.net/solr/forge'])
+ # check constructors of first and second Solr() instantiations
+ assert_equal(set([Solr.call_args_list[0][0][0], Solr.call_args_list[1][0][0]]),
+ set(['http://blah.com/solr/forge', 'https://other.net/solr/forge'])
+ )
+
@patch('allura.command.show_models.utils')
def test_project_regex(self, utils):
cmd = show_models.ReindexCommand('reindex')
@@ -373,6 +388,7 @@ class TestReindexCommand(object):
def test_chunked_add_artifacts(self, add_artifacts):
cmd = show_models.ReindexCommand('reindex')
cmd.options = Mock()
+ cmd.add_artifact_kwargs = {}
ref_ids = list(range(100 * 1000 * 2 + 20))
cmd._chunked_add_artifacts(ref_ids)
assert_equal(len(add_artifacts.post.call_args_list), 3)
@@ -389,6 +405,7 @@ class TestReindexCommand(object):
add_artifacts.post.side_effect = on_post
cmd = show_models.ReindexCommand('reindex')
cmd.options = Mock()
+ cmd.add_artifact_kwargs = {}
cmd._post_add_artifacts(range(5))
kw = {'update_solr': cmd.options.solr, 'update_refs': cmd.options.refs}
expected = [
@@ -411,5 +428,6 @@ class TestReindexCommand(object):
add_artifacts.post.side_effect = on_post
cmd = show_models.ReindexCommand('reindex')
cmd.options = Mock()
+ cmd.add_artifact_kwargs = {}
with td.raises(pymongo.errors.InvalidDocument):
cmd._post_add_artifacts(range(5))